// Package doc parses a file written in the conventions language: its front // matter, its rules and their blocks. // // The parsing model is taken straight from the language. A rule is a heading of // the form `### -. `; the area of a rule runs from that // heading to the next heading of any level. Inside the area text belongs to the // last block opened: a mark opens a block, and the block lasts until the next // mark or until the end of the area. Prose is what lies outside rule areas. // // The boundary is counted by the markup rather than by a judgement about where // a rule ended: that is exactly what makes the "no modal words outside rules" // check implementable at all. package doc import ( "fmt" "os" "regexp" "strconv" "strings" "git.vakhrushev.me/av/convy/internal/lang" ) // Heading is a heading of any level. type Heading struct { Level int Text string Line int } // BlockKind tells a norm block from a marked one. type BlockKind int const ( // Norm is a block of the norm, opened by a modal word. Norm BlockKind = iota + 1 // Marked is a block under a mark: rationale, examples, retired, // mechanized. Marked ) // Block is a part of a rule opened by a vocabulary word at the start of a // paragraph. type Block struct { Kind BlockKind Word string Level lang.Level Mark lang.Mark // Start is the line the opening mark stands on. Start int // End is the last line of the block: a block lasts until the next mark // or until the end of the rule area. End int // Rest is the text of the paragraph after the mark. Rest string } // Rule is a rule: a heading carrying an identifier, plus its area. type Rule struct { Prefix string Num int Title string // Line is the line of the heading. Line int // HeadingLevel is the level of the heading; the canonical form is three. HeadingLevel int // Malformed is set when a heading was recognized as a rule but is not // written in the form `### <PREFIX>-<number>. <title>`. Malformed string // Start and End bound the rule area: from the line after the heading to // the line before the next heading, inclusive. Start, End int Blocks []Block } // ID returns the identifier of the rule. func (r Rule) ID() string { return fmt.Sprintf("%s-%d", r.Prefix, r.Num) } // Block finds the first block under the given mark. func (r Rule) Block(m lang.Mark) (Block, bool) { for _, b := range r.Blocks { if b.Kind == Marked && b.Mark == m { return b, true } } return Block{}, false } // Norms lists the norm blocks. There must be exactly zero of them (for a // retired rule) or one: a norm is a single statement, and two norms under one // number leave no way to address either on its own. func (r Rule) Norms() []Block { var out []Block for _, b := range r.Blocks { if b.Kind == Norm { out = append(out, b) } } return out } // Paragraph is a paragraph: the lines between blank ones. type Paragraph struct { Start, End int Lines []string } // Text joins the paragraph into a single string. func (p Paragraph) Text() string { return strings.Join(p.Lines, " ") } // Document is a parsed file. type Document struct { // Path is the path from the root of the suite, in slash form. Path string Front Front lines []string fence []bool // Body is the first line of the body, past the front matter. Body int Headings []Heading Rules []Rule } var ( headingRe = regexp.MustCompile(`^(#{1,6})\s+(.*)$`) // ruleHeadRe catches a heading that starts with a rule identifier — // including one written out of form: otherwise a typo in a heading would // turn a rule into prose and vanish from the numbering unnoticed. ruleHeadRe = regexp.MustCompile(`^([A-Z]{4})-(\d+)(.*)$`) fenceRe = regexp.MustCompile("^\\s*(`{3,}|~{3,})") ) // Load reads and parses a file. path is the path from the root of the suite, // name the path in the file system. func Load(path, name string) (*Document, error) { data, err := os.ReadFile(name) if err != nil { return nil, err } return Parse(path, string(data)) } // Parse parses the contents of a file. func Parse(path, content string) (*Document, error) { d := &Document{Path: path} d.lines = strings.Split(strings.ReplaceAll(content, "\r\n", "\n"), "\n") front, body, err := parseFront(d.lines) d.Front = front d.Body = body if err != nil { return d, fmt.Errorf("%s: front matter: %w", path, err) } d.markFences() d.collectHeadings() d.collectRules() return d, nil } // markFences marks the lines inside fenced code blocks. Everything checked by // parsing text skips them: a SQL sample with an uppercase WHEN does not make // the suite bilingual, and a `### XKEY-5` inside a documentation sample is not // a rule. func (d *Document) markFences() { d.fence = make([]bool, len(d.lines)) open := "" for i, line := range d.lines { m := fenceRe.FindStringSubmatch(line) if open == "" { if m != nil { open = m[1] d.fence[i] = true } continue } d.fence[i] = true if m != nil && len(m[1]) >= len(open) && m[1][0] == open[0] { open = "" } } } func (d *Document) collectHeadings() { for i, line := range d.lines { num := i + 1 if num < d.Body || d.fence[i] { continue } m := headingRe.FindStringSubmatch(line) if m == nil { continue } d.Headings = append(d.Headings, Heading{ Level: len(m[1]), Text: strings.TrimSpace(m[2]), Line: num, }) } } func (d *Document) collectRules() { for i, h := range d.Headings { m := ruleHeadRe.FindStringSubmatch(h.Text) if m == nil { continue } num, err := strconv.Atoi(m[2]) if err != nil { continue } rule := Rule{ Prefix: m[1], Num: num, Line: h.Line, HeadingLevel: h.Level, Start: h.Line + 1, End: d.Len(), } if i+1 < len(d.Headings) { rule.End = d.Headings[i+1].Line - 1 } tail := m[3] switch { case strings.HasPrefix(tail, ". "): rule.Title = strings.TrimSpace(tail[2:]) case tail == "": rule.Malformed = "the rule heading has no title" case strings.HasPrefix(tail, "."): rule.Title = strings.TrimSpace(tail[1:]) default: rule.Malformed = "no period after the identifier in the heading" rule.Title = strings.TrimSpace(tail) } d.Rules = append(d.Rules, rule) } } // Blocks marks up the rule areas using the suite's vocabulary. The markup is // deferred until the manifest is loaded: before that it is unknown which // vocabulary the suite is written in. func (d *Document) Blocks(v lang.Vocabulary) { for i := range d.Rules { r := &d.Rules[i] r.Blocks = nil for _, p := range d.Paragraphs(r.Start, r.End) { word, rest, ok := boldLead(p.Lines[0]) if !ok { continue } w, ok := v.Lead(word) if !ok { continue } b := Block{Word: w, Start: p.Start, End: r.End, Rest: strings.TrimSpace(rest)} if level, ok := v.Modal(w); ok { b.Kind, b.Level = Norm, level } else { mark, _ := v.Mark(w) b.Kind, b.Mark = Marked, mark } if n := len(r.Blocks); n > 0 { r.Blocks[n-1].End = p.Start - 1 } r.Blocks = append(r.Blocks, b) } } } // boldLead extracts the contents of the first bold span if the paragraph opens // with one. A mark stands first in its paragraph, in bold and with a period — // that is exactly what tells it from a mention of a step mid-sentence. func boldLead(line string) (bold, rest string, ok bool) { line = strings.TrimSpace(line) if !strings.HasPrefix(line, "**") { return "", "", false } end := strings.Index(line[2:], "**") if end < 0 { return "", "", false } return line[2 : 2+end], line[2+end+2:], true } // Paragraphs cuts a range of lines into paragraphs. Bounds are inclusive and // numbering starts at one. Lines inside fenced blocks do not enter paragraphs: // code is an illustration, not the text of a rule. func (d *Document) Paragraphs(from, to int) []Paragraph { var out []Paragraph var cur *Paragraph for n := max(from, 1); n <= min(to, d.Len()); n++ { line := d.lines[n-1] if d.fence[n-1] || strings.TrimSpace(line) == "" { cur = nil continue } if cur == nil { out = append(out, Paragraph{Start: n, End: n}) cur = &out[len(out)-1] } cur.Lines = append(cur.Lines, line) cur.End = n } return out } // Preamble returns the bounds of the introductory prose: from the body to the // first rule. func (d *Document) Preamble() (from, to int) { if len(d.Rules) == 0 { return d.Body, d.Len() } return d.Body, d.Rules[0].Line - 1 } // InRule reports whether a line lies inside the area of some rule. func (d *Document) InRule(n int) bool { for _, r := range d.Rules { if n >= r.Line && n <= r.End { return true } } return false } // Line returns line number n. func (d *Document) Line(n int) string { if n < 1 || n > d.Len() { return "" } return d.lines[n-1] } // Fenced reports whether a line lies inside a fenced code block. func (d *Document) Fenced(n int) bool { return n >= 1 && n <= d.Len() && d.fence[n-1] } // Prose walks the lines of the body that did not land in fenced blocks and // hands them over with the contents of inline code cut out. Inside backticks an // identifier stands as a sample of the notation rather than as a reference — // and it is the markup that tells the two apart. func (d *Document) Prose(yield func(n int, text string) bool) { for n := d.Body; n <= d.Len(); n++ { if d.fence[n-1] { continue } if !yield(n, StripInline(d.lines[n-1])) { return } } } // StripInline cuts out the contents of inline code, keeping the delimiters. func StripInline(line string) string { var b strings.Builder inCode := false for _, r := range line { if r == '`' { inCode = !inCode b.WriteRune(' ') continue } if inCode { b.WriteRune(' ') continue } b.WriteRune(r) } return b.String() } // Len returns the number of lines in the file. func (d *Document) Len() int { return len(d.lines) }