convy/internal/doc/doc.go

// Package doc parses a file written in the conventions language: its front
// matter, its rules and their blocks.
//
// The parsing model is taken straight from the language. A rule is a heading of
// the form `### <PREFIX>-<number>. <title>`; the area of a rule runs from that
// heading to the next heading of any level. Inside the area text belongs to the
// last block opened: a mark opens a block, and the block lasts until the next
// mark or until the end of the area. Prose is what lies outside rule areas.
//
// The boundary is counted by the markup rather than by a judgement about where
// a rule ended: that is exactly what makes the "no modal words outside rules"
// check implementable at all.
package doc

import (
	"fmt"
	"os"
	"regexp"
	"strconv"
	"strings"

	"git.vakhrushev.me/av/convy/internal/lang"
)

// Heading is a heading of any level.
type Heading struct {
	Level int
	Text  string
	Line  int
}

// BlockKind tells a norm block from a marked one.
type BlockKind int

const (
	// Norm is a block of the norm, opened by a modal word.
	Norm BlockKind = iota + 1
	// Marked is a block under a mark: rationale, examples, retired,
	// mechanized.
	Marked
)

// Block is a part of a rule opened by a vocabulary word at the start of a
// paragraph.
type Block struct {
	Kind  BlockKind
	Word  string
	Level lang.Level
	Mark  lang.Mark
	// Start is the line the opening mark stands on.
	Start int
	// End is the last line of the block: a block lasts until the next mark
	// or until the end of the rule area.
	End int
	// Rest is the text of the paragraph after the mark.
	Rest string
}

// Rule is a rule: a heading carrying an identifier, plus its area.
type Rule struct {
	Prefix string
	Num    int
	Title  string
	// Line is the line of the heading.
	Line int
	// HeadingLevel is the level of the heading; the canonical form is three.
	HeadingLevel int
	// Malformed is set when a heading was recognized as a rule but is not
	// written in the form `### <PREFIX>-<number>. <title>`.
	Malformed string
	// Start and End bound the rule area: from the line after the heading to
	// the line before the next heading, inclusive.
	Start, End int
	Blocks     []Block
}

// ID returns the identifier of the rule.
func (r Rule) ID() string {
	return fmt.Sprintf("%s-%d", r.Prefix, r.Num)
}

// Block finds the first block under the given mark.
func (r Rule) Block(m lang.Mark) (Block, bool) {
	for _, b := range r.Blocks {
		if b.Kind == Marked && b.Mark == m {
			return b, true
		}
	}
	return Block{}, false
}

// Norms lists the norm blocks. There must be exactly zero of them (for a
// retired rule) or one: a norm is a single statement, and two norms under one
// number leave no way to address either on its own.
func (r Rule) Norms() []Block {
	var out []Block
	for _, b := range r.Blocks {
		if b.Kind == Norm {
			out = append(out, b)
		}
	}
	return out
}

// Paragraph is a paragraph: the lines between blank ones.
type Paragraph struct {
	Start, End int
	Lines      []string
}

// Text joins the paragraph into a single string.
func (p Paragraph) Text() string {
	return strings.Join(p.Lines, " ")
}

// Document is a parsed file.
type Document struct {
	// Path is the path from the root of the suite, in slash form.
	Path  string
	Front Front
	lines []string
	fence []bool
	// Body is the first line of the body, past the front matter.
	Body     int
	Headings []Heading
	Rules    []Rule
}

var (
	headingRe = regexp.MustCompile(`^(#{1,6})\s+(.*)$`)
	// ruleHeadRe catches a heading that starts with a rule identifier —
	// including one written out of form: otherwise a typo in a heading would
	// turn a rule into prose and vanish from the numbering unnoticed.
	ruleHeadRe = regexp.MustCompile(`^([A-Z]{4})-(\d+)(.*)$`)
	fenceRe    = regexp.MustCompile("^\\s*(`{3,}|~{3,})")
)

// Load reads and parses a file. path is the path from the root of the suite,
// name the path in the file system.
func Load(path, name string) (*Document, error) {
	data, err := os.ReadFile(name)
	if err != nil {
		return nil, err
	}
	return Parse(path, string(data))
}

// Parse parses the contents of a file.
func Parse(path, content string) (*Document, error) {
	d := &Document{Path: path}
	d.lines = strings.Split(strings.ReplaceAll(content, "\r\n", "\n"), "\n")

	front, body, err := parseFront(d.lines)
	d.Front = front
	d.Body = body
	if err != nil {
		return d, fmt.Errorf("%s: front matter: %w", path, err)
	}

	d.markFences()
	d.collectHeadings()
	d.collectRules()
	return d, nil
}

// markFences marks the lines inside fenced code blocks. Everything checked by
// parsing text skips them: a SQL sample with an uppercase WHEN does not make
// the suite bilingual, and a `### XKEY-5` inside a documentation sample is not
// a rule.
func (d *Document) markFences() {
	d.fence = make([]bool, len(d.lines))
	open := ""
	for i, line := range d.lines {
		m := fenceRe.FindStringSubmatch(line)
		if open == "" {
			if m != nil {
				open = m[1]
				d.fence[i] = true
			}
			continue
		}
		d.fence[i] = true
		if m != nil && len(m[1]) >= len(open) && m[1][0] == open[0] {
			open = ""
		}
	}
}

func (d *Document) collectHeadings() {
	for i, line := range d.lines {
		num := i + 1
		if num < d.Body || d.fence[i] {
			continue
		}
		m := headingRe.FindStringSubmatch(line)
		if m == nil {
			continue
		}
		d.Headings = append(d.Headings, Heading{
			Level: len(m[1]),
			Text:  strings.TrimSpace(m[2]),
			Line:  num,
		})
	}
}

func (d *Document) collectRules() {
	for i, h := range d.Headings {
		m := ruleHeadRe.FindStringSubmatch(h.Text)
		if m == nil {
			continue
		}
		num, err := strconv.Atoi(m[2])
		if err != nil {
			continue
		}
		rule := Rule{
			Prefix:       m[1],
			Num:          num,
			Line:         h.Line,
			HeadingLevel: h.Level,
			Start:        h.Line + 1,
			End:          d.Len(),
		}
		if i+1 < len(d.Headings) {
			rule.End = d.Headings[i+1].Line - 1
		}

		tail := m[3]
		switch {
		case strings.HasPrefix(tail, ". "):
			rule.Title = strings.TrimSpace(tail[2:])
		case tail == "":
			rule.Malformed = "the rule heading has no title"
		case strings.HasPrefix(tail, "."):
			rule.Title = strings.TrimSpace(tail[1:])
		default:
			rule.Malformed = "no period after the identifier in the heading"
			rule.Title = strings.TrimSpace(tail)
		}
		d.Rules = append(d.Rules, rule)
	}
}

// Blocks marks up the rule areas using the suite's vocabulary. The markup is
// deferred until the manifest is loaded: before that it is unknown which
// vocabulary the suite is written in.
func (d *Document) Blocks(v lang.Vocabulary) {
	for i := range d.Rules {
		r := &d.Rules[i]
		r.Blocks = nil
		for _, p := range d.Paragraphs(r.Start, r.End) {
			word, rest, ok := boldLead(p.Lines[0])
			if !ok {
				continue
			}
			w, ok := v.Lead(word)
			if !ok {
				continue
			}
			b := Block{Word: w, Start: p.Start, End: r.End, Rest: strings.TrimSpace(rest)}
			if level, ok := v.Modal(w); ok {
				b.Kind, b.Level = Norm, level
			} else {
				mark, _ := v.Mark(w)
				b.Kind, b.Mark = Marked, mark
			}
			if n := len(r.Blocks); n > 0 {
				r.Blocks[n-1].End = p.Start - 1
			}
			r.Blocks = append(r.Blocks, b)
		}
	}
}

// boldLead extracts the contents of the first bold span if the paragraph opens
// with one. A mark stands first in its paragraph, in bold and with a period —
// that is exactly what tells it from a mention of a step mid-sentence.
func boldLead(line string) (bold, rest string, ok bool) {
	line = strings.TrimSpace(line)
	if !strings.HasPrefix(line, "**") {
		return "", "", false
	}
	end := strings.Index(line[2:], "**")
	if end < 0 {
		return "", "", false
	}
	return line[2 : 2+end], line[2+end+2:], true
}

// Paragraphs cuts a range of lines into paragraphs. Bounds are inclusive and
// numbering starts at one. Lines inside fenced blocks do not enter paragraphs:
// code is an illustration, not the text of a rule.
func (d *Document) Paragraphs(from, to int) []Paragraph {
	var out []Paragraph
	var cur *Paragraph
	for n := max(from, 1); n <= min(to, d.Len()); n++ {
		line := d.lines[n-1]
		if d.fence[n-1] || strings.TrimSpace(line) == "" {
			cur = nil
			continue
		}
		if cur == nil {
			out = append(out, Paragraph{Start: n, End: n})
			cur = &out[len(out)-1]
		}
		cur.Lines = append(cur.Lines, line)
		cur.End = n
	}
	return out
}

// Preamble returns the bounds of the introductory prose: from the body to the
// first rule.
func (d *Document) Preamble() (from, to int) {
	if len(d.Rules) == 0 {
		return d.Body, d.Len()
	}
	return d.Body, d.Rules[0].Line - 1
}

// InRule reports whether a line lies inside the area of some rule.
func (d *Document) InRule(n int) bool {
	for _, r := range d.Rules {
		if n >= r.Line && n <= r.End {
			return true
		}
	}
	return false
}

// Line returns line number n.
func (d *Document) Line(n int) string {
	if n < 1 || n > d.Len() {
		return ""
	}
	return d.lines[n-1]
}

// Fenced reports whether a line lies inside a fenced code block.
func (d *Document) Fenced(n int) bool {
	return n >= 1 && n <= d.Len() && d.fence[n-1]
}

// Prose walks the lines of the body that did not land in fenced blocks and
// hands them over with the contents of inline code cut out. Inside backticks an
// identifier stands as a sample of the notation rather than as a reference —
// and it is the markup that tells the two apart.
func (d *Document) Prose(yield func(n int, text string) bool) {
	for n := d.Body; n <= d.Len(); n++ {
		if d.fence[n-1] {
			continue
		}
		if !yield(n, StripInline(d.lines[n-1])) {
			return
		}
	}
}

// StripInline cuts out the contents of inline code, keeping the delimiters.
func StripInline(line string) string {
	var b strings.Builder
	inCode := false
	for _, r := range line {
		if r == '`' {
			inCode = !inCode
			b.WriteRune(' ')
			continue
		}
		if inCode {
			b.WriteRune(' ')
			continue
		}
		b.WriteRune(r)
	}
	return b.String()
}

// Len returns the number of lines in the file.
func (d *Document) Len() int { return len(d.lines) }

// LocalMarker is the boundary inside a copy between what the suite wrote and
// what the consuming repository wrote. It is the one piece of markup inside a
// convention that means something to the tool, and it is single and nameless,
// so there is no name to be orphaned by a rename.
//
// It lives here, next to the parsing, because both halves of the tool need it
// and need it read the same way: the assembler to know what to keep, the check
// to know what answers to which rules. Two constants would drift in silence.
const LocalMarker = "<!-- conv:local -->"

// Marker returns the line the local marker stands on, or zero.
//
// A marker inside a fenced block is a quotation of the markup rather than the
// markup itself — a convention about keeping copies would carry one — and
// taking it for the boundary would hand the whole document to the repository.
func (d *Document) Marker() int {
	for n := 1; n <= d.Len(); n++ {
		if !d.fence[n-1] && strings.TrimSpace(d.lines[n-1]) == LocalMarker {
			return n
		}
	}
	return 0
}

// Markers counts the local markers outside fenced blocks.
func (d *Document) Markers() []int {
	var out []int
	for n := 1; n <= d.Len(); n++ {
		if !d.fence[n-1] && strings.TrimSpace(d.lines[n-1]) == LocalMarker {
			out = append(out, n)
		}
	}
	return out
}

// Below returns the lines from n to the end, joined.
func (d *Document) Below(n int) string {
	if n < 1 || n > d.Len() {
		return ""
	}
	return strings.Join(d.lines[n-1:], "\n")
}