diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 7ff9c6b..88a70a3 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -31,11 +31,12 @@ jobs: run: | curl -fsSL -o pandoc.deb "https://github.com/jgm/pandoc/releases/download/$PANDOC/pandoc-$PANDOC-1-amd64.deb" sudo dpkg -i pandoc.deb - - name: Build and test + - name: Build, test and prove run: | go vet ./... go test ./... go build -o bin/litgo . + bin/litgo prove litgo.lit.md examples/isqrt.lit.md examples/vego-tutorial.lit.md examples/heap.lit.md - name: Weave run: | mkdir _site diff --git a/README.md b/README.md index 93816a9..f4f149d 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,9 @@ [![License: MIT](https://img.shields.io/github/license/tlehman/litgo)](LICENSE) Literate programming for Go. Write a program as Markdown; litgo tangles it to -source, weaves it to a page, and runs it with every compiler error pointing at -the Markdown. +source, weaves it to a page, proves what its `//@` annotations promise, and +runs it with every compiler error and every failed proof pointing at the +Markdown. litgo is written in itself: [litgo.lit.md](litgo.lit.md) is the whole program and its documentation. Every other file here is generated from it. Read it @@ -29,8 +30,9 @@ Or download a binary for Linux or macOS (amd64, arm64) from the ## Use ```sh -litgo run prog.lit.md # tangle, compile, run -litgo check prog.lit.md # type-check without writing anything +litgo run prog.lit.md # tangle, prove, compile, run +litgo prove prog.lit.md # check the //@ annotations, and list what was proved +litgo check prog.lit.md # chunks, syntax, types and proofs, without writing anything litgo tangle prog.lit.md # write prog.go litgo weave prog.lit.md # write prog.pdf (needs pandoc and typst) ``` @@ -40,11 +42,60 @@ The PDF's Mermaid diagrams are drawn by mermaid.js if `mmdc` is installed Start from [examples/worker_pool.lit.md](examples/worker_pool.lit.md). +## Proofs + +litgo checks [VeGo](https://arxiv.org/abs/2608.22630) annotations: contracts, +loop invariants and variants, measures for recursion, written as `//@` +comments in ordinary Go. + +```go +//@ Requires n >= 0 +func Isqrt(n int) (r int) { + lo, hi := 0, n+1 + //@ Invariant 0 <= lo < hi + //@ Invariant lo*lo <= n ^ n < hi*hi + //@ Variant hi - lo + for hi-lo > 1 { + mid := lo + (hi-lo)/2 + if mid*mid <= n { + lo = mid + } else { + hi = mid + } + } + return lo +} +//@ Ensures r*r <= n ^ n < (r+1)*(r+1) +``` + +`litgo run` proves before it compiles, and a program that is not proved is not +run (`--no-prove` runs it anyway). The verifier is litgo's own, in pure Go with +no solver to install, since the paper's `vegop` is not published. It is sound +and deliberately small: integers, booleans, slices of integers that are read, +loops, calls and recursion. Anything else in an annotated function is reported +as not verified rather than passed. Integers are mathematical, so overflow is +not modelled. + +[examples/isqrt.lit.md](examples/isqrt.lit.md) is the program above with its +argument written out, and +[examples/vego-tutorial.lit.md](examples/vego-tutorial.lit.md) goes through +every annotation, and [examples/heap.lit.md](examples/heap.lit.md) proves a +binary heap and a ring buffer. litgo proves the column arithmetic of its own source map +this way. + ## Neovim Renders LaTeX and Mermaid in the buffer, lines up the columns of Markdown tables, shows type errors under the code as you type (`litgo lsp`), and adds -`:TangleCompileAndRun`, `:Untangle`, `:RenameChunk`, `:Tangle`, `:Weave`. +`:TangleCompileAndRun`, `:Prove`, `:Untangle`, `:RenameChunk`, `:Tangle`, +`:Weave`. + +`//@` annotations are violet and their formulas are typeset like the math in +the prose, so `lo*lo <= n ^ n < hi*hi` reads `π‘™π‘œΒ² ≀ 𝑛 ∧ 𝑛 < β„Žπ‘–Β²` until the +cursor is on the line (`render.proofs = false` turns that off). Failed proofs +are violet too, and arrive as you type: red means the compiler will not take it, violet means it runs and +has not been shown to keep its promise. The groups are `LitgoProofKeyword`, +`LitgoProof`, `LitgoProofError`, `LitgoProofWarn` and `LitgoProofUnderline`. With [gopls](https://go.dev/gopls/) installed (on `$PATH`, in `$GOPATH/bin`, or by Mason), a Go block is a Go buffer: completion, hover, go to definition, @@ -92,7 +143,7 @@ Edit `litgo.lit.md`, never the generated files. ```sh go build -o bin/litgo . # bootstrap from the committed sources -bin/litgo run litgo.lit.md # tangle, vet, test, rebuild bin/litgo +bin/litgo run litgo.lit.md # tangle, prove, vet, test, rebuild bin/litgo ``` Setting a new `VERSION` in the command and pushing to `master` publishes that diff --git a/examples/heap.lit.md b/examples/heap.lit.md new file mode 100644 index 0000000..26e91c0 --- /dev/null +++ b/examples/heap.lit.md @@ -0,0 +1,268 @@ + + + + +# A tree in an array, and a queue in a circle + +Two data structures that are nothing but integers and a slice, which is what +makes them good first subjects for a proof: a binary heap, which is a tree +with no pointers in it, and a ring buffer, which is a queue with no end. + +Everything here is proved by `litgo prove`, in the notation of +[VeGo](https://arxiv.org/abs/2608.22630). What "proved" means for a data +structure comes in three kinds, and each shows up below: + +* **The shape is right.** The arithmetic that finds a parent or a child really + does describe a tree: children know their parent, nobody is their own + ancestor, no index falls off the slice. +* **The invariant means something.** From "every node is at most its parent", + which is a local fact, it follows that the root is the maximum, which is a + global one. That step is an induction, and a loop is how an induction is + written. +* **The operations keep the invariant.** Whatever state a ring buffer is in, + if it was a valid one before `Push`, it is a valid one after. + +```mermaid +flowchart TD + n0["A[0] = 9"] --> n1["A[1] = 7"] & n2["A[2] = 8"] + n1 --> n3["A[3] = 3"] & n4["A[4] = 5"] + n2 --> n5["A[5] = 6"] +``` + +## The shape of the tree + +The tree is in the numbering. Node $i$ has its children at $2i+1$ and $2i+2$, +and so its parent at $\lfloor (i-1)/2 \rfloor$. Three one-line functions, and +their contracts say that they fit together. + +```go +//@ Requires i >= 0 +func Left(i int) (c int) { return 2*i + 1 } +//@ Ensures Parent(c) = i ^ c > i + +//@ Requires i >= 0 +func Right(i int) (c int) { return 2*i + 2 } +//@ Ensures Parent(c) = i ^ c = Left(i) + 1 + +//@ Requires i > 0 +func Parent(i int) (p int) { return (i - 1) / 2 } +//@ Ensures 0 <= p < i +//@ Ensures i = Left(p) v i = Right(p) +``` + +`Parent(Left(i)) = i` needs the verifier to know that $2i/2 = i$ and that +$(2i+1)/2 = i$ too, which is what rounding down means. A function that is one +`return` of one expression can be used inside a formula, where it stands for +that expression, so the contracts can talk about each other. + +`0 <= p < i` is the important one. It says a parent's number is smaller than +its child's. So walking upwards always makes progress, there are no cycles, +and every walk ends at node 0. That is what makes this numbering a *tree*. + +## The heap property + +A max-heap is a tree in which no node is larger than its parent. As a +predicate over the whole slice: + +```go +//@ Predicate Heap(A) ::= Forall k in [1, |A|) : A[Parent(k)] >= A[k] +``` + +Checking it is a loop over the nodes that have a parent. The contract is an +*if and only if*: `true` means it is a heap, and `false` means it is not, which +is the half people forget to promise. + +```go +func IsHeap(A []int) (ok bool) { + //@ Invariant 1 <= i + //@ Invariant Forall k in [1, i) : k < |A| -> A[Parent(k)] >= A[k] + //@ Variant |A| - i + for i := 1; i < len(A); i++ { + if A[Parent(i)] < A[i] { + return false + } + } + return true +} +//@ Ensures ok <-> Heap(A) +``` + +Nobody wrote that `A[Parent(i)]` is inside the slice. The verifier works it +out from `1 <= i < len(A)` and the meaning of division, or refuses the +function. + +## From local to global: the root is the maximum + +The heap property only compares a node with its parent. That the root beats +*every* node follows by climbing: if $A[j] \ge A[i]$ and $j$ is not the root, +then $A[\mathrm{Parent}(j)] \ge A[j] \ge A[i]$, and the parent is closer to +the root. + +```go +//@ Requires Heap(A) ^ 0 <= i < |A| +func Depth(A []int, i int) (d int) { + j := i + //@ Invariant 0 <= j <= i ^ A[j] >= A[i] ^ d >= 0 + //@ Variant j + for j > 0 { + j = Parent(j) + d++ + } + return d +} +//@ Ensures d >= 0 ^ A[0] >= A[i] +``` + +The function counts how deep node `i` is. Its proof is the interesting part: +the invariant carries `A[j] >= A[i]` up the tree, and when the loop ends `j` +is 0. The `Variant` is `j` itself, which shrinks because `Parent(j) < j`. So +the same contract proves that the climb is correct and that it ends. + +That was one node. For all of them at once, the induction goes over the +numbering instead of up a path: if the root beats every node below `k`, it +beats `Parent(k)`, which beats `k`. + +```go +//@ Requires Heap(A) ^ |A| > 0 +func Max(A []int) (m int) { + //@ Invariant 1 <= k <= |A| ^ A[0:k) <= A[0] + //@ Variant |A| - k + for k := 1; k < len(A); k++ { + //@ Assert A[Parent(k)] <= A[0] + } + return A[0] +} +//@ Ensures A[:] <= m ^ m in A[:] +``` + +This loop does nothing. (The `Assert` is the step of the induction, written +out for the reader. The prover finds it without being told.) The loop is there +because it is the proof: VeGo has no ghost +code, so an induction has to be a loop that could run, and this one runs. It +costs a pass over the slice to return `A[0]`, which a real heap would not pay, +and the honest way to use it is as a lemma, once, in a test. It is worth +seeing anyway, because it shows what a loop invariant *is*: the induction +hypothesis, with the loop as the induction. + +## One step of sifting down + +Repairing a heap means swapping a node with its larger child. Choosing that +child is where the off-by-one errors live, because a node can have two +children, one, or none. + +```go +//@ Requires i >= 0 ^ Left(i) < |A| +func LargerChild(A []int, i int) (c int) { + c = Left(i) + if Right(i) < len(A) && A[Right(i)] > A[c] { + c = Right(i) + } + return c +} +//@ Ensures (c = Left(i) v c = Right(i)) ^ c < |A| ^ Parent(c) = i +//@ Ensures A[c] >= A[Left(i)] ^ (Right(i) < |A| -> A[c] >= A[Right(i)]) +``` + +Go stops evaluating `&&` at the first `false`, and the verifier knows it, so +`A[Right(i)]` is only obliged to be in range when `Right(i) < len(A)` has just +been checked. Swap the two halves of that condition and the proof fails at the +index. + +The swap itself is `A[i], A[c] = A[c], A[i]`, and that is where this verifier +stops: it does not follow writes into a slice, because another slice may share +the array. So `main` below does the swapping, unproved, with the proved +function telling it where. + +## A queue in a circle + +A ring buffer is three integers and a slice: where the oldest element is, how +many there are, and how much room. Its invariant is what makes those three +numbers a queue. + +```go +//@ Predicate Ring(head, size, room) ::= room > 0 ^ 0 <= head < room ^ 0 <= size <= room +``` + +Every operation has the same contract in outline: *given a valid ring, here is +a slot inside the buffer, and a valid ring again*. The parameters cannot +change, being integers passed by value, so the new state comes back as +results. + +```go +//@ Requires Ring(head, size, room) ^ size < room +func Push(head, size, room int) (slot, h, s int) { + return (head + size) % room, head, size + 1 +} +//@ Ensures 0 <= slot < room ^ Ring(h, s, room) ^ s = size + 1 + +//@ Requires Ring(head, size, room) ^ size > 0 +func Pop(head, size, room int) (slot, h, s int) { + return head, (head + 1) % room, size - 1 +} +//@ Ensures 0 <= slot < room ^ Ring(h, s, room) ^ s = size - 1 +``` + +`size < room` and `size > 0` are the two ways to misuse a queue, and they are +now the caller's problem, in writing. A caller that is itself verified has to +prove them: + +```go +//@ Requires Ring(head, size, room) ^ size < room +func PushThenPop(head, size, room int) (h, s int) { + _, h, s = Push(head, size, room) + _, h, s = Pop(h, s, room) + return h, s +} +//@ Ensures Ring(h, s, room) ^ s = size +``` + +`Pop` is legal there because `Push` promised `s = size + 1`, which is more +than zero. Take that out of `Push`'s contract and this function stops +verifying, though `Push` itself still does: a contract is everything a caller +gets to know. + +## Running it + +```go +func main() { + A := []int{9, 7, 8, 3, 5, 6} + fmt.Println("IsHeap:", IsHeap(A), " Max:", Max(A), " Depth of node 5:", Depth(A, 5)) + + // Replace the root and sift it down, one proved step at a time. + A[0] = 1 + for i := 0; Left(i) < len(A); { + c := LargerChild(A, i) + if A[i] >= A[c] { + break + } + A[i], A[c] = A[c], A[i] + i = c + } + fmt.Println("after sifting 1 down:", A, " IsHeap:", IsHeap(A)) + + buf := make([]int, 3) + head, size := 0, 0 + for _, x := range []int{10, 20, 30} { + var slot int + slot, head, size = Push(head, size, len(buf)) + buf[slot] = x + } + var slot int + slot, head, size = Pop(head, size, len(buf)) + fmt.Println("popped", buf[slot]) + slot, head, size = Push(head, size, len(buf)) + buf[slot] = 40 + fmt.Println("buffer:", buf, " head:", head, " size:", size) +} +``` + +## Things to break + +* In `Parent`, return `i / 2`. `Left` still verifies. `Right` does not, nor + does `Parent` itself, nor `LargerChild`: the tree has lost its right + children, and everything that relied on them says so. +* In `Depth`, climb with `j = j - 1`. It still ends, and the invariant is not + maintained, because the node before `j` is no relation of it. +* In `LargerChild`, test `A[Right(i)] > A[c]` first. The index is no longer + known to be in range. +* In `Push`, drop the `% room`. The slot escapes the buffer. diff --git a/examples/isqrt.lit.md b/examples/isqrt.lit.md new file mode 100644 index 0000000..9a16512 --- /dev/null +++ b/examples/isqrt.lit.md @@ -0,0 +1,143 @@ + + + + +# A square root you can trust + +The integer square root of $n$ is the largest integer whose square does not +exceed $n$: + +$$ +r = \lfloor \sqrt{n} \rfloor \quad\Longleftrightarrow\quad r^2 \le n < (r+1)^2 +$$ + +This program finds it by bisection, and it comes with a proof. Not a test that +it gets $\sqrt{17}$ right, but a proof that the inequality on the right holds +for the result of *every* call with $n \ge 0$, that the loop always stops, and +that the division in it can never divide by zero. The proof is written in the +`//@` comments, in the notation of +[VeGo](https://arxiv.org/abs/2608.22630), and `litgo run` checks it before it +compiles anything: + +```sh +litgo run examples/isqrt.lit.md # prove, then compile, then run +litgo prove examples/isqrt.lit.md # only prove +``` + +```mermaid +flowchart LR + doc[isqrt.lit.md] -->|tangle| go[isqrt.go] + go -->|prove| ok{proved?} + ok -->|yes| build[go build, run] + ok -->|no| line[a violet line under the annotation that failed] +``` + +## The contract + +The function promises the defining inequality, and asks for one thing in +return. `Requires` is what the caller owes, `Ensures` is what the function +owes back, and the result has a name, `r`, so that the promise can mention it. + +```go +//@ Requires n >= 0 +func Isqrt(n int) (r int) { + // <> +} +//@ Ensures r*r <= n ^ n < (r+1)*(r+1) +``` + +`^` is *and*. The `Ensures` goes under the closing brace, where a conclusion +belongs. + +## The loop + +Bisection keeps two numbers, one whose square is known to be small enough and +one whose square is known to be too big, and moves one of them to the middle +until they are neighbours. + + +```go +lo, hi := 0, n+1 +//@ Invariant 0 <= lo < hi +//@ Invariant lo*lo <= n ^ n < hi*hi +//@ Variant hi - lo +for hi-lo > 1 { + mid := lo + (hi-lo)/2 + // <> +} +return lo +``` + +The sentence above *is* the invariant: `lo*lo <= n ^ n < hi*hi`. A loop +invariant is something true before the loop and true again after each time +round, and the verifier checks exactly those two things. It is true before, +because $0 \le n$ and $n < (n+1)^2$. Then the verifier forgets everything it +knew about `lo` and `hi` except the invariant, goes round once, and has to get +the invariant back. + + +```go +if mid*mid <= n { + lo = mid +} else { + hi = mid +} +``` + +Whichever branch runs, the test it just made is word for word the half of the +invariant it has to restore, so that part is immediate. The other line of the +invariant, `0 <= lo < hi`, needs the middle to be strictly between the ends, +and that is where the loop condition earns its keep: `hi - lo > 1` means +`(hi-lo)/2` is at least 1 and less than `hi - lo`. + +The same fact proves that the loop stops. The `Variant` is a quantity that is +never negative while the loop runs and gets smaller on every round. Here it is +the width of the interval, and both branches narrow it. + +After the loop the verifier knows the invariant and that the condition is +false. From `lo < hi` and not `hi - lo > 1` it follows that `hi = lo + 1`, and +putting that into `n < hi*hi` gives the `Ensures`. Nobody had to say so: those +few lines of arithmetic are what the prover is for. + +## What the proof does not say + +The integers of the proof are the integers of mathematics. Go's `int` has 64 +bits, and `(n+1)*(n+1)` overflows long before `n` runs out of them. For +$n < 3 \cdot 10^9$ the two agree; beyond that, the theorem is about a program +this one only resembles. VeGo makes the same simplification, and it is the +kind of thing worth knowing about a proof before leaning on it. + +## Running it + +`main` has no annotations, so the verifier leaves it alone: a program can be +proved one function at a time. + +```go +func main() { + for _, n := range []int{0, 1, 2, 15, 16, 17, 1_000_000, 2_147_395_599} { + fmt.Printf("isqrt(%d) = %d\n", n, Isqrt(n)) + } +} +``` + +## Breaking it + +The quickest way to believe a verifier is to lie to it. Change `lo = mid` to +`lo = mid + 1`, which looks like the usual bisection and is wrong here, and +run it again: + +```text +isqrt.lit.md:61:1: error: Invariant is not maintained by the body of the loop at line 64: 0 <= lo < hi +isqrt.lit.md:62:1: error: Invariant is not maintained by the body of the loop at line 64: lo*lo <= n ^ n < hi*hi +litgo: proved 0 of 1 annotated functions in isqrt.lit.md (11 obligations, 79 ms) +litgo: not proved, so not run (--no-prove runs it anyway) +``` + +Both complaints are right. `(mid+1)*(mid+1) <= n` is not what the test +established, and `mid + 1` can be `hi` itself, which leaves the interval +empty. In the editor the two lines are violet rather than red, because the +program still compiles: it is the argument that is broken, not the syntax. + +Or weaken the loop condition to `hi-lo > 0`, and the verifier points at the +`Variant`: with an interval of width one the middle is `lo`, nothing moves, +and the loop never ends. A test would have hung. The proof just says no. diff --git a/examples/vego-tutorial.lit.md b/examples/vego-tutorial.lit.md new file mode 100644 index 0000000..b6a6fa4 --- /dev/null +++ b/examples/vego-tutorial.lit.md @@ -0,0 +1,483 @@ + + + + +# VeGo in litgo: a tutorial + +[VeGo](https://arxiv.org/abs/2608.22630) (Verified Go, by Tina Massoudi and +Chris Dutchyn) is a way of writing down what a Go function promises, and why +its loops are right, in comments that start with `//@`. The program stays +ordinary Go. litgo reads the comments and proves them, or tells you which one +it could not prove and for which line of code. + +This document is a tour of every annotation litgo checks, one small function +at a time. Every function here is proved each time the document is run: + +```sh +litgo prove examples/vego-tutorial.lit.md # prove, and list what was proved +litgo run examples/vego-tutorial.lit.md # prove, then compile and run +litgo check examples/vego-tutorial.lit.md # chunks, syntax, types, proofs +``` + +In Neovim the annotations are violet and their formulas are typeset, so that +`^` reads ∧, `<=` reads ≀ and `Forall` reads βˆ€ until the cursor is on the line, +where you get back what you typed. Anything the verifier has to say about them +is violet too, as you type. A red line means the compiler will not take the +program. A violet line means the program runs and has not been shown to keep +its word. + +The best way to read this is to break things. Change a `<` to a `<=` in any +block and see what the verifier says. + +```mermaid +flowchart LR + req[Requires] --> body[the body, one path at a time] + inv[Invariant, Variant] --> body + body --> ens[Ensures at every return] + body --> safe[every index in range, every divisor not zero] + body --> calls[the Requires of every call] +``` + +## A contract + +`Requires` is what the caller owes. `Ensures` is what the function owes back. +`Requires` goes above the function and `Ensures` directly under its closing +brace, which is where a conclusion belongs. The result needs a name, so that +the `Ensures` has something to call it. + +```go +//@ Requires x > 0 ^ y > 0 +func AddPositives(x, y int) (sum int) { + return x + y +} +//@ Ensures sum > x ^ sum > y +``` + +Without the `Requires` this is false (try `0` and `0`), and the verifier says +so: *Ensures is not proved for the return at line 50*. + +A function without annotations is not looked at. So a program can be proved a +function at a time, and `main` below is left alone. + +One thing to know before your editor surprises you: gofmt rewrites `//@` in a +doc comment as `// @`, and puts a blank line above an `Ensures`. litgo reads +both spellings and looks upward past the blank line, so formatting on save +changes how a contract looks and not what it means. + +## The language of formulas + +| | | +|:--|:--| +| `^` `v` `~` | and, or, not | +| `->` `<-` `<->` | implies, is implied by, if and only if | +| `=` `<>` `<` `<=` `>` `>=` | comparisons. They chain: `0 <= i < n`. Go's `==` and `!=` are accepted too. | +| `+` `-` `*` `/` `%` | integer arithmetic; `/` and `%` are Go's, which round towards zero | +| `len(A)`, `A[i]` | the length and the elements of a slice of integers | +| `s.Len` | an integer or boolean field of a struct | +| `Forall k in [a, b) : P` | for every integer `k` with `a <= k < b` | +| `Exists k in [a, b] : P` | for some `k` with `a <= k <= b` | +| `Unique k in (a, b) : P` | for exactly one `k` with `a < k < b` | +| `Z`, `[0, ...)` | all integers; an interval without an upper end | +| `A[a:b) <= m` | every element of that range of `A` is at most `m` | +| `m in A[a:b)`, `m = A[a:b)` | some element of the range is `m` | +| `x'` | `x` after an assignment, where `x` is before it | + +The length of a slice is usually written the way mathematicians write it, +`|A|`, and that is how the rest of this tutorial writes it. Go's own `&&`, +`||` and `!` are accepted for *and*, *or* and *not*. + +A formula that is too long for a line goes on in the next `//@` comment, if +its line ends in something that cannot end a formula: an operator, say. + +## Safety comes for free + +Some obligations are not written by anybody, because the code implies them. +Every index has to be inside its slice, and every divisor has to be different +from zero. This function needs its `Requires` for no other reason: + +```go +//@ Requires 0 <= i ^ i < |A| ^ d <> 0 +func ElementOver(A []int, i, d int) (q int) { + return A[i] / d +} +//@ Ensures q * d + A[i] % d = A[i] +``` + +Take away `i < |A|` and the complaint is at the `A[i]` itself: *the index may +be out of range*. Go stops evaluating `&&` and `||` early, and the verifier +knows that, so `i < len(A) && A[i] > 0` is fine with nothing said about `i`. + +The `Ensures` shows that the prover knows what division means: a quotient and +a remainder add up. + +```go +//@ Requires n >= 0 +func Halve(n int) (h int) { + return n / 2 +} +//@ Ensures 2*h <= n ^ n < 2*h + 2 +``` + +## Loops + +Nobody knows how often a loop goes round, so the verifier does not try to +follow it. It asks for an **invariant**: a formula that is true when the loop +is reached and true again after every round. It proves those two things, and +after the loop the invariant, together with the loop condition being false, is +everything it knows. + +A **variant** is for termination: an integer expression that is not negative +while the loop runs and that every round makes smaller. A loop without one is +still proved correct *if* it ends, and gets a warning that says so. + +```go +//@ Requires n >= 0 ^ d > 0 +func Divide(n, d int) (q, r int) { + r = n + //@ Invariant n = q*d + r ^ r >= 0 + //@ Variant r + for r >= d { + q, r = q+1, r-d + } + return q, r +} +//@ Ensures n = q*d + r ^ 0 <= r < d +``` + +Read the invariant against the `Ensures`. They differ only in `r < d`, and +that is exactly the loop condition being false. Finding an invariant is mostly +this: take the postcondition and weaken it until it is true before the loop +starts. + +`Requires d > 0` is there for the variant. With `d = 0` the loop would run +forever, and the verifier notices: *Variant is not shown to decrease*. + +The two annotations may also stand at the end of the loop's body, which is +where VeGo's own examples tend to put them. + +### Counted loops, `range`, `continue` + +The counter of a `for` loop is in scope for the invariant. + +```go +//@ Requires n >= 0 +func SumTo(n int) (s int) { + //@ Invariant 0 <= i <= n + 1 ^ 2*s = i*(i-1) + //@ Variant n + 1 - i + for i := 0; i <= n; i++ { + s += i + } + return s +} +//@ Ensures 2*s = n*(n+1) +``` + +That is Gauss's formula, with the division multiplied away. Notice that +nothing here is linear, and the proof goes through anyway, because +`(i+1)*i` and `i*i + i` are the same polynomial. + +A `range` loop always ends, so it needs no variant. Its counter runs from `0` +to the length, and the verifier knows that without being told. `continue` +jumps to the end of the round, where the invariant is due as usual. The call +of `fmt.Println` is let through because all it is handed are numbers and a +string, so it cannot change anything the proof is about. + +```go +func Count(A []int, x int) (n int) { + //@ Invariant 0 <= n <= i + for i, a := range A { + if a != x { + continue + } + n++ + fmt.Println(" found", x, "at", i) + } + return n +} +//@ Ensures 0 <= n <= |A| +``` + +### `break`, and what cannot happen + +A `break` leaves the loop with whatever is known at that point. The invariant +is not assumed there, since the round was not finished. + +`Exsures` is the negative of `Ensures`. It describes a state that no return +may be in, which is sometimes the more natural thing to say: *it never +happens that a position is returned and the element there is not `x`*. + +```go +func Find(A []int, x int) (at int) { + at = -1 + //@ Invariant 0 <= i <= |A| ^ at = -1 + //@ Invariant x <> A[0:i) + //@ Variant |A| - i + for i := 0; i < len(A); i++ { + if A[i] == x { + at = i + break + } + } + return at +} +//@ Ensures at = -1 v 0 <= at < |A| +//@ Ensures at = -1 -> Forall k in [0, |A|) : A[k] <> x +//@ Exsures at >= 0 ^ A[at] <> x +``` + +## Quantifiers, ranges and predicates + +`x <> A[0:i)` in the last example was a quantifier in disguise: *`x` differs +from every element of `A` from `0` up to but not including `i`*. The ends of a +range are written like the ends of an interval, `[` or `]` for an end that is +included and `(` or `)` for one that is not. An order comparison with a range +holds for all of its elements; `=` and `in` hold for some element. + +```go +//@ Requires |A| > 0 +func Max(A []int) (m int) { + m = A[0] + //@ Invariant 1 <= i <= |A| + //@ Invariant A[0:i) <= m ^ m in A[0:i) + //@ Variant |A| - i + for i := 1; i < len(A); i++ { + if A[i] > m { + m = A[i] + } + } + return m +} +//@ Ensures A[:] <= m ^ m in A[:] +``` + +Both halves of that `Ensures` matter. `A[:] <= m` by itself is satisfied by +returning a billion. A contract that is too weak is proved just as happily as +a good one, and it is the reader who has to notice. + +A `Predicate` gives a formula a name. It is not a function and is never run; +using it means its body. This one also shows a formula going on in the next +line, after the `:`. + +```go +//@ Predicate Sorted(A) ::= Forall i in [0, |A|) : +//@ Forall j in [i, |A|) : A[i] <= A[j] + +//@ Predicate Between(lo, x, hi) ::= lo <= x ^ x <= hi +``` + +## Calls + +A call is where verification is modular. The caller never looks inside the +callee: it proves the callee's `Requires`, and what it learns about the result +is the callee's `Ensures`, and nothing else. + +```go +//@ Requires Between(0, x, 10) +func Twice(x int) (r int) { + return x + x +} +//@ Ensures r = double(x) ^ Between(0, r, 20) + +//@ Requires Between(1, k, 5) +func UseTwice(k int) (r int) { + return Twice(k) + Twice(k+5) +} +//@ Ensures r = 4*k + 10 +``` + +Make that `Between(1, k, 6)` and the second call is refused: *this call does +not establish what Twice requires*. + +`double` in that `Ensures` is a Go function. A function whose whole body is +`return` and one expression can be used in a formula, where it means that +expression. It is a way to give the contract a vocabulary that the program +shares. + +```go +func double(x int) int { return 2 * x } +``` + +## Recursion + +Recursion is induction. Inside `Triangle`, the call `Triangle(n-1)` is known +by the contract being proved: that is the induction hypothesis, and using it is +sound as long as the induction is well-founded. `Measure` is what makes it so: +an expression that is not negative and is smaller at every recursive call. +`BaseCase` marks where the induction starts, and the verifier checks that no +recursive call can be reached once it has held. `InductionHypothesis` states, +and proves, what makes the recursive call legal. + +```go +//@ Requires n >= 0 +//@ Measure n +func Triangle(n int) (t int) { + //@ BaseCase n = 0 + if n == 0 { + return 0 + } + //@ InductionHypothesis n - 1 >= 0 + return n + Triangle(n-1) +} +//@ Ensures 2*t = n*(n+1) +``` + +Write `Triangle(n)` instead and the contract would still "follow" from itself. +The `Measure` is what catches it: *Measure is not shown to get smaller in the +recursive call*. + +## Assertions and primes + +`Assert` is a claim about the state at one point. It is proved there, and +known from there on, so it doubles as a stepping stone when the prover needs +to be led. The word is optional: a bare formula is an assertion. + +A primed name is a value *after* an assignment. In a formula that mentions +`x'`, the `x'` is the value now and the plain `x` is the value one assignment +earlier, and with `x''` everything moves one further back. This is the old +trick of swapping two numbers without a third: + +```go +func Swap(a, b int) (x, y int) { + x, y = a, b + x = x + y + y = x - y + x = x - y + //@ Assert x = a + b ^ x' = b + //@ y = a + return x, y +} +//@ Ensures x = b ^ y = a +``` + +In an `Ensures`, a parameter without a prime is the value the caller passed, +whatever the body has done to its copy since. + +## Promises that last: `Preserves` + +`Preserves` states something once and has it checked after every assignment +until the block ends. A loop inside that block may assume it, so it does not +have to be repeated in the invariant. + +```go +//@ Requires amount >= 0 +func Transfer(from, to, amount int) (f, t int) { + f, t = from, to + //@ Preserves f + t = from + to + //@ Invariant 0 <= moved <= amount ^ f = from - moved + //@ Variant amount - moved + for moved := 0; moved < amount; moved++ { + f, t = f-1, t+1 + } + return f, t +} +//@ Ensures f + t = from + to ^ f = from - amount +``` + +No money is created or destroyed. The parallel assignment is the point: `f--` +followed by `t++` would break the promise for the length of one statement, and +the verifier would say which one. + +## The escape hatch: `Axiom` + +The prover is complete for nothing. Its arithmetic is linear, with products +treated as unknowns about which it knows a few facts, and the cube of a +number is past what it knows. `Axiom` tells it, and it believes you without +proof. That is exactly as dangerous as it sounds, so every `Axiom` is reported +as a warning, every time. + +```go +//@ Requires x >= 0 +func Cube(x int) (c int) { + //@ Axiom x*x*x >= 0 + return x * x * x +} +//@ Ensures c >= 0 +``` + +## All of it at once: binary search + +The most famous loop with a bug in it. The invariant says that everything to +the left of `lo` is too small and everything from `hi` on is too big, and +`Sorted` is what lets one comparison with `A[mid]` speak for a whole half of +the slice. + +```go +//@ Requires Sorted(A) +func Search(A []int, x int) (at int, found bool) { + lo, hi := 0, len(A) + //@ Invariant 0 <= lo <= hi <= |A| + //@ Invariant A[0:lo) < x ^ x < A[hi:|A|) + //@ Variant hi - lo + for lo < hi { + mid := lo + (hi-lo)/2 + if A[mid] == x { + return mid, true + } else if A[mid] < x { + lo = mid + 1 + } else { + hi = mid + } + } + return 0, false +} +//@ Ensures found -> 0 <= at < |A| ^ A[at] = x +//@ Ensures ~found -> x <> A[:] +``` + +Four things are proved here, and only two of them were written down: if +`found`, then `at` is a position of `x`; if not, `x` is nowhere in `A`; and, +unasked, `A[mid]` is always inside the slice, and the loop ends. Try +`lo = mid` (it loops forever, and the `Variant` says so), `hi = mid - 1` (it +skips an element, and the invariant is not maintained), or drop the +`Requires` (and nothing about the halves follows). + +## What is left out + +litgo's checker is its own, because the paper's `vegop` has not been +published, and it is deliberately smaller than what the paper describes. + +* **The fragment.** Integers, booleans, slices of integers that are read and + not written, integer and boolean fields of structs, `if`, `for`, `range`, + `break`, `continue`, `return`, and calls of annotated functions. An + annotated function that uses anything else is reported as *not verified*, + with the line that was too much. It is never silently passed. +* **No writes through a slice or a pointer.** `A[i] = x` may change another + slice that shares the array, and a proof that did not notice would prove + something false. So sorting is out, for now. +* **Not Go, so not here.** VeGo's `.vgo` dialect has `while`, `skip` and + functions inside functions. A litgo block is Go, where `for cond {}`, an + empty statement and a second function say the same. +* **Read but not checked.** `Property`, and contracts on interface methods. + They get a warning rather than silence. +* **The integers are mathematical.** Overflow is not modelled, here or in the + paper. +* **"Not proved" is not "false".** The prover never calls a false thing + proved, and it sometimes fails to prove a true one. The cure is usually a + stronger invariant or an `Assert` on the way. + +## Running it + +```go +func main() { + fmt.Println("AddPositives(2, 3) =", AddPositives(2, 3)) + fmt.Println("ElementOver({7, 9}, 1, 2) =", ElementOver([]int{7, 9}, 1, 2)) + fmt.Println("Halve(7) =", Halve(7)) + q, r := Divide(17, 5) + fmt.Println("Divide(17, 5) =", q, r) + fmt.Println("SumTo(100) =", SumTo(100)) + A := []int{1, 3, 3, 7, 9, 12} + fmt.Println("Count(A, 3):") + fmt.Println(" =", Count(A, 3)) + fmt.Println("Find(A, 7) =", Find(A, 7)) + fmt.Println("Max(A) =", Max(A)) + fmt.Println("UseTwice(2) =", UseTwice(2)) + fmt.Println("Triangle(10) =", Triangle(10)) + x, y := Swap(1, 2) + fmt.Println("Swap(1, 2) =", x, y) + f, t := Transfer(10, 0, 4) + fmt.Println("Transfer(10, 0, 4) =", f, t) + fmt.Println("Cube(3) =", Cube(3)) + at, found := Search(A, 9) + fmt.Println("Search(A, 9) =", at, found) +} +``` diff --git a/internal/lit/analyze.go b/internal/lit/analyze.go index e9eed42..f703692 100644 --- a/internal/lit/analyze.go +++ b/internal/lit/analyze.go @@ -13,7 +13,7 @@ import ( // Item is something an editor can render in place of, or next to, source text. type Item struct { - Kind string `json:"kind"` // math, math-block, mermaid, table, chunk-def, chunk-ref, file + Kind string `json:"kind"` // math, math-block, mermaid, table, chunk-def, chunk-ref, file, vego, vego-math Line int `json:"line"` Col int `json:"col"` EndLine int `json:"end_line"` @@ -92,6 +92,8 @@ func (d *Doc) Analyze(o AnalyzeOptions) *Analysis { a.Items = append(a.Items, Item{Kind: "math-block", Line: b.Open, EndLine: min(b.Close, len(d.Lines)-1), Lines: latex.Render(src, latex.Options{Display: true, Upright: o.Upright})}) } + case "go": + a.Items = append(a.Items, d.proofItems(b, o)...) } } a.Items = append(a.Items, d.mathItems(inBlock, o)...) @@ -159,6 +161,214 @@ func plural(n int) string { return "s" } +// ProofKeywords are the words an annotation can start with. One that starts +// with none of them is an Assert. +var ProofKeywords = map[string]bool{"Requires": true, "Ensures": true, "Exsures": true, "Invariant": true, + "Variant": true, "Measure": true, "BaseCase": true, "InductionHypothesis": true, "Assert": true, + "Axiom": true, "Preserves": true, "Predicate": true, "Immutable": true, "Property": true} + +// ProofComment reports whether a line comment is an annotation, and returns +// what follows the marker. The marker is //@. gofmt rewrites that to // @ in +// a doc comment, so that is accepted too, but only in front of a keyword or +// where the annotation above is not finished (continuing), because a comment +// may well start with an @ for reasons of its own. +func ProofComment(text string, continuing bool) (body string, ok bool) { + if strings.HasPrefix(text, "//@") { + return text[3:], true + } + if !strings.HasPrefix(text, "// @") { + return "", false + } + body = text[4:] + word, _, _ := strings.Cut(strings.TrimLeft(body, " \t"), " ") + return body, continuing || ProofKeywords[word] +} + +// proofItems finds the VeGo annotations of a Go block: one item for the +// annotation, and one for its formula, typeset. +func (d *Doc) proofItems(b *Block, o AnalyzeOptions) []Item { + var out []Item + continuing := false + for i := b.First(); i <= b.Last() && i < len(d.Lines); i++ { + line := d.Lines[i] + text := strings.TrimLeft(line, " \t") + body, ok := ProofComment(text, continuing) + if !ok { + continuing = false + continue + } + word, _, _ := strings.Cut(strings.TrimLeft(body, " \t"), " ") + if continuing || !ProofKeywords[word] { + word = "" // a formula by itself is an Assert without the word + } + trimmed := strings.TrimRight(body, " \t") + continuing = strings.HasSuffix(trimmed, "||") || strings.HasSuffix(trimmed, " v") || strings.HasSuffix(trimmed, " in") || + trimmed != "" && strings.ContainsAny(trimmed[len(trimmed)-1:], "^:=<>+-*/%(,[~!&") + out = append(out, Item{Kind: "vego", Line: i, EndLine: i, Col: len(line) - len(text), EndCol: len(line), Name: word}) + formula := strings.TrimLeft(body, " \t") + if word != "" { + formula = strings.TrimLeft(formula[len(word):], " \t") + } + if math := proofMath(formula, o); math != "" { + out = append(out, Item{Kind: "vego-math", Line: i, EndLine: i, Col: len(line) - len(formula), + EndCol: len(strings.TrimRight(line, " \t")), Text: math}) + } + } + return out +} + +// proofOps are the operators of a formula, longest first, and their LaTeX. +var proofOps = []struct{ op, tex string }{ + {"<->", `\leftrightarrow `}, {"::=", `\coloneqq `}, {"...", `\infty `}, {"->", `\to `}, {"<-", `\leftarrow `}, + {"<=", `\leq `}, {">=", `\geq `}, {"<>", `\neq `}, {"!=", `\neq `}, {"==", `= `}, {"&&", `\land `}, {"||", `\lor `}, + {"^", `\land `}, {"~", `\lnot `}, {"!", `\lnot `}, {"*", `\cdot `}, {"%", `\bmod `}, +} + +var proofWords = map[string]string{"Forall": `\forall `, "Exists": `\exists `, "Unique": `\exists! `, "in": `\in `, + "true": `\text{true}`, "false": `\text{false}`, "len": `\text{len}`} + +// proofPower counts the factors of a product of one name with itself, +// x*x*x, where the first x ends at j, and says where the product ends. A +// name that is called or indexed is not the same factor. +func proofPower(formula string, j int, word string) (n, end int) { + n, end = 1, j + for { + k := end + for k < len(formula) && formula[k] == ' ' { + k++ + } + if k >= len(formula) || formula[k] != '*' { + return n, end + } + for k++; k < len(formula) && formula[k] == ' '; k++ { + } + if !strings.HasPrefix(formula[k:], word) { + return n, end + } + k += len(word) + if k < len(formula) { + if c := formula[k]; c == '_' || c == '.' || c == '\'' || c == '(' || c == '[' || + c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' { + return n, end + } + } + n, end = n+1, k + } +} + +// proofClose finds the ) that closes the ( at i, or returns 0. +func proofClose(formula string, i int) int { + depth := 0 + for k := i; k < len(formula); k++ { + switch formula[k] { + case '(': + depth++ + case ')': + if depth--; depth == 0 { + return k + } + } + } + return 0 +} + +// proofMath typesets a formula the way the math in the prose is typeset. +func proofMath(formula string, o AnalyzeOptions) string { + var tex strings.Builder + operand := false // was the last thing something an operator can follow + quantifiers := 0 // that still wait for the colon before their body + powers := map[int][2]int{} // by the ) of a repeated factor: how often, and where the product ends + last := "" + for i := 0; i < len(formula); { + c := formula[i] + switch { + case c == ' ' || c == '\t': + i++ + case c == '_' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z': + j := i + for j < len(formula) && (formula[j] == '_' || formula[j] == '.' && j+1 < len(formula) && formula[j+1] != '.' || + formula[j] >= 'a' && formula[j] <= 'z' || formula[j] >= 'A' && formula[j] <= 'Z' || formula[j] >= '0' && formula[j] <= '9') { + j++ + } + word := formula[i:j] + for j < len(formula) && formula[j] == '\'' { + j++ + } + primes := formula[i+len(word) : j] + switch { + case word == "v" && primes == "" && operand: + tex.WriteString(`\lor `) + operand = false + case word == "Z" && last == "in": + tex.WriteString(`\mathbb{Z}`) + operand = true + case proofWords[word] != "" && primes == "": + if word == "Forall" || word == "Exists" || word == "Unique" { + quantifiers++ + } + tex.WriteString(proofWords[word]) + operand = word == "true" || word == "false" + default: + tex.WriteString(strings.ReplaceAll(word, "_", `\_`) + primes) + // x*x is a square, and is read more easily as one. After a / or a % + // it is not: a/x*x is (a/x)*x. + if n, end := proofPower(formula, j, word); n > 1 && primes == "" && last != "/" && last != "%" { + fmt.Fprintf(&tex, "^{%d}", n) + j = end + } + tex.WriteString(" ") + operand = true + } + last = word + i = j + default: + found := false + for _, op := range proofOps { + if strings.HasPrefix(formula[i:], op.op) { + tex.WriteString(op.tex) + i += len(op.op) + found, operand, last = true, op.op == "...", op.op + break + } + } + if !found && c == ':' && quantifiers > 0 { + // The colon of a quantifier gets room; the one in A[a:b) does not. + quantifiers-- + tex.WriteString(`\;:\;`) + found, operand, last = true, false, ":" + i++ + } + if c == '(' && !operand && last != "/" && last != "%" { + // (r+1)*(r+1) is a square too. Remember it until its first ) comes by. + if close := proofClose(formula, i); close > 0 { + if n, end := proofPower(formula, close+1, formula[i:close+1]); n > 1 { + powers[close] = [2]int{n, end} + } + } + } + if pw, ok := powers[i]; ok && c == ')' { + fmt.Fprintf(&tex, ")^{%d}", pw[0]) + found, operand, last = true, true, ")" + i = pw[1] + } + if !found { + tex.WriteByte(c) + operand = c == ')' || c == ']' || c == '|' || c >= '0' && c <= '9' + last = string(c) + i++ + } + } + } + if tex.Len() == 0 { + return "" + } + out := latex.Render(tex.String(), latex.Options{Upright: o.Upright}) + if len(out) != 1 || strings.Contains(out[0], `\`) { + return "" + } + return out[0] +} + var displayOpenRe = regexp.MustCompile(`^\s*(\$\$|\\\[)`) // mathItems finds LaTeX in prose: $inline$, \(inline\), and display math in diff --git a/internal/lit/lit_test.go b/internal/lit/lit_test.go index 6d0c7a8..20d9db8 100644 --- a/internal/lit/lit_test.go +++ b/internal/lit/lit_test.go @@ -448,6 +448,42 @@ func TestAnalyzeExample(t *testing.T) { } } +func TestProofMath(t *testing.T) { + for formula, want := range map[string]string{ + "lo*lo <= n ^ n < hi*hi": "π‘™π‘œΒ² ≀ 𝑛 ∧ 𝑛 < β„Žπ‘–Β²", + "x * x*x >= 0 ^ y*x*x = a/x*x": "π‘₯Β³ β‰₯ 0 ∧ 𝑦 Β· π‘₯Β² = π‘Ž/π‘₯ Β· π‘₯", + "n < (r+1)*(r+1) ^ (i+1)*i = f(x)*(x)": "𝑛 < (π‘Ÿ + 1)Β² ∧ (𝑖 + 1) Β· 𝑖 = 𝑓(π‘₯) Β· (π‘₯)", + "Forall k in [0, n) : (k)*(k) >= 0": "βˆ€π‘˜ ∈ [0, 𝑛) : (π‘˜)Β² β‰₯ 0", + "f*f(x) + A*A[i] + x*xs + x'*x'": "𝑓 Β· 𝑓(π‘₯) + 𝐴 Β· 𝐴[𝑖] + π‘₯ Β· π‘₯𝑠 + π‘₯β€² Β· π‘₯β€²", + "at = -1 v 0 <= at < |A|": "π‘Žπ‘‘ = βˆ’1 ∨ 0 ≀ π‘Žπ‘‘ < |𝐴|", + "v >= 0 v v' <> v": "𝑣 β‰₯ 0 ∨ 𝑣′ β‰  𝑣", + "~found -> x <> A[:]": "Β¬π‘“π‘œπ‘’π‘›π‘‘ β†’ π‘₯ β‰  𝐴[:]", + "Forall k in [0, |A|) : A[k] <> x": "βˆ€π‘˜ ∈ [0, |𝐴|) : 𝐴[π‘˜] β‰  π‘₯", + "Exists k in Z : a <-> b && c || !d": "βˆƒπ‘˜ ∈ β„€ : π‘Ž ↔ 𝑏 ∧ 𝑐 ∨ ¬𝑑", + "Sorted(A) ::= A[0:n) <= m": "π‘†π‘œπ‘Ÿπ‘‘π‘’π‘‘(𝐴) ≔ 𝐴[0:𝑛) ≀ π‘š", + "q * d + A[i] % d = A[i] ^ ret_0 == 1": "π‘ž Β· 𝑑 + 𝐴[𝑖] mod 𝑑 = 𝐴[𝑖] ∧ π‘Ÿπ‘’π‘‘_0 = 1", + "s.OutCol <= c ^ c <= s.OutCol + s.Len": "𝑠.π‘‚π‘’π‘‘πΆπ‘œπ‘™ ≀ 𝑐 ∧ 𝑐 ≀ 𝑠.π‘‚π‘’π‘‘πΆπ‘œπ‘™ + 𝑠.𝐿𝑒𝑛", + "Forall k in [0, ...) : k >= 0": "βˆ€π‘˜ ∈ [0, ∞) : π‘˜ β‰₯ 0", + "": "", + } { + if got := proofMath(formula, AnalyzeOptions{}); got != want { + t.Errorf("proofMath(%q) = %q, want %q", formula, got, want) + } + } + // The item covers the formula and nothing else, so that the keyword stays. + d := Parse("t.lit.md", []byte("```go\n\t//@ Invariant lo <= hi \n\t//@ a ^ b\n\t// @ Ensures b\n\t//@ c\n```\n")) + var got []string + for _, it := range d.Analyze(AnalyzeOptions{}).Items { + if it.Kind == "vego-math" { + got = append(got, fmt.Sprintf("%d:%d-%d %s", it.Line, it.Col, it.EndCol, it.Text)) + } + } + want := []string{"1:15-23 π‘™π‘œ ≀ β„Žπ‘–", "2:5-10 π‘Ž ∧ 𝑏", "3:14-15 𝑏", "4:7-8 𝑐"} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Errorf("items = %q, want %q", got, want) + } +} + func TestTables(t *testing.T) { src := "| Key | Does | Cost |\n|---|:-:|--:|\n| `\\r` | **run** it, see [docs](x) | $x^2$ |\n| a \\| b | short |\n\nnot | a table\n\n```go\n// | a | b |\n// |---|---|\n```\n" d := Parse("t.lit.md", []byte(src)) diff --git a/internal/lit/tangle.go b/internal/lit/tangle.go index c65d297..3d2ff03 100644 --- a/internal/lit/tangle.go +++ b/internal/lit/tangle.go @@ -326,7 +326,7 @@ func (o *Output) Map(line, col int) (srcLine, srcCol int, via []int, ok bool) { best := segs[0] for _, s := range segs { if col >= s.OutCol && col < s.OutCol+s.Len { - return s.SrcLine, s.SrcCol + col - s.OutCol, s.Via, true + return s.SrcLine, s.src(col), s.Via, true } if s.OutCol <= col { best = s @@ -377,7 +377,7 @@ func (r *Result) Locate(line, col int) (o *Output, outLine, outCol int, ok bool) for n, l := range o.Lines { for _, s := range l.Segs { if s.SrcLine == line && col >= s.SrcCol && col <= s.SrcCol+s.Len { - return o, n, s.OutCol + col - s.SrcCol, true + return o, n, s.out(col), true } } } @@ -398,9 +398,25 @@ func (o *Output) Exact(line, col int, end bool) (srcLine, srcCol int, ok bool) { lo, hi = lo+1, hi+1 } if col >= lo && col <= hi { - return s.SrcLine, s.SrcCol + col - s.OutCol, true + return s.SrcLine, s.src(col), true } } } return 0, 0, false } + +// out is where the byte at column col of the source line went in the output. +// +// @ Requires s.SrcCol <= col ^ col <= s.SrcCol + s.Len +func (s Seg) out(col int) (c int) { return s.OutCol + col - s.SrcCol } + +//@ Ensures s.OutCol <= c ^ c <= s.OutCol + s.Len +//@ Ensures s.src(c) = col + +// src is where the byte at column col of the output line came from. +// +// @ Requires s.OutCol <= col ^ col <= s.OutCol + s.Len +func (s Seg) src(col int) (c int) { return s.SrcCol + col - s.OutCol } + +//@ Ensures s.SrcCol <= c ^ c <= s.SrcCol + s.Len +//@ Ensures s.out(c) = col diff --git a/internal/lsp/lsp.go b/internal/lsp/lsp.go index a899b4f..159582b 100644 --- a/internal/lsp/lsp.go +++ b/internal/lsp/lsp.go @@ -14,6 +14,7 @@ import ( "github.com/tlehman/litgo/internal/check" "github.com/tlehman/litgo/internal/lit" + "github.com/tlehman/litgo/internal/vego" ) type message struct { @@ -304,6 +305,9 @@ func (s *server) check() { s.checking = true go func() { j.typed = check.Types(j.res, &s.cache) + if len(j.typed) == 0 { + j.typed, _ = vego.Prove(j.res) + } select { case s.checked <- j: case <-s.quit: diff --git a/internal/vego/logic.go b/internal/vego/logic.go new file mode 100644 index 0000000..8aa7fa4 --- /dev/null +++ b/internal/vego/logic.go @@ -0,0 +1,767 @@ +// Code generated by litgo from litgo.lit.md; DO NOT EDIT. + +package vego + +import ( + "fmt" + "sort" + "strconv" + "strings" +) + +type op int + +const ( + opNum op = iota + opVar + opTrue + opFalse + opAdd + opSub + opMul + opDiv + opMod + opNeg + opSel // args: array, index + opLen // args: array + opCall // name(args): a predicate, or a Go function that is one expression + opRange // name[lo:hi) as one side of a comparison; args: lo, hi (nil when omitted) + opLt + opLe + opEq + opNe + opIn // x in A[lo:hi) + opAnd + opOr + opNot + opImp + opIff + opForall // name is the bound variable, args[0] the body + opExists +) + +type sort_ int + +const ( + sInt sort_ = iota + sBool + sArr +) + +// expr is a term or a formula. Annotations parse to it, Go expressions are +// translated to it, and the symbolic values of program variables are made of it. +type expr struct { + op op + val int64 + name string + primes int + sort sort_ + args []*expr + pos int // byte offset in the annotation's text + open [2]bool // opRange: is the low, the high end exclusive +} + +func num(v int64) *expr { return &expr{op: opNum, val: v} } +func sym(name string, s sort_) *expr { return &expr{op: opVar, name: name, sort: s} } +func mk(o op, args ...*expr) *expr { return &expr{op: o, args: args} } + +var ( + eTrue = &expr{op: opTrue} + eFalse = &expr{op: opFalse} +) + +func and(a, b *expr) *expr { + if a.op == opTrue { + return b + } + if b.op == opTrue { + return a + } + return mk(opAnd, a, b) +} + +func not(a *expr) *expr { return mk(opNot, a) } + +func conj(list []*expr) *expr { + out := eTrue + for _, e := range list { + out = and(out, e) + } + return out +} + +// isBool says whether e is a formula rather than an integer term. +func isBool(e *expr) bool { + switch e.op { + case opTrue, opFalse, opLt, opLe, opEq, opNe, opIn, opAnd, opOr, opNot, opImp, opIff, opForall, opExists: + return true + case opVar: + return e.sort == sBool + } + return false +} + +func (e *expr) String() string { + bin := map[op]string{opAdd: "+", opSub: "-", opMul: "*", opDiv: "/", opMod: "%", opLt: "<", opLe: "<=", + opEq: "=", opNe: "<>", opAnd: "^", opOr: "v", opImp: "->", opIff: "<->", opIn: "in"} + switch e.op { + case opNum: + return strconv.FormatInt(e.val, 10) + case opVar: + return e.name + strings.Repeat("'", e.primes) + case opTrue: + return "true" + case opFalse: + return "false" + case opNeg: + return "-" + e.args[0].String() + case opNot: + return "~" + e.args[0].String() + case opSel: + return e.args[0].String() + "[" + e.args[1].String() + "]" + case opLen: + return "|" + e.args[0].String() + "|" + case opCall: + parts := make([]string, len(e.args)) + for i, a := range e.args { + parts[i] = a.String() + } + return e.name + "(" + strings.Join(parts, ", ") + ")" + case opRange: + return e.name + "[:]" + case opForall, opExists: + q := "Forall " + if e.op == opExists { + q = "Exists " + } + return "(" + q + e.name + " : " + e.args[0].String() + ")" + } + return "(" + e.args[0].String() + " " + bin[e.op] + " " + e.args[1].String() + ")" +} + +// rewrite rebuilds e bottom-up. f sees every variable that is not bound by a +// quantifier inside e, and returns its replacement or nil to keep it. +func rewrite(e *expr, f func(v *expr) *expr) *expr { + return rewriteBound(e, f, nil) +} + +func rewriteBound(e *expr, f func(v *expr) *expr, bound []string) *expr { + if e == nil { + return nil + } + switch e.op { + case opNum, opTrue, opFalse: + return e + case opVar: + for _, b := range bound { + if b == e.name && e.primes == 0 { + return e + } + } + if r := f(e); r != nil { + return r + } + return e + } + c := *e + c.args = make([]*expr, len(e.args)) + if e.op == opForall || e.op == opExists { + bound = append(bound[:len(bound):len(bound)], e.name) + } + for i, a := range e.args { + c.args[i] = rewriteBound(a, f, bound) + } + return &c +} + +// subst replaces the free variable name by r. +func subst(e *expr, name string, r *expr) *expr { + return rewrite(e, func(v *expr) *expr { + if v.name == name && v.primes == 0 { + return r + } + return nil + }) +} + +// A predicate is a named formula with parameters; using it means its body. +type predicate struct { + name string + params []string + body *expr +} + +type lexeme struct { + kind string // "num", "id", "op", "eof" + text string + pos int +} + +type syntaxError struct { + pos int + msg string +} + +func (e *syntaxError) Error() string { return e.msg } + +var operators = []string{"<->", "::=", "...", "->", "<-", "<=", ">=", "<>", "==", "!=", "&&", "||", + "^", "~", "!", "=", "<", ">", "+", "-", "*", "/", "%", "(", ")", "[", "]", ",", ":", "|"} + +func lex(s string) ([]lexeme, error) { + var out []lexeme + for i := 0; i < len(s); { + c := s[i] + switch { + case c == ' ' || c == '\t': + i++ + case c >= '0' && c <= '9': + j := i + for j < len(s) && s[j] >= '0' && s[j] <= '9' { + j++ + } + out = append(out, lexeme{"num", s[i:j], i}) + i = j + case c == '_' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z': + j := i + for j < len(s) && (s[j] == '_' || s[j] == '.' && j+1 < len(s) && s[j+1] != '.' || s[j] >= 'a' && s[j] <= 'z' || s[j] >= 'A' && s[j] <= 'Z' || s[j] >= '0' && s[j] <= '9') { + j++ + } + for j < len(s) && s[j] == '\'' { + j++ + } + out = append(out, lexeme{"id", s[i:j], i}) + i = j + default: + found := false + for _, o := range operators { + if strings.HasPrefix(s[i:], o) { + out = append(out, lexeme{"op", o, i}) + i += len(o) + found = true + break + } + } + if !found { + return nil, &syntaxError{i, fmt.Sprintf("unexpected %q", string(c))} + } + } + } + return append(out, lexeme{"eof", "", len(s)}), nil +} + +// continues reports whether an annotation line ends in a token that cannot +// end a formula, which is how a formula says it goes on on the next line. +func continues(text string) bool { + toks, err := lex(text) + if err != nil || len(toks) < 2 { + return false + } + last := toks[len(toks)-2] + if last.kind == "id" { + return last.text == "v" || last.text == "in" + } + if last.kind != "op" { + return false + } + switch last.text { + case ")", "]", "...", "|": + return false + } + return true +} + +type reader struct { + toks []lexeme + i int + fresh *int +} + +func (p *reader) peek() lexeme { return p.toks[p.i] } +func (p *reader) next() lexeme { + t := p.toks[p.i] + if t.kind != "eof" { + p.i++ + } + return t +} + +func (p *reader) is(text string) bool { + t := p.peek() + return t.kind == "op" && t.text == text +} + +func (p *reader) isWord(text string) bool { + t := p.peek() + return t.kind == "id" && t.text == text +} + +func (p *reader) fail(t lexeme, format string, args ...any) { + panic(&syntaxError{t.pos, fmt.Sprintf(format, args...)}) +} + +func (p *reader) expect(text string) { + if t := p.next(); t.kind != "op" || t.text != text { + what := t.text + if t.kind == "eof" { + what = "the end of the annotation" + } + p.fail(t, "expected %q, found %s", text, what) + } +} + +// parseFormula parses a whole annotation body. +func parseFormula(text string, fresh *int) (e *expr, err error) { + toks, err := lex(text) + if err != nil { + return nil, err + } + p := &reader{toks: toks, fresh: fresh} + defer func() { + if r := recover(); r != nil { + se, ok := r.(*syntaxError) + if !ok { + panic(r) + } + e, err = nil, se + } + }() + e = p.iff() + if t := p.peek(); t.kind != "eof" { + p.fail(t, "unexpected %q", t.text) + } + return e, nil +} + +// parsePredicate parses "Name(a, b) ::= body". +func parsePredicate(text string, fresh *int) (pr *predicate, err error) { + toks, err := lex(text) + if err != nil { + return nil, err + } + p := &reader{toks: toks, fresh: fresh} + defer func() { + if r := recover(); r != nil { + se, ok := r.(*syntaxError) + if !ok { + panic(r) + } + pr, err = nil, se + } + }() + name := p.next() + if name.kind != "id" { + p.fail(name, "a predicate needs a name") + } + pr = &predicate{name: name.text} + p.expect("(") + for !p.is(")") { + t := p.next() + if t.kind != "id" { + p.fail(t, "expected a parameter name") + } + pr.params = append(pr.params, t.text) + if !p.is(")") { + p.expect(",") + } + } + p.expect(")") + p.expect("::=") + pr.body = p.iff() + if t := p.peek(); t.kind != "eof" { + p.fail(t, "unexpected %q", t.text) + } + return pr, nil +} + +func (p *reader) iff() *expr { + l := p.imp() + for p.is("<->") { + t := p.next() + l = &expr{op: opIff, args: []*expr{l, p.imp()}, pos: t.pos} + } + return l +} + +func (p *reader) imp() *expr { + l := p.or() + if p.is("->") { + t := p.next() + return &expr{op: opImp, args: []*expr{l, p.imp()}, pos: t.pos} + } + if p.is("<-") { + t := p.next() + return &expr{op: opImp, args: []*expr{p.imp(), l}, pos: t.pos} + } + return l +} + +func (p *reader) or() *expr { + l := p.and() + for p.isWord("v") || p.is("||") { + t := p.next() + l = &expr{op: opOr, args: []*expr{l, p.and()}, pos: t.pos} + } + return l +} + +func (p *reader) and() *expr { + l := p.neg() + for p.is("^") || p.is("&&") { + t := p.next() + l = &expr{op: opAnd, args: []*expr{l, p.neg()}, pos: t.pos} + } + return l +} + +func (p *reader) neg() *expr { + if p.is("~") || p.is("!") { + t := p.next() + return &expr{op: opNot, args: []*expr{p.neg()}, pos: t.pos} + } + if p.isWord("Forall") || p.isWord("Exists") || p.isWord("Unique") { + return p.quantifier() + } + return p.comparison() +} + +var comparisons = map[string]bool{"=": true, "==": true, "<>": true, "!=": true, "<": true, "<=": true, ">": true, ">=": true} + +// comparison parses a chain: 0 <= i < n means 0 <= i and i < n. +func (p *reader) comparison() *expr { + l := p.sum() + var out *expr + for { + t := p.peek() + in := t.kind == "id" && t.text == "in" + if !in && !(t.kind == "op" && comparisons[t.text]) { + break + } + p.next() + r := p.sum() + c := p.compare(t, l, r) + if out == nil { + out = c + } else { + out = &expr{op: opAnd, args: []*expr{out, c}, pos: t.pos} + } + l = r + } + if out == nil { + if l.op == opRange { + p.fail(p.peek(), "a range like %s[a:b) can only be compared with something", l.name) + } + return l + } + return out +} + +func (p *reader) compare(t lexeme, l, r *expr) *expr { + if l.op == opRange || r.op == opRange { + return p.overRange(t, l, r) + } + switch t.text { + case "=", "==": + if isBool(l) || isBool(r) { + return &expr{op: opIff, args: []*expr{l, r}, pos: t.pos} + } + return &expr{op: opEq, args: []*expr{l, r}, pos: t.pos} + case "<>", "!=": + if isBool(l) || isBool(r) { + return &expr{op: opNot, args: []*expr{{op: opIff, args: []*expr{l, r}, pos: t.pos}}, pos: t.pos} + } + return &expr{op: opNe, args: []*expr{l, r}, pos: t.pos} + case "<": + return &expr{op: opLt, args: []*expr{l, r}, pos: t.pos} + case "<=": + return &expr{op: opLe, args: []*expr{l, r}, pos: t.pos} + case ">": + return &expr{op: opLt, args: []*expr{r, l}, pos: t.pos} + case ">=": + return &expr{op: opLe, args: []*expr{r, l}, pos: t.pos} + } + p.fail(t, "%q needs a range like A[a:b) on its right", t.text) + return nil +} + +// overRange turns a comparison with A[lo:hi) into a quantifier over a fresh +// index: an order comparison holds for all elements, = and in for some. +func (p *reader) overRange(t lexeme, l, r *expr) *expr { + if l.op == opRange && r.op == opRange { + p.fail(t, "only one side of a comparison can be a range") + } + rng := r + if l.op == opRange { + rng = l + } + *p.fresh++ + idx := &expr{op: opVar, name: fmt.Sprintf("_idx%d", *p.fresh)} + elem := &expr{op: opSel, args: []*expr{{op: opVar, name: rng.name, sort: sArr}, idx}} + lo, hi := rng.args[0], rng.args[1] + if lo == nil { + lo = num(0) + } + if hi == nil { + hi = &expr{op: opLen, args: []*expr{{op: opVar, name: rng.name, sort: sArr}}} + } + within := mk(opAnd, mk(bound(rng.open[0]), lo, idx), mk(bound(rng.open[1]), idx, hi)) + if rng.args[1] == nil { + within.args[1] = mk(opLt, idx, hi) + } + if t.text == "in" || t.text == "=" || t.text == "==" { + other := l + if l.op == opRange { + other = r + } + return &expr{op: opExists, name: idx.name, args: []*expr{mk(opAnd, within, mk(opEq, other, elem))}, pos: t.pos} + } + if t.text == "<>" || t.text == "!=" { + other := l + if l.op == opRange { + other = r + } + return &expr{op: opForall, name: idx.name, args: []*expr{mk(opImp, within, mk(opNe, other, elem))}, pos: t.pos} + } + var c *expr + if l.op == opRange { + c = p.compare(t, elem, r) + } else { + c = p.compare(t, l, elem) + } + return &expr{op: opForall, name: idx.name, args: []*expr{mk(opImp, within, c)}, pos: t.pos} +} + +func bound(open bool) op { + if open { + return opLt + } + return opLe +} + +func (p *reader) sum() *expr { + l := p.product() + for p.is("+") || p.is("-") { + t := p.next() + o := opAdd + if t.text == "-" { + o = opSub + } + l = &expr{op: o, args: []*expr{l, p.product()}, pos: t.pos} + } + return l +} + +func (p *reader) product() *expr { + l := p.unary() + for p.is("*") || p.is("/") || p.is("%") { + t := p.next() + o := map[string]op{"*": opMul, "/": opDiv, "%": opMod}[t.text] + l = &expr{op: o, args: []*expr{l, p.unary()}, pos: t.pos} + } + return l +} + +func (p *reader) unary() *expr { + if p.is("-") { + t := p.next() + return &expr{op: opNeg, args: []*expr{p.unary()}, pos: t.pos} + } + return p.primary() +} + +func (p *reader) primary() *expr { + t := p.next() + switch { + case t.kind == "num": + v, err := strconv.ParseInt(t.text, 10, 64) + if err != nil { + p.fail(t, "%s is too large", t.text) + } + return &expr{op: opNum, val: v, pos: t.pos} + case t.kind == "op" && t.text == "(": + e := p.iff() + p.expect(")") + return e + case t.kind == "op" && t.text == "|": + a := p.next() + if a.kind != "id" { + p.fail(a, "|…| takes the name of a slice") + } + p.expect("|") + return &expr{op: opLen, args: []*expr{{op: opVar, name: a.text, sort: sArr, pos: a.pos}}, pos: t.pos} + case t.kind == "id": + return p.named(t) + case t.kind == "eof": + p.fail(t, "the formula stops short") + } + p.fail(t, "unexpected %q", t.text) + return nil +} + +// named parses what starts with an identifier: a variable, true, len(A), +// A[i], a range A[a:b), or P(x, y). +func (p *reader) named(t lexeme) *expr { + name := strings.TrimRight(t.text, "'") + primes := len(t.text) - len(name) + switch name { + case "true": + return &expr{op: opTrue, pos: t.pos} + case "false": + return &expr{op: opFalse, pos: t.pos} + case "Forall", "Exists", "Unique", "in": + p.fail(t, "%s cannot be used as a name", name) + } + v := &expr{op: opVar, name: name, primes: primes, pos: t.pos} + if p.is("[") { + p.next() + if r := p.rangeAfter(v, false); r != nil { + return r + } + idx := p.sum() + if p.is(":") { + p.next() + return p.rangeEnd(v, idx, false) + } + p.expect("]") + v.sort = sArr + return &expr{op: opSel, args: []*expr{v, idx}, pos: t.pos} + } + if p.is("(") { + p.next() + if r := p.rangeAfter(v, true); r != nil { + return r + } + var args []*expr + for !p.is(")") { + a := p.iff() + if len(args) == 0 && p.is(":") { + p.next() + return p.rangeEnd(v, a, true) + } + args = append(args, a) + if !p.is(")") { + p.expect(",") + } + } + p.expect(")") + if name == "len" && len(args) == 1 && args[0].op == opVar { + args[0].sort = sArr + return &expr{op: opLen, args: args, pos: t.pos} + } + return &expr{op: opCall, name: name, args: args, pos: t.pos} + } + return v +} + +// rangeAfter handles A[:b) and A[:], where the low end is left out. +func (p *reader) rangeAfter(v *expr, open bool) *expr { + if !p.is(":") { + return nil + } + p.next() + return p.rangeEnd(v, nil, open) +} + +func (p *reader) rangeEnd(v, lo *expr, open bool) *expr { + r := &expr{op: opRange, name: v.name, args: []*expr{lo, nil}, pos: v.pos} + r.open[0] = open + if !p.is("]") && !p.is(")") { + r.args[1] = p.sum() + } + t := p.next() + switch { + case t.kind == "op" && t.text == ")": + r.open[1] = true + case t.kind == "op" && t.text == "]": + default: + p.fail(t, "a range ends in ] or )") + } + return r +} + +// quantifier parses Forall x in D : body. The body reaches as far as it can. +func (p *reader) quantifier() *expr { + q := p.next() + x := p.next() + if x.kind != "id" || strings.HasSuffix(x.text, "'") { + p.fail(x, "%s needs a variable", q.text) + } + if !p.isWord("in") { + p.fail(p.peek(), "expected \"in\" and a domain, as in %s %s in [0, n) : …", q.text, x.text) + } + p.next() + v := &expr{op: opVar, name: x.text} + within := p.domain(v) + p.expect(":") + body := p.iff() + switch q.text { + case "Forall": + if within != nil { + body = mk(opImp, within, body) + } + return &expr{op: opForall, name: x.text, args: []*expr{body}, pos: q.pos} + case "Exists": + if within != nil { + body = mk(opAnd, within, body) + } + return &expr{op: opExists, name: x.text, args: []*expr{body}, pos: q.pos} + } + // Unique x: some x makes the body true, and whatever makes it true is x. + if within != nil { + body = mk(opAnd, within, body) + } + *p.fresh++ + other := fmt.Sprintf("_u%d", *p.fresh) + same := &expr{op: opForall, name: other, args: []*expr{ + mk(opImp, subst(body, x.text, &expr{op: opVar, name: other}), mk(opEq, &expr{op: opVar, name: other}, v))}} + return &expr{op: opExists, name: x.text, args: []*expr{mk(opAnd, body, same)}, pos: q.pos} +} + +// domain parses Z or an interval, and returns what it says about v. +func (p *reader) domain(v *expr) *expr { + t := p.next() + if t.kind == "id" && t.text == "Z" { + return nil + } + if t.kind == "id" && t.text == "B" { + p.fail(t, "litgo prove quantifies over integers only") + } + if t.kind != "op" || t.text != "[" && t.text != "(" { + p.fail(t, "expected a domain: Z, or an interval like [0, n)") + } + var parts []*expr + if p.is("...") { + p.next() + } else { + parts = append(parts, mk(bound(t.text == "("), p.sum(), v)) + } + p.expect(",") + if p.is("...") { + p.next() + if c := p.next(); c.kind != "op" || c.text != ")" && c.text != "]" { + p.fail(c, "an interval ends in ] or )") + } + } else { + hi := p.sum() + c := p.next() + if c.kind != "op" || c.text != ")" && c.text != "]" { + p.fail(c, "an interval ends in ] or )") + } + parts = append(parts, mk(bound(c.text == ")"), v, hi)) + } + return conj(parts) +} + +// names lists the free variables of e, sorted. +func names(e *expr) []string { + seen := map[string]bool{} + rewrite(e, func(v *expr) *expr { + seen[v.name] = true + return nil + }) + out := make([]string, 0, len(seen)) + for n := range seen { + out = append(out, n) + } + sort.Strings(out) + return out +} diff --git a/internal/vego/prove.go b/internal/vego/prove.go new file mode 100644 index 0000000..d0c525f --- /dev/null +++ b/internal/vego/prove.go @@ -0,0 +1,925 @@ +// Code generated by litgo from litgo.lit.md; DO NOT EDIT. + +package vego + +import ( + "fmt" + "sort" + "strings" +) + +// A poly is a sum of monomials with integer coefficients. A monomial is a +// product of atoms, written as their names joined by sep; the constant's +// monomial is the empty string. +type poly map[string]int64 + +const sep = "\x1f" + +func constant(v int64) poly { + if v == 0 { + return poly{} + } + return poly{"": v} +} + +func (p poly) add(q poly, k int64) poly { + out := poly{} + for m, c := range p { + out[m] = c + } + for m, c := range q { + out[m] += k * c + if out[m] == 0 { + delete(out, m) + } + } + return out +} + +func (p poly) mul(q poly) poly { + out := poly{} + for m1, c1 := range p { + for m2, c2 := range q { + var atoms []string + if m1 != "" { + atoms = append(atoms, strings.Split(m1, sep)...) + } + if m2 != "" { + atoms = append(atoms, strings.Split(m2, sep)...) + } + sort.Strings(atoms) + m := strings.Join(atoms, sep) + out[m] += c1 * c2 + if out[m] == 0 { + delete(out, m) + } + } + } + return out +} + +func (p poly) isConst() (int64, bool) { + if len(p) == 0 { + return 0, true + } + if c, ok := p[""]; ok && len(p) == 1 { + return c, true + } + return 0, false +} + +func (p poly) String() string { + keys := make([]string, 0, len(p)) + for m := range p { + keys = append(keys, m) + } + sort.Strings(keys) + var b strings.Builder + for _, m := range keys { + fmt.Fprintf(&b, "%+d", p[m]) + if m != "" { + b.WriteString("*" + strings.ReplaceAll(m, sep, "*")) + } + } + if b.Len() == 0 { + return "0" + } + return b.String() +} + +// A node is a formula whose leaves are constraints: p <= 0, p = 0, or a +// boolean atom with a sign. +type node struct { + kind byte // 'a' and, 'o' or, 'l' p <= 0, 'e' p = 0, 'b' boolean atom, 't' true, 'f' false + kids []*node + p poly + atom string + truth bool +} + +var ( + nTrue = &node{kind: 't'} + nFalse = &node{kind: 'f'} +) + +func le0(p poly) *node { + if c, ok := p.isConst(); ok { + if c <= 0 { + return nTrue + } + return nFalse + } + return &node{kind: 'l', p: p} +} + +func eq0(p poly) *node { + if c, ok := p.isConst(); ok { + if c == 0 { + return nTrue + } + return nFalse + } + return &node{kind: 'e', p: p} +} + +// lt0 is p < 0, which over the integers is p + 1 <= 0. +func lt0(p poly) *node { return le0(p.add(constant(1), 1)) } + +func ne0(p poly) *node { return nOr(lt0(p), lt0(poly{}.add(p, -1))) } + +func nAnd(kids ...*node) *node { + var out []*node + for _, k := range kids { + switch k.kind { + case 't': + case 'f': + return nFalse + case 'a': + out = append(out, k.kids...) + default: + out = append(out, k) + } + } + if len(out) == 0 { + return nTrue + } + if len(out) == 1 { + return out[0] + } + return &node{kind: 'a', kids: out} +} + +func nOr(kids ...*node) *node { + var out []*node + for _, k := range kids { + switch k.kind { + case 'f': + case 't': + return nTrue + case 'o': + out = append(out, k.kids...) + default: + out = append(out, k) + } + } + if len(out) == 0 { + return nFalse + } + if len(out) == 1 { + return out[0] + } + return &node{kind: 'o', kids: out} +} + +// The budgets keep a hopeless proof from holding up the editor. Running out +// of budget means "not proved", never "proved". +const ( + maxSteps = 2000 + maxConstraints = 1500 + maxInstances = 4000 + maxTerms = 14 +) + +type prover struct { + fresh int + steps int + instances int + gaveUp bool + atoms map[string]bool // atoms whose axioms have been stated + axioms []*node + sels map[string][]selAtom // by array +} + +type selAtom struct { + name string + idx poly +} + +// proves reports whether the hypotheses entail the goal. It answers by +// refutation: the hypotheses and the goal's negation must have no model. +func proves(hyps []*expr, goal *expr) bool { + // Most proofs need every Forall only at the terms that are already there. + // Some need it again at the terms those instances mention, and that costs + // much more, so it is the second thing to try and not the first. + return attempt(hyps, goal, 1) || attempt(hyps, goal, 2) +} + +func attempt(hyps []*expr, goal *expr, rounds int) bool { + pr := &prover{atoms: map[string]bool{}, sels: map[string][]selAtom{}} + f := nnf(and(conj(hyps), not(goal)), true) + f = pr.skolem(f) + for round := 0; round < rounds; round++ { + f = pr.instantiate(f, pr.terms(f), round < rounds-1) + } + f = pr.instantiate(f, nil, false) + g := pr.ground(f) + pr.congruence() + all := nAnd(append([]*node{g}, pr.axioms...)...) + return pr.refute(nil, map[string]bool{}, []*node{all}) && !pr.gaveUp +} + +// nnf pushes negations down to the atoms. pos is false under a negation. +func nnf(e *expr, pos bool) *expr { + switch e.op { + case opTrue, opFalse: + if pos == (e.op == opTrue) { + return eTrue + } + return eFalse + case opNot: + return nnf(e.args[0], !pos) + case opAnd, opOr: + o := e.op + if !pos { + o = opAnd + opOr - o + } + return mk(o, nnf(e.args[0], pos), nnf(e.args[1], pos)) + case opImp: + if pos { + return mk(opOr, nnf(e.args[0], false), nnf(e.args[1], true)) + } + return mk(opAnd, nnf(e.args[0], true), nnf(e.args[1], false)) + case opIff: + a, b := e.args[0], e.args[1] + if pos { + return mk(opAnd, mk(opOr, nnf(a, false), nnf(b, true)), mk(opOr, nnf(b, false), nnf(a, true))) + } + return mk(opOr, mk(opAnd, nnf(a, true), nnf(b, false)), mk(opAnd, nnf(b, true), nnf(a, false))) + case opForall, opExists: + o := e.op + if !pos { + o = opForall + opExists - o + } + return &expr{op: o, name: e.name, args: []*expr{nnf(e.args[0], pos)}} + } + if pos { + return e + } + switch e.op { + case opLt: + return mk(opLe, e.args[1], e.args[0]) + case opLe: + return mk(opLt, e.args[1], e.args[0]) + case opEq: + return mk(opNe, e.args[0], e.args[1]) + case opNe: + return mk(opEq, e.args[0], e.args[1]) + } + return not(e) // a boolean atom +} + +// skolem names a witness for every Exists that is not under a Forall. One +// under a Forall waits until the Forall has been instantiated. +func (pr *prover) skolem(e *expr) *expr { + switch e.op { + case opAnd, opOr: + return mk(e.op, pr.skolem(e.args[0]), pr.skolem(e.args[1])) + case opExists: + pr.fresh++ + w := sym(fmt.Sprintf("%s?%d", e.name, pr.fresh), sInt) + return pr.skolem(subst(e.args[0], e.name, w)) + } + return e +} + +// terms collects what a Forall might be worth instantiating with: the ground +// terms that index a slice, and the witnesses. +func (pr *prover) terms(e *expr) []*expr { + seen := map[string]bool{} + var out []*expr + var walk func(e *expr, bound []string) + walk = func(e *expr, bound []string) { + if e == nil { + return + } + if e.op == opForall || e.op == opExists { + bound = append(bound[:len(bound):len(bound)], e.name) + } + candidate := e.op == opSel && len(e.args) == 2 + var t *expr + if candidate { + t = e.args[1] + } else if e.op == opVar && strings.Contains(e.name, "?") { + t, candidate = e, true + } + if candidate { + free := true + for _, n := range names(t) { + for _, b := range bound { + free = free && n != b + } + } + if key := t.String(); free && !seen[key] && len(out) < maxTerms { + seen[key] = true + out = append(out, t) + } + } + for _, a := range e.args { + walk(a, bound) + } + } + walk(e, nil) + return out +} + +// instantiate replaces every outermost Forall by its instances at the terms, +// and keeps the Forall itself for a later round if asked to. +func (pr *prover) instantiate(e *expr, terms []*expr, keep bool) *expr { + switch e.op { + case opAnd, opOr: + return mk(e.op, pr.instantiate(e.args[0], terms, keep), pr.instantiate(e.args[1], terms, keep)) + case opForall: + out := eTrue + if keep { + out = e + } + for _, t := range terms { + if pr.instances++; pr.instances > maxInstances { + pr.gaveUp = true + break + } + inst := pr.skolem(subst(e.args[0], e.name, t)) + out = and(out, pr.instantiate(inst, terms, keep)) + } + return out + } + return e +} + +// ground turns a quantifier-free formula into constraints. +func (pr *prover) ground(e *expr) *node { + switch e.op { + case opTrue: + return nTrue + case opFalse: + return nFalse + case opAnd: + return nAnd(pr.ground(e.args[0]), pr.ground(e.args[1])) + case opOr: + return nOr(pr.ground(e.args[0]), pr.ground(e.args[1])) + case opForall: + return nTrue // a fact we could not use + case opExists: + pr.gaveUp = true + return nTrue + case opNot: + return &node{kind: 'b', atom: e.args[0].String(), truth: false} + case opVar, opCall: + return &node{kind: 'b', atom: e.String(), truth: true} + } + if len(e.args) != 2 || isBool(e.args[0]) || isBool(e.args[1]) { + pr.gaveUp = true // a number where a formula belongs, or the other way round + return nTrue + } + d := pr.poly(e.args[0]).add(pr.poly(e.args[1]), -1) + switch e.op { + case opLe: + return le0(d) + case opLt: + return lt0(d) + case opEq: + return eq0(d) + case opNe: + return ne0(d) + } + pr.gaveUp = true + return nTrue +} + +// poly normalises an integer term. Whatever is not arithmetic becomes an +// atom: a variable, an element of a slice, a length, a quotient. +func (pr *prover) poly(e *expr) poly { + switch e.op { + case opNum: + return constant(e.val) + case opVar: + return poly{e.name: 1} + case opAdd: + return pr.poly(e.args[0]).add(pr.poly(e.args[1]), 1) + case opSub: + return pr.poly(e.args[0]).add(pr.poly(e.args[1]), -1) + case opNeg: + return poly{}.add(pr.poly(e.args[0]), -1) + case opMul: + p := pr.poly(e.args[0]).mul(pr.poly(e.args[1])) + pr.products(p) + return p + case opLen: + name := "len(" + e.args[0].name + ")" + if !pr.atoms[name] { + pr.atoms[name] = true + pr.axioms = append(pr.axioms, le0(poly{name: -1})) + } + return poly{name: 1} + case opSel: + idx := pr.poly(e.args[1]) + arr := e.args[0].name + name := arr + "[" + idx.String() + "]" + if !pr.atoms[name] { + pr.atoms[name] = true + pr.sels[arr] = append(pr.sels[arr], selAtom{name, idx}) + } + return poly{name: 1} + case opDiv, opMod: + return pr.divide(e) + } + pr.gaveUp = true + return poly{e.String(): 1} +} + +// divide states what Go's division means. For a = q*b + r with b > 0, the +// remainder has the sign of a and is smaller than b. +func (pr *prover) divide(e *expr) poly { + a, b := pr.poly(e.args[0]), pr.poly(e.args[1]) + key := "(" + a.String() + ")/(" + b.String() + ")" + q, r := "quo"+key, "rem"+key + if !pr.atoms[q] { + pr.atoms[q] = true + qb := poly{q: 1}.mul(b) + pr.products(qb) + // a = q*b + r + pr.axioms = append(pr.axioms, eq0(a.add(qb, -1).add(poly{r: 1}, -1))) + pos := lt0(poly{}.add(b, -1)) // 0 < b + nonneg, nonpos := le0(poly{}.add(a, -1)), le0(a) // 0 <= a, a <= 0 + small := lt0(poly{r: 1}.add(b, -1)) // r < b + smallNeg := lt0(poly{r: -1}.add(b, -1)) // -r < b + pr.axioms = append(pr.axioms, + nOr(negate(pos), negate(nonneg), nAnd(le0(poly{r: -1}), small, le0(poly{q: -1}))), + nOr(negate(pos), negate(nonpos), nAnd(le0(poly{r: 1}), smallNeg, le0(poly{q: 1})))) + } + if e.op == opDiv { + return poly{q: 1} + } + return poly{r: 1} +} + +// negate works on the constraints divide builds: p <= 0 becomes -p + 1 <= 0. +func negate(n *node) *node { + switch n.kind { + case 't': + return nFalse + case 'f': + return nTrue + case 'l': + return lt0(poly{}.add(n.p, -1)) + } + panic("negate: not an inequality") +} + +// products says the little that is known about x*y: it is not negative when +// neither factor is, and a square never is. +func (pr *prover) products(p poly) { + for m := range p { + atoms := strings.Split(m, sep) + if len(atoms) != 2 || pr.atoms[m] { + continue + } + pr.atoms[m] = true + x, y := atoms[0], atoms[1] + if x == y { + pr.axioms = append(pr.axioms, le0(poly{m: -1})) + continue + } + pr.axioms = append(pr.axioms, nOr(lt0(poly{x: 1}), lt0(poly{y: 1}), le0(poly{m: -1})), + nOr(lt0(poly{x: 1}), le0(poly{y: 1}), le0(poly{x: 1, m: -1})), // 0 <= x and 1 <= y: x <= xy + nOr(lt0(poly{y: 1}), le0(poly{x: 1}), le0(poly{y: 1, m: -1}))) + } +} + +// congruence says that equal indices select equal elements. +func (pr *prover) congruence() { + pairs := 0 + for _, list := range pr.sels { + for i := range list { + for j := i + 1; j < len(list); j++ { + d := list[i].idx.add(list[j].idx, -1) + if c, ok := d.isConst(); ok && c != 0 { + continue + } + if pairs++; pairs > 80 { + return + } + pr.axioms = append(pr.axioms, nOr(ne0(d), eq0(poly{list[i].name: 1, list[j].name: -1}))) + } + } + } +} + +// refute reports whether the constraints, the boolean assignment and the +// formulas still to do are unsatisfiable together. +func (pr *prover) refute(cons []poly2, bools map[string]bool, todo []*node) bool { + if pr.steps++; pr.steps > maxSteps { + pr.gaveUp = true + return false + } + cons = cons[:len(cons):len(cons)] + var ors []*node + copied := false + for len(todo) > 0 { + n := todo[len(todo)-1] + todo = todo[: len(todo)-1 : len(todo)-1] + switch n.kind { + case 'f': + return true + case 'a': + todo = append(todo, n.kids...) + case 'o': + ors = append(ors, n) + case 'l', 'e': + cons = append(cons, poly2{n.p, n.kind == 'e'}) + case 'b': + if v, ok := bools[n.atom]; ok && v != n.truth { + return true + } + if !copied { + c := make(map[string]bool, len(bools)+1) + for k, v := range bools { + c[k] = v + } + bools, copied = c, true + } + bools[n.atom] = n.truth + } + } + if pr.unsat(cons) { + return true + } + if len(ors) == 0 { + return false + } + // Drop the disjuncts that are already impossible; a disjunction with one + // disjunct left is a fact, and one with none closes the branch. + var rest []*node + var units []*node + for _, o := range ors { + var live []*node + for _, k := range o.kids { + if !pr.dead(cons, bools, k) { + live = append(live, k) + } + } + switch len(live) { + case 0: + return true + case 1: + units = append(units, live[0]) + default: + rest = append(rest, &node{kind: 'o', kids: live}) + } + } + if len(units) > 0 { + return pr.refute(cons, bools, append(rest, units...)) + } + sort.SliceStable(rest, func(i, j int) bool { return len(rest[i].kids) < len(rest[j].kids) }) + for _, k := range rest[0].kids { + if !pr.refute(cons, bools, append(rest[1:len(rest):len(rest)], k)) { + return false + } + } + return true +} + +// dead reports whether a disjunct is a constraint that contradicts what is +// already known. Anything more complicated is left for the branch to find. +func (pr *prover) dead(cons []poly2, bools map[string]bool, n *node) bool { + switch n.kind { + case 'f': + return true + case 'b': + v, ok := bools[n.atom] + return ok && v != n.truth + case 'l', 'e': + return pr.unsat(append(cons[:len(cons):len(cons)], poly2{n.p, n.kind == 'e'})) + } + return false +} + +// poly2 is one constraint: p <= 0, or p = 0. +type poly2 struct { + p poly + eq bool +} + +// unsat decides whether linear constraints have no rational solution, with +// the rounding that makes it sharper over the integers. Every monomial is a +// variable here, so a product is just another unknown. +func (pr *prover) unsat(cons []poly2) bool { + var eqs, les []poly + for _, c := range cons { + if c.eq { + eqs = append(eqs, c.p) + } else { + les = append(les, c.p) + } + } + eqs, les = squeeze(eqs, les) + eqs, les = unfoldProducts(eqs, les) + // Equalities first: solve each for one variable and substitute. + for len(eqs) > 0 { + e := eqs[0] + eqs = eqs[1:] + e, ok := tighten(e, true) + if !ok { + return true + } + if len(e) == 0 { + continue + } + v := pivot(e) + for i := range eqs { + eqs[i] = eliminate(eqs[i], e, v) + } + for i := range les { + les[i] = eliminate(les[i], e, v) + } + } + for { + // Normalise, drop what is trivially true, and keep the tightest of the + // constraints that differ only in their constant. + best := map[string]poly{} + for _, p := range les { + p, _ := tighten(p, false) + if c, ok := p.isConst(); ok { + if c > 0 { + return true + } + continue + } + k := key(p) + if old, ok := best[k]; !ok || p[""] > old[""] { + best[k] = p + } + } + if len(best) == 0 { + return false + } + if len(best) > maxConstraints { + pr.gaveUp = true + return false + } + les = les[:0] + keys := make([]string, 0, len(best)) + for k := range best { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + les = append(les, best[k]) + } + // Eliminate the variable that multiplies the fewest pairs. + up, down := map[string]int{}, map[string]int{} + for _, p := range les { + for m, c := range p { + if m == "" { + continue + } + if c > 0 { + up[m]++ + } else { + down[m]++ + } + } + } + v, cost := "", -1 + for _, p := range les { + for m := range p { + if m == "" { + continue + } + if c := up[m] * down[m]; cost < 0 || c < cost || c == cost && m < v { + v, cost = m, c + } + } + } + var next, pos, neg []poly + for _, p := range les { + switch c := p[v]; { + case c > 0: + pos = append(pos, p) + case c < 0: + neg = append(neg, p) + default: + next = append(next, p) + } + } + for _, p := range pos { + for _, n := range neg { + a, b := p[v], -n[v] + if big(a) || big(b) { + pr.gaveUp = true + return false + } + c := poly{}.add(p, b).add(n, a) + for _, k := range c { + if big(k) { + pr.gaveUp = true + return false + } + } + next = append(next, c) + } + } + les = next + } +} + +// squeeze finds the equalities hidden in pairs of inequalities: p <= 0 and +// -p <= 0. +func squeeze(eqs, les []poly) ([]poly, []poly) { + byKey := map[string]bool{} + for _, p := range les { + byKey[p.String()] = true + } + for _, p := range les { + if len(p) > 0 && byKey[poly{}.add(p, -1).String()] { + eqs = append(eqs, p) + } + } + return eqs, les +} + +// unfoldProducts uses an equality x = e inside the products x occurs in. +// Elimination cannot do that, because to it x*y is one unknown and x another; +// but i = n + 1 is exactly what turns i*i into something about n. +func unfoldProducts(eqs, les []poly) ([]poly, []poly) { + for round := 0; round < 8; round++ { + x, by := "", poly(nil) + for _, e := range eqs { + for m, c := range e { + if m == "" || strings.Contains(m, sep) || c != 1 && c != -1 || !inProduct(m, eqs, les) { + continue + } + rest := poly{}.add(e, -c) // x = rest, once x's own term is taken out + delete(rest, m) + if mentions(rest, m) { + continue + } + if x == "" || m < x { + x, by = m, rest + } + } + } + if x == "" { + break + } + for i := range eqs { + eqs[i] = substitute(eqs[i], x, by) + } + for i := range les { + les[i] = substitute(les[i], x, by) + } + eqs = append(eqs, poly{x: 1}.add(by, -1)) // what substituting made trivial + } + return eqs, les +} + +func mentions(p poly, x string) bool { + for m := range p { + for _, a := range strings.Split(m, sep) { + if a == x { + return true + } + } + } + return false +} + +func inProduct(x string, lists ...[]poly) bool { + for _, list := range lists { + for _, p := range list { + for m := range p { + if strings.Contains(m, sep) && mentions(poly{m: 1}, x) { + return true + } + } + } + } + return false +} + +func substitute(p poly, x string, by poly) poly { + if !mentions(p, x) { + return p + } + out := poly{} + for m, c := range p { + term := poly{"": c} + for _, a := range strings.Split(m, sep) { + switch { + case a == x: + term = term.mul(by) + case a != "": + term = term.mul(poly{a: 1}) + } + } + out = out.add(term, 1) + } + return out +} + +func big(v int64) bool { return v > 1<<30 || v < -(1<<30) } + +func key(p poly) string { + ms := make([]string, 0, len(p)) + for m := range p { + if m != "" { + ms = append(ms, m) + } + } + sort.Strings(ms) + var b strings.Builder + for _, m := range ms { + fmt.Fprintf(&b, "%d*%s;", p[m], m) + } + return b.String() +} + +// tighten divides a constraint by the gcd of its coefficients. Over the +// integers the constant of an inequality rounds up, and an equality whose +// constant does not divide has no solution. +func tighten(p poly, eq bool) (poly, bool) { + var g int64 + for m, c := range p { + if m != "" { + g = gcd(g, c) + } + } + if g == 0 { + c := p[""] + return p, !eq || c == 0 + } + if g == 1 { + return p, true + } + out := poly{} + for m, c := range p { + if m != "" { + out[m] = c / g + } + } + c := p[""] + if eq { + if c%g != 0 { + return p, false + } + c /= g + } else { + c = ceilDiv(c, g) + } + if c != 0 { + out[""] = c + } + return out, true +} + +func gcd(a, b int64) int64 { + if a < 0 { + a = -a + } + if b < 0 { + b = -b + } + for b != 0 { + a, b = b, a%b + } + return a +} + +func ceilDiv(a, b int64) int64 { + q := a / b + if a%b != 0 && (a < 0) == (b < 0) { + q++ + } + return q +} + +// pivot picks the variable of an equality to solve for: one with coefficient +// Β±1 if there is one, so that substituting it loses nothing over the integers. +func pivot(e poly) string { + v := "" + for m, c := range e { + if m == "" { + continue + } + unit := c == 1 || c == -1 + bestUnit := v != "" && (e[v] == 1 || e[v] == -1) + if v == "" || unit && !bestUnit || unit == bestUnit && m < v { + v = m + } + } + return v +} + +// eliminate removes v from p using the equality e = 0. +func eliminate(p, e poly, v string) poly { + c := p[v] + if c == 0 { + return p + } + a := e[v] + // |a|*p - sign(a)*c*e has no v, and scaling an inequality by |a| > 0 is fine. + if a < 0 { + return poly{}.add(p, -a).add(e, c) + } + return poly{}.add(p, a).add(e, -c) +} diff --git a/internal/vego/vego.go b/internal/vego/vego.go new file mode 100644 index 0000000..da32194 --- /dev/null +++ b/internal/vego/vego.go @@ -0,0 +1,572 @@ +// Code generated by litgo from litgo.lit.md; DO NOT EDIT. + +package vego + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "path/filepath" + "sort" + "strings" + + "github.com/tlehman/litgo/internal/lit" +) + +// Marker starts a comment that is a VeGo annotation. +const Marker = "//@" + +// Source is a Go file to verify. +type Source struct { + Path string + Text []byte +} + +// Pos is a 0-based position in a Source. +type Pos struct { + Path string + Line, Col int +} + +// Finding is something the verifier has to say. Ref, when it has a Path, is a +// second position; the message mentions it as {ref}. +type Finding struct { + Pos + EndCol int + Warning bool + Message string + Ref Pos +} + +// Summary counts what a verification did. +type Summary struct { + Functions int // functions with annotations + Proved int // of which every obligation was proved + Obligations int + Names []string // the proved functions +} + +// annot is one annotation: a //@ comment, with the lines that continue it. +type annot struct { + kind string + text string // the formula + e *expr + pos token.Pos + where Pos + endCol int + textCol int // where the formula starts on the line + first int // how much of the formula is on that line + lines int // how many comment lines it takes + used bool +} + +// Annotated reports whether a Go file has annotations at all, which is what +// decides whether verifying it is worth the time. +func Annotated(text []byte) bool { + return strings.Contains(string(text), Marker) || strings.Contains(string(text), "// @ ") +} + +// Prove verifies the annotated Go a document tangles to, and reports at the +// document's own lines. Nothing is written, so it works on a buffer. +func Prove(res *lit.Result) ([]lit.Diag, Summary) { + var total Summary + if res.HasErrors() { + return nil, total + } + packages := map[string][]*lit.Output{} + var dirs []string + for _, o := range res.Files { + if o.File.Lang != "go" || strings.HasSuffix(o.Path, "_test.go") { + continue + } + dir := filepath.Dir(o.Path) + if packages[dir] == nil { + dirs = append(dirs, dir) + } + packages[dir] = append(packages[dir], o) + } + var diags []lit.Diag + seen := map[string]bool{} + for _, dir := range dirs { + var sources []Source + outputs := map[string]*lit.Output{} + annotated := false + for _, o := range packages[dir] { + text := o.Bytes() + annotated = annotated || Annotated(text) + sources = append(sources, Source{o.Path, text}) + outputs[o.Path] = o + } + if !annotated { + continue + } + findings, sum := Verify(sources) + total.Functions += sum.Functions + total.Proved += sum.Proved + total.Obligations += sum.Obligations + total.Names = append(total.Names, sum.Names...) + for _, f := range findings { + o := outputs[f.Path] + line, col, _, ok := o.Map(f.Line, f.Col) + if !ok { + continue + } + d := lit.Diag{File: res.Doc.Path, Line: line, Col: col, Severity: lit.SevError, Message: f.Message, Source: "vego"} + if f.Warning { + d.Severity = lit.SevWarning + } + if f.EndCol > f.Col { + d.EndCol = col + f.EndCol - f.Col + } + if f.Ref.Path != "" { + if l, _, _, ok := outputs[f.Ref.Path].Map(f.Ref.Line, f.Ref.Col); ok { + d.Message = strings.ReplaceAll(d.Message, "{ref}", fmt.Sprintf("line %d", l+1)) + } + } + // A chunk that is used twice is verified twice, and fails twice. + key := fmt.Sprintf("%d:%d:%s", d.Line, d.Col, d.Message) + if !seen[key] { + seen[key] = true + diags = append(diags, d) + } + } + } + return diags, total +} + +// Verify checks every annotated function of one package. +func Verify(sources []Source) ([]Finding, Summary) { + v := &verifier{fset: token.NewFileSet(), funcs: map[string]*fn{}, structs: map[string]map[string]string{}, + consts: map[string]int64{}, preds: map[string]*predicate{}, reported: map[string]bool{}} + for _, s := range sources { + f, err := parser.ParseFile(v.fset, s.Path, s.Text, parser.ParseComments|parser.SkipObjectResolution) + if err != nil { + continue // the syntax check has more to say about it than we do + } + v.files = append(v.files, f) + } + for _, f := range v.files { + v.declarations(f) + } + for _, f := range v.files { + v.functions(f) + } + for _, f := range v.files { + v.annotations(f) + } + var sum Summary + for _, f := range v.ordered { + if !f.annotated { + continue + } + sum.Functions++ + before := v.errors + v.verify(f) + sum.Obligations += f.obligations + if v.errors == before && !f.broken { + sum.Proved++ + sum.Names = append(sum.Names, f.key) + } + } + for _, a := range v.all { + if !a.used && a.e != nil { + v.report(a.where, a.endCol, false, a.kind+" is not attached to anything: Requires goes above a function, Ensures below it, Invariant and Variant at a loop") + } + } + sort.SliceStable(v.findings, func(i, j int) bool { + a, b := v.findings[i], v.findings[j] + if a.Path != b.Path { + return a.Path < b.Path + } + return a.Line < b.Line + }) + return v.findings, sum +} + +type vkind int + +const ( + kNone vkind = iota + kInt + kBool + kSlice + kStruct +) + +type param struct { + name string + kind vkind + typ string // the struct's name, for kStruct +} + +// fn is a function of the package, with its contract. +type fn struct { + key string // name, or Type.name for a method + decl *ast.FuncDecl + recv *param + params []param + results []param + requires []*annot + ensures []*annot + exsures []*annot + measure []*annot + inside []*annot // the annotations in the body + annotated bool + broken bool // an annotation of it does not parse + obligations int +} + +type verifier struct { + fset *token.FileSet + files []*ast.File + funcs map[string]*fn + ordered []*fn + structs map[string]map[string]string + consts map[string]int64 + preds map[string]*predicate + all []*annot + findings []Finding + reported map[string]bool + errors int + fresh int + parseFresh int +} + +func (v *verifier) position(p token.Pos) Pos { + pp := v.fset.Position(p) + return Pos{pp.Filename, pp.Line - 1, pp.Column - 1} +} + +func (v *verifier) report(at Pos, endCol int, warning bool, msg string) { + v.reportRef(at, endCol, warning, msg, Pos{}) +} + +// reportRef reports once per place and message: a contract that fails on +// three paths is still one problem. +func (v *verifier) reportRef(at Pos, endCol int, warning bool, msg string, ref Pos) { + key := fmt.Sprintf("%s:%d:%d:%s", at.Path, at.Line, at.Col, msg) + if v.reported[key] { + return + } + v.reported[key] = true + if !warning { + v.errors++ + } + v.findings = append(v.findings, Finding{Pos: at, EndCol: endCol, Warning: warning, Message: msg, Ref: ref}) +} + +// kindOf reads a type the way the verifier sees it: integers, booleans, +// slices of integers, and structs declared in the package. +func (v *verifier) kindOf(t ast.Expr) (vkind, string) { + switch t := t.(type) { + case *ast.Ident: + switch t.Name { + case "int", "int8", "int16", "int32", "int64": + return kInt, "" + case "bool": + return kBool, "" + } + if _, ok := v.structs[t.Name]; ok { + return kStruct, t.Name + } + case *ast.StarExpr: + if k, name := v.kindOf(t.X); k == kStruct { + return k, name + } + case *ast.ArrayType: + if k, _ := v.kindOf(t.Elt); t.Len == nil && k == kInt { + return kSlice, "" + } + case *ast.ParenExpr: + return v.kindOf(t.X) + } + return kNone, "" +} + +// declarations notes the structs and the integer constants. +func (v *verifier) declarations(f *ast.File) { + for _, d := range f.Decls { + g, ok := d.(*ast.GenDecl) + if !ok { + continue + } + for _, spec := range g.Specs { + switch s := spec.(type) { + case *ast.TypeSpec: + if st, ok := s.Type.(*ast.StructType); ok { + fields := map[string]string{} + for _, fl := range st.Fields.List { + for _, n := range fl.Names { + fields[n.Name] = typeText(fl.Type) + } + } + v.structs[s.Name.Name] = fields + } + case *ast.ValueSpec: + if g.Tok != token.CONST { + continue + } + for i, n := range s.Names { + if i < len(s.Values) { + if c, ok := constInt(s.Values[i]); ok { + v.consts[n.Name] = c + } + } + } + } + } + } +} + +// functions notes the functions, once every struct they may mention is known. +func (v *verifier) functions(f *ast.File) { + for _, d := range f.Decls { + fd, ok := d.(*ast.FuncDecl) + if !ok || fd.Body == nil { + continue + } + f := &fn{key: fd.Name.Name, decl: fd} + if fd.Recv != nil && len(fd.Recv.List) == 1 { + r := fd.Recv.List[0] + k, typ := v.kindOf(r.Type) + name := "_" + if len(r.Names) == 1 { + name = r.Names[0].Name + } + f.recv = ¶m{name, k, typ} + f.key = strings.TrimPrefix(typeText(r.Type), "*") + "." + f.key + } + f.params = v.fields(fd.Type.Params, "") + f.results = v.fields(fd.Type.Results, "ret_") + v.funcs[f.key] = f + v.ordered = append(v.ordered, f) + } +} + +func (v *verifier) fields(list *ast.FieldList, unnamed string) []param { + var out []param + if list == nil { + return nil + } + for _, fl := range list.List { + k, typ := v.kindOf(fl.Type) + if len(fl.Names) == 0 { + out = append(out, param{fmt.Sprintf("%s%d", unnamed, len(out)), k, typ}) + } + for _, n := range fl.Names { + out = append(out, param{n.Name, k, typ}) + } + } + return out +} + +func typeText(t ast.Expr) string { + switch t := t.(type) { + case *ast.Ident: + return t.Name + case *ast.StarExpr: + return "*" + typeText(t.X) + case *ast.ArrayType: + if t.Len == nil { + return "[]" + typeText(t.Elt) + } + } + return "?" +} + +func constInt(e ast.Expr) (int64, bool) { + switch e := e.(type) { + case *ast.BasicLit: + if e.Kind == token.INT { + var v int64 + if _, err := fmt.Sscan(e.Value, &v); err == nil { + return v, true + } + } + case *ast.UnaryExpr: + if c, ok := constInt(e.X); ok && e.Op == token.SUB { + return -c, true + } + case *ast.ParenExpr: + return constInt(e.X) + } + return 0, false +} + +// annotations reads a file's //@ comments, parses them, and hands each to the +// function it belongs to. +func (v *verifier) annotations(f *ast.File) { + var list []*annot + for _, group := range f.Comments { + var open *annot // an annotation whose formula is not finished + for _, c := range group.List { + body, ok := lit.ProofComment(c.Text, open != nil) + if !ok { + open = nil + continue + } + if open != nil { + open.text += " " + strings.TrimSpace(body) + open.lines++ + if !continues(body) { + open = nil + } + continue + } + rest := strings.TrimLeft(body, " \t") + a := &annot{kind: "Assert", pos: c.Pos(), where: v.position(c.Pos()), lines: 1} + a.endCol = a.where.Col + len(c.Text) + word, after, _ := strings.Cut(rest, " ") + if lit.ProofKeywords[word] { + a.kind = word + rest = strings.TrimLeft(after, " \t") + } + a.text = strings.TrimSpace(rest) + a.first = len(a.text) + a.textCol = a.where.Col + len(c.Text) - len(rest) + list = append(list, a) + if continues(body) { + open = a + } + } + } + for _, a := range list { + v.parse(a) + } + v.all = append(v.all, list...) + + var decls []*fn + for _, fd := range v.ordered { + if v.fset.File(fd.decl.Pos()) == v.fset.File(f.Pos()) { + decls = append(decls, fd) + } + } + for _, a := range list { + if a.kind == "Predicate" { + a.used = true + continue + } + if v.inType(f, a) { + v.report(a.where, a.endCol, true, "litgo prove does not check contracts on interface methods") + a.used = true + continue + } + var inside, before, after *fn + for _, fd := range decls { + switch { + case a.pos > fd.decl.Body.Lbrace && a.pos < fd.decl.Body.Rbrace: + inside = fd + case fd.decl.End() <= a.pos: + before = fd + case a.pos < fd.decl.Pos() && after == nil: + after = fd + } + } + // An Ensures belongs to the function above it, unless it is part of + // the doc comment of the function below and does not touch the one above. + if after != nil && after.decl.Doc != nil && a.pos >= after.decl.Doc.Pos() && a.pos < after.decl.Doc.End() && + !(before != nil && v.adjacent(before, a, list)) { + before = nil + } + var owner *fn + switch a.kind { + case "Requires": + if owner = after; inside != nil { + owner = nil + } + if owner != nil { + owner.requires = append(owner.requires, a) + } + case "Ensures", "Exsures": + if inside == nil { + if owner = before; owner == nil { + owner = after + } + } + if owner != nil && a.kind == "Ensures" { + owner.ensures = append(owner.ensures, a) + } else if owner != nil { + owner.exsures = append(owner.exsures, a) + } + case "Measure": + if owner = inside; owner == nil { + owner = after + } + if owner != nil { + owner.measure = append(owner.measure, a) + } + case "Property": + v.report(a.where, a.endCol, true, "litgo prove does not check Property annotations") + a.used = true + continue + default: + if owner = inside; owner != nil { + owner.inside = append(owner.inside, a) + } + } + if owner != nil { + owner.annotated = true + owner.broken = owner.broken || a.e == nil + if a.kind != "Requires" && a.kind != "Ensures" && a.kind != "Exsures" && a.kind != "Measure" { + continue // the body marks its own as used, when it reaches them + } + a.used = true + } + } +} + +// inType reports whether an annotation is written inside a type declaration, +// which is where VeGo puts the contract of an interface method. +func (v *verifier) inType(f *ast.File, a *annot) bool { + for _, d := range f.Decls { + if g, ok := d.(*ast.GenDecl); ok && g.Tok == token.TYPE && a.pos > g.Pos() && a.pos < g.End() { + return true + } + } + return false +} + +// adjacent reports whether nothing but annotations lies between the end of a +// function and an annotation below it. +func (v *verifier) adjacent(f *fn, a *annot, list []*annot) bool { + line := v.position(f.decl.End()).Line + for _, b := range list { + if b.where.Line == line+1 && b.pos > f.decl.End() { + line += b.lines + if b == a { + return true + } + } + } + return false +} + +func (v *verifier) parse(a *annot) { + var err error + switch a.kind { + case "Predicate": + var p *predicate + if p, err = parsePredicate(a.text, &v.parseFresh); err == nil { + v.preds[p.name] = p + a.e = p.body + } + case "Immutable": + a.e = eTrue + default: + a.e, err = parseFormula(a.text, &v.parseFresh) + number := a.kind == "Variant" || a.kind == "Measure" + if err == nil && a.e.op != opCall && isBool(a.e) == number { + a.e = nil + err = &syntaxError{0, "this has to be a formula, something that is true or false"} + if number { + err = &syntaxError{0, "this has to be an integer expression, something that can get smaller"} + } + } + } + if se, ok := err.(*syntaxError); ok { + at := a.where + at.Col = a.textCol + min(se.pos, a.first) + v.report(at, a.endCol, false, "in this "+a.kind+": "+se.msg) + } +} diff --git a/internal/vego/vego_test.go b/internal/vego/vego_test.go new file mode 100644 index 0000000..40e92b7 --- /dev/null +++ b/internal/vego/vego_test.go @@ -0,0 +1,347 @@ +// Code generated by litgo from litgo.lit.md; DO NOT EDIT. + +package vego + +import ( + "go/format" + "os" + "strings" + "testing" + + "github.com/tlehman/litgo/internal/lit" +) + +func verifySrc(t *testing.T, src string) ([]Finding, Summary) { + t.Helper() + return Verify([]Source{{"t.go", []byte("package p\n" + src)}}) +} + +func expectProved(t *testing.T, src string, n int) { + t.Helper() + fs, sum := verifySrc(t, src) + for _, f := range fs { + if !f.Warning { + t.Errorf("unexpected: %d:%d %s", f.Line+1, f.Col+1, f.Message) + } + } + if sum.Proved != n { + t.Errorf("proved %d of %d, want %d", sum.Proved, sum.Functions, n) + } +} + +func expectFail(t *testing.T, src, want string) { + t.Helper() + fs, _ := verifySrc(t, src) + for _, f := range fs { + if !f.Warning && strings.Contains(f.Message, want) { + return + } + } + t.Errorf("no error containing %q in %v", want, fs) +} + +const divide = ` +//@ Requires n >= 0 ^ d > 0 +func Divide(n, d int) (q, r int) { + r = n + //@ Invariant n = q*d + r ^ r >= 0 + //@ Variant r + for r >= d { + q++ + r -= d + } + return q, r +} +//@ Ensures n = q*d + r +//@ Ensures 0 <= r < d +` + +func TestDivide(t *testing.T) { expectProved(t, divide, 1) } + +// gofmt moves annotations around: //@ becomes // @ in a doc comment, and an +// Ensures gets a blank line above it. The proof has to come out the same, and +// so does the refusal. +func TestSurvivesGofmt(t *testing.T) { + for _, src := range []string{divide, search} { + formatted, err := format.Source([]byte("package p\n" + src)) + if err != nil { + t.Fatal(err) + } + text := strings.TrimPrefix(string(formatted), "package p\n") + if !strings.Contains(text, "// @ Requires") || !strings.Contains(text, "}\n\n//@ Ensures") { + t.Fatalf("gofmt no longer does what this test is about:\n%s", text) + } + expectProved(t, text, 1) + } + formatted, _ := format.Source([]byte("package p\n" + strings.Replace(divide, "r -= d", "r -= 1", 1))) + expectFail(t, strings.TrimPrefix(string(formatted), "package p\n"), "Invariant is not maintained") +} + +// A contract may also be written whole above its function. +func TestContractAbove(t *testing.T) { + const above = ` +func First() (x int) { return 1 } + +//@ Requires n >= 0 +//@ Ensures r = n + 1 +func Next(n int) (r int) { return n + 1 } +` + expectProved(t, above, 1) + expectFail(t, strings.Replace(above, "return n + 1", "return n", 1), "Ensures is not proved") +} + +// What the verifier cannot follow, it refuses. +func TestRefusals(t *testing.T) { + expectFail(t, ` +//@ Requires 0 <= i ^ i < |A| +func Set(A []int, i int) (x int) { + A[i] = 0 + return A[i] +} +//@ Ensures x = 0 +`, "only variables can be assigned to") + expectFail(t, ` +func scramble(A []int) { + for i := range A { + A[i] = 0 + } +} + +//@ Requires |A| > 0 ^ A[0] = 1 +func Keeps(A []int) (x int) { + scramble(A) + return A[0] +} +//@ Ensures x = 1 +`, "scramble has no annotations") + expectFail(t, ` +//@ Requires n >= 0 +func Bad(n int) (x int) { + //@ Assert n + 1 + return n +} +`, "this has to be a formula") + expectFail(t, ` +//@ Requires n >= 0 +func Bad(n int) (x int) { + //@ Variant n > 0 + for n > 0 { + n-- + } + return n +} +`, "this has to be an integer expression") + fs, sum := verifySrc(t, ` +type Counter interface { + //@ Requires self.Value >= 0 + Increment() +} + +//@ Requires n >= 0 +func Id(n int) (x int) { return n } +//@ Ensures x = n +`) + if sum.Proved != 1 || len(fs) != 1 || !fs[0].Warning { + t.Errorf("findings = %+v, sum = %+v", fs, sum) + } +} + +func TestDivideWrong(t *testing.T) { + expectFail(t, strings.Replace(divide, "r -= d", "r -= 1", 1), "Invariant is not maintained") + expectFail(t, strings.Replace(divide, "d > 0", "d >= 0", 1), "Variant is not shown to decrease") + expectFail(t, strings.Replace(divide, "0 <= r < d", "0 <= r <= d - 2", 1), "Ensures is not proved") +} + +func TestGauss(t *testing.T) { + expectProved(t, ` +//@ Requires n >= 0 +func Sum(n int) (s int) { + //@ Invariant 0 <= i <= n + 1 ^ 2*s = i*(i-1) + //@ Variant n + 1 - i + for i := 0; i <= n; i++ { + s += i + } + return s +} +//@ Ensures 2*s = n*(n+1) +`, 1) +} + +func TestSqrt(t *testing.T) { + expectProved(t, ` +//@ Requires n >= 0 +func Isqrt(n int) (r int) { + //@ Invariant r >= 0 ^ r*r <= n + //@ Variant n - r*r + for (r+1)*(r+1) <= n { + r++ + } + return r +} +//@ Ensures r*r <= n ^ n < (r+1)*(r+1) +`, 1) +} + +const search = ` +//@ Predicate Sorted(A) ::= Forall i in [0, |A|) : Forall j in [i, |A|) : A[i] <= A[j] + +//@ Requires Sorted(A) +func Search(A []int, x int) (at int, found bool) { + lo, hi := 0, len(A) + //@ Invariant 0 <= lo <= hi <= |A| + //@ Invariant A[0:lo) < x ^ x < A[hi:|A|) + //@ Variant hi - lo + for lo < hi { + mid := lo + (hi-lo)/2 + if A[mid] == x { + return mid, true + } else if A[mid] < x { + lo = mid + 1 + } else { + hi = mid + } + } + return 0, false +} +//@ Ensures found -> 0 <= at < |A| ^ A[at] = x +//@ Ensures ~found -> x <> A[:] +` + +func TestSearch(t *testing.T) { expectProved(t, search, 1) } + +func TestSearchWrong(t *testing.T) { + expectFail(t, strings.Replace(search, "lo = mid + 1", "lo = mid", 1), "Variant is not shown to decrease") + expectFail(t, strings.Replace(search, "hi = mid\n", "hi = mid - 1\n", 1), "Invariant is not maintained") + expectFail(t, strings.Replace(search, "//@ Requires Sorted(A)\n", "", 1), "Invariant is not maintained") +} + +func TestSafety(t *testing.T) { + expectFail(t, ` +//@ Requires i >= 0 +func At(A []int, i int) (x int) { return A[i] } +`, "index may be out of range") + expectFail(t, ` +//@ Requires a >= 0 +func Div(a, b int) (x int) { return a / b } +`, "divisor may be zero") +} + +func TestCallsAndRecursion(t *testing.T) { + expectProved(t, ` +//@ Requires n >= 0 +//@ Measure n +func Fact(n int) (f int) { + //@ BaseCase n = 0 + if n == 0 { + return 1 + } + return n * Fact(n-1) +} +//@ Ensures f >= 1 + +//@ Requires a >= 0 ^ b >= 0 +func Use(a, b int) (x int) { + return Fact(a) + Fact(b) +} +//@ Ensures x >= 2 +`, 2) + expectFail(t, ` +//@ Requires n >= 0 +//@ Measure n +func F(n int) (f int) { + if n == 0 { + return 1 + } + return F(n) +} +`, "Measure is not shown to get smaller") +} + +func TestPrimesAndPreserves(t *testing.T) { + expectProved(t, ` +//@ Requires a = a +func P(a int) (x int) { + x = 0 + x = x + 1 + x = x + 2 + //@ Assert x = 0 ^ x' = 1 ^ x'' = 3 + y := a + //@ Preserves x + y >= a + x++ + y-- + x++ + return x + y +} +//@ Ensures x >= a +`, 1) +} + +// The examples are documentation, and documentation that claims a proof had +// better have one. +func TestExamples(t *testing.T) { + for name, want := range map[string]int{"isqrt": 1, "vego-tutorial": 15, "heap": 10} { + src, err := os.ReadFile("../../examples/" + name + ".lit.md") + if err != nil { + t.Skip(err) + } + diags, sum := Prove(lit.Parse(name+".lit.md", src).Tangle()) + for _, d := range diags { + if d.Severity == lit.SevError { + t.Errorf("%s:%d: %s", name, d.Line+1, d.Message) + } + } + if sum.Proved != want || sum.Functions != want { + t.Errorf("%s: proved %d of %d, want %d", name, sum.Proved, sum.Functions, want) + } + } +} + +// A failure is reported where the annotation is in the Markdown, and the +// line it mentions is a line of the Markdown too. +// litgo proves the arithmetic of its own source map. The test names the two +// functions, so that deleting their annotations does not pass for a proof. +func TestLitgoProvesItself(t *testing.T) { + src, err := os.ReadFile("../../litgo.lit.md") + if err != nil { + t.Skip(err) + } + diags, sum := Prove(lit.Parse("litgo.lit.md", src).Tangle()) + for _, d := range diags { + if d.Severity == lit.SevError { + t.Errorf("litgo.lit.md:%d: %s", d.Line+1, d.Message) + } + } + proved := strings.Join(sum.Names, " ") + if !strings.Contains(proved, "Seg.out") || !strings.Contains(proved, "Seg.src") || sum.Proved != sum.Functions { + t.Errorf("proved %d of %d: %s", sum.Proved, sum.Functions, proved) + } +} + +func TestProveMapsToTheDocument(t *testing.T) { + doc := "\n# Halving\n\n```go\nfunc Halve(n int) (h int) {\n\t// <>\n}\n//@ Ensures 2*h = n\n```\n\n" + + "\n```go\nreturn n / 2\n```\n" + diags, sum := Prove(lit.Parse("halve.lit.md", []byte(doc)).Tangle()) + if sum.Proved != 0 || len(diags) != 1 { + t.Fatalf("diags = %+v, sum = %+v", diags, sum) + } + d := diags[0] + if d.Line != 7 || d.Source != "vego" || !strings.Contains(d.Message, "the return at line 13") { + t.Errorf("diag = %+v", d) + } +} + +func TestOneLine(t *testing.T) { + expectProved(t, ` +type Seg struct{ OutCol, Len, SrcLine, SrcCol int } + +//@ Requires s.SrcCol <= col ^ col <= s.SrcCol + s.Len +func (s Seg) out(col int) (c int) { return s.OutCol + col - s.SrcCol } +//@ Ensures s.OutCol <= c ^ c <= s.OutCol + s.Len +//@ Ensures s.src(c) = col + +//@ Requires s.OutCol <= col ^ col <= s.OutCol + s.Len +func (s Seg) src(col int) (c int) { return s.SrcCol + col - s.OutCol } +//@ Ensures s.SrcCol <= c ^ c <= s.SrcCol + s.Len +//@ Ensures s.out(c) = col +`, 2) +} diff --git a/internal/vego/verify.go b/internal/vego/verify.go new file mode 100644 index 0000000..a8fea96 --- /dev/null +++ b/internal/vego/verify.go @@ -0,0 +1,1451 @@ +// Code generated by litgo from litgo.lit.md; DO NOT EDIT. + +package vego + +import ( + "fmt" + "go/ast" + "go/token" + "sort" + "strings" +) + +// binding is a program variable on one path: its kind, and every value it +// has had since it was declared or since the loop around it was cut. The +// last one is its value now; the ones before are what primes reach for. +type binding struct { + kind vkind + typ string + hist []*expr +} + +// state is one path through a function: what the variables hold, and what is +// known to be true. +type state struct { + vars map[string]*binding + scopes []map[string]*binding // what each open block shadowed + hyps []*expr + preserved []kept + base []*expr // the BaseCase conditions passed so far +} + +type kept struct { + a *annot + depth int +} + +func (s *state) clone() *state { + c := &state{vars: make(map[string]*binding, len(s.vars)), hyps: s.hyps[:len(s.hyps):len(s.hyps)], + preserved: s.preserved[:len(s.preserved):len(s.preserved)], base: s.base[:len(s.base):len(s.base)]} + for n, b := range s.vars { + c.vars[n] = &binding{b.kind, b.typ, append([]*expr{}, b.hist...)} + } + for _, sc := range s.scopes { + m := make(map[string]*binding, len(sc)) + for n, b := range sc { + m[n] = b + } + c.scopes = append(c.scopes, m) + } + return c +} + +func (s *state) push() { s.scopes = append(s.scopes, map[string]*binding{}) } + +func (s *state) pop() { + top := s.scopes[len(s.scopes)-1] + s.scopes = s.scopes[:len(s.scopes)-1] + for n, old := range top { + for f := range s.vars { + if strings.HasPrefix(f, n+".") { + delete(s.vars, f) + } + } + if old == nil { + delete(s.vars, n) + } else { + s.vars[n] = old + } + } + for len(s.preserved) > 0 && s.preserved[len(s.preserved)-1].depth > len(s.scopes) { + s.preserved = s.preserved[:len(s.preserved)-1] + } +} + +func (s *state) popTo(depth int) { + for len(s.scopes) > depth { + s.pop() + } +} + +func (s *state) declare(name string, b *binding) { + if name == "_" { + return + } + if top := s.scopes[len(s.scopes)-1]; top != nil { + if _, ok := top[name]; !ok { + top[name] = s.vars[name] + } + } + for f := range s.vars { + if strings.HasPrefix(f, name+".") { + delete(s.vars, f) + } + } + s.vars[name] = b +} + +// outcomes are the states a statement leaves behind, by how it was left. +// A return leaves none: its obligations are settled on the spot. +type outcomes struct { + normal, brk, cont []*state +} + +func (o *outcomes) add(p outcomes) { + o.normal = append(o.normal, p.normal...) + o.brk = append(o.brk, p.brk...) + o.cont = append(o.cont, p.cont...) +} + +// run verifies one function. +type run struct { + v *verifier + f *fn + entry map[string]*expr // the parameters' values on entry + blocks map[*ast.BlockStmt][]*annot + measure0 []*expr + paths int + depth int // of one-line functions being unfolded +} + +// unsupported is how the verifier backs out of a function it cannot follow. +type unsupported struct { + pos token.Pos + msg string +} + +func (r *run) no(n ast.Node, format string, args ...any) { + panic(unsupported{n.Pos(), fmt.Sprintf(format, args...)}) +} + +const maxPaths = 400 + +func (v *verifier) verify(f *fn) { + r := &run{v: v, f: f, entry: map[string]*expr{}, blocks: map[*ast.BlockStmt][]*annot{}} + defer func() { + if p := recover(); p != nil { + u, ok := p.(unsupported) + if !ok { + // A bug in the verifier. The editor is on the other end of this, + // so say so at the function instead of taking the server down. + u = unsupported{f.decl.Name.Pos(), fmt.Sprintf("the verifier failed (%v); please report this", p)} + } + at := v.position(u.pos) + v.report(at, 0, false, fmt.Sprintf("%s is not verified: %s", f.key, u.msg)) + } + }() + r.place() + st := &state{vars: map[string]*binding{}} + st.push() + all := append([]param{}, f.params...) + if f.recv != nil { + all = append(all, *f.recv) + } + for _, p := range all { + if p.kind == kNone && p.name != "_" { + // Whatever it is, the proof cannot mention it. + continue + } + b := &binding{kind: p.kind, typ: p.typ} + if p.kind != kStruct { + b.hist = []*expr{r.fresh(p.name, p.kind)} + r.entry[p.name] = b.hist[0] + } + st.declare(p.name, b) + } + for _, p := range f.results { + if p.kind != kInt && p.kind != kBool { + r.no(f.decl, "its result %s is neither an integer nor a boolean", p.name) + } + zero := num(0) + if p.kind == kBool { + zero = eFalse + } + st.declare(p.name, &binding{kind: p.kind, hist: []*expr{zero}}) + } + for _, a := range f.requires { + if a.e != nil { + st.hyps = append(st.hyps, r.formula(st, a, false)) + } + } + for _, a := range f.measure { + if a.e != nil { + r.measure0 = append(r.measure0, r.formula(st, a, false)) + } + } + r.lint() + if r.recursive() && len(f.measure) == 0 { + v.report(v.position(f.decl.Name.Pos()), 0, true, f.key+" calls itself and has no Measure, so nothing shows that it terminates") + } + out := r.block(st, f.decl.Body) + for _, s := range out.normal { + r.leave(s, f.decl.Body.Rbrace) + } +} + +// place sorts the annotations of the body into the blocks they are written in. +func (r *run) place() { + for _, a := range r.f.inside { + var inner *ast.BlockStmt + ast.Inspect(r.f.decl.Body, func(n ast.Node) bool { + if b, ok := n.(*ast.BlockStmt); ok && a.pos > b.Lbrace && a.pos < b.Rbrace { + inner = b + } + return true + }) + if inner != nil { + r.blocks[inner] = append(r.blocks[inner], a) + } + } +} + +// lint points at contracts that say nothing. +func (r *run) lint() { + for _, list := range [][]*annot{r.f.requires, r.f.ensures} { + for _, a := range list { + if a.e != nil && a.e.op == opTrue { + r.v.report(a.where, a.endCol, true, a.kind+" true says nothing") + } + } + } +} + +func (r *run) recursive() bool { + found := false + ast.Inspect(r.f.decl.Body, func(n ast.Node) bool { + if c, ok := n.(*ast.CallExpr); ok { + if callee, _ := r.callee(nil, c); callee == r.f { + found = true + } + } + return true + }) + return found +} + +func (r *run) fresh(name string, k vkind) *expr { + r.v.fresh++ + s := sInt + if k == kBool { + s = sBool + } else if k == kSlice { + s = sArr + } + return sym(fmt.Sprintf("%s#%d", name, r.v.fresh), s) +} + +// check is one proof obligation. It reports at the annotation, or at a +// place in the code, and mentions a second place if there is one. +func (r *run) check(st *state, goal *expr, a *annot, at ast.Node, ref ast.Node, msg string) bool { + r.f.obligations++ + if proves(st.hyps, goal) { + return true + } + where, end := Pos{}, 0 + if a != nil { + where, end = a.where, a.endCol + } else { + where = r.v.position(at.Pos()) + end = where.Col + int(at.End()-at.Pos()) + } + var rp Pos + if ref != nil { + rp = r.v.position(ref.Pos()) + } + r.v.reportRef(where, end, false, msg, rp) + return false +} + +func (r *run) assume(st *state, f *expr) { st.hyps = append(st.hyps, f) } + +// feasible reports whether a path can be taken at all. Pruning the ones that +// cannot keeps their obligations from being reported as failures of logic. +func (r *run) feasible(st *state) bool { return !proves(st.hyps, eFalse) } + +// formula gives an annotation's formula its meaning in a state: predicates +// and one-line functions unfolded, names replaced by values. In a +// postcondition a parameter means its value on entry, and primed, its value +// at the end. +func (r *run) formula(st *state, a *annot, post bool) *expr { + e := r.unfold(st, a, a.e, 0) + most := map[string]int{} + var count func(e *expr) + count = func(e *expr) { + if e == nil { + return + } + if e.op == opVar { + most[e.name] = max(most[e.name], e.primes) + } + for _, x := range e.args { + count(x) + } + } + count(e) + return rewrite(e, func(v *expr) *expr { + if strings.ContainsAny(v.name, "#?") { + return nil + } + if old, ok := r.entry[v.name]; ok && post { + if v.primes == 0 { + return old + } + return r.lookup(st, a, v.name, 0) + } + return r.lookup(st, a, v.name, most[v.name]-v.primes) + }) +} + +// lookup finds the value a name had `back` assignments ago. +func (r *run) lookup(st *state, a *annot, name string, back int) *expr { + b := r.binding(st, name) + if b == nil { + if c, ok := r.v.consts[name]; ok { + return num(c) + } + r.bad(a, "%s is not a variable here", name) + } + if b.kind == kStruct || len(b.hist) == 0 { + r.bad(a, "%s is not an integer, a boolean or a slice of integers", name) + } + if back >= len(b.hist) { + r.bad(a, "%s has not been assigned often enough on this path for that many primes", name) + } + return b.hist[len(b.hist)-1-back] +} + +// binding finds a variable, or makes one for the field of a struct the +// first time it is mentioned. Nothing in the fragment assigns to a field, so +// a field holds one value throughout. +func (r *run) binding(st *state, name string) *binding { + if b := st.vars[name]; b != nil { + return b + } + base, field, ok := strings.Cut(name, ".") + if !ok { + return nil + } + s := st.vars[base] + if s == nil || s.kind != kStruct { + return nil + } + var k vkind + switch r.v.structs[s.typ][field] { + case "int", "int8", "int16", "int32", "int64": + k = kInt + case "bool": + k = kBool + case "[]int": + k = kSlice + default: + return nil + } + b := &binding{kind: k, hist: []*expr{r.fresh(name, k)}} + st.vars[name] = b + return b +} + +// bad gives up on an annotation that does not make sense. +func (r *run) bad(a *annot, format string, args ...any) { + msg := fmt.Sprintf(format, args...) + if a == nil { + panic(unsupported{r.f.decl.Pos(), msg}) + } + r.v.report(a.where, a.endCol, false, "in this "+a.kind+": "+msg) + panic(unsupported{r.f.decl.Name.Pos(), "one of its annotations does not make sense"}) +} + +// unfold replaces P(x, y) by what it stands for: the body of a predicate, or +// the expression a one-line Go function returns. +func (r *run) unfold(st *state, a *annot, e *expr, depth int) *expr { + if e == nil { + return nil + } + if depth > 12 { + r.bad(a, "%s is defined in terms of itself", e.name) + } + c := *e + c.args = make([]*expr, len(e.args)) + for i, x := range e.args { + c.args[i] = r.unfold(st, a, x, depth) + } + if e.op != opCall { + return &c + } + if p := r.v.preds[e.name]; p != nil { + if len(p.params) != len(c.args) { + r.bad(a, "%s takes %d arguments", p.name, len(p.params)) + } + return r.unfold(st, a, replace(p.body, p.params, c.args, "", ""), depth+1) + } + callee, recv := r.named(st, e.name) + if callee == nil { + r.bad(a, "%s is neither a predicate nor a function of this package", e.name) + } + body := r.oneLine(callee) + if body == nil { + r.bad(a, "%s cannot be used in a formula: only a function whose body is one return of one expression can", e.name) + } + if len(callee.params) != len(c.args) { + r.bad(a, "%s takes %d arguments", e.name, len(callee.params)) + } + var formals []string + for _, p := range callee.params { + formals = append(formals, p.name) + } + from := "" + if callee.recv != nil { + from = callee.recv.name + } + return r.unfold(st, a, replace(body, formals, c.args, from, recv), depth+1) +} + +// replace substitutes all the parameters at once, and renames the fields of +// the callee's receiver to fields of the caller's. +func replace(body *expr, formals []string, actuals []*expr, from, to string) *expr { + return rewrite(body, func(v *expr) *expr { + for i, f := range formals { + if v.name == f && v.primes == 0 { + return actuals[i] + } + } + if from != "" && strings.HasPrefix(v.name, from+".") { + c := *v + c.name = to + strings.TrimPrefix(v.name, from) + return &c + } + return nil + }) +} + +// named finds the function a formula or a call mentions: f, or x.m where x +// is a struct. +func (r *run) named(st *state, name string) (*fn, string) { + if f := r.v.funcs[name]; f != nil && f.recv == nil { + return f, "" + } + if i := strings.LastIndex(name, "."); i > 0 && st != nil { + if b := st.vars[name[:i]]; b != nil && b.kind == kStruct { + return r.v.funcs[b.typ+name[i:]], name[:i] + } + } + return nil, "" +} + +// oneLine is the expression a function returns, if that is all it does. +func (r *run) oneLine(f *fn) *expr { + if len(f.decl.Body.List) != 1 || len(f.results) != 1 || len(f.inside) > 0 { + return nil + } + ret, ok := f.decl.Body.List[0].(*ast.ReturnStmt) + if !ok || len(ret.Results) != 1 { + return nil + } + kinds := map[string]vkind{} + for _, p := range f.params { + kinds[p.name] = p.kind + } + var e *expr + func() { + defer func() { + if p := recover(); p != nil { + if _, ok := p.(unsupported); !ok { + panic(p) + } + e = nil + } + }() + e = r.pure(ret.Results[0], func(name string) vkind { + if k, ok := kinds[name]; ok { + return k + } + if f.recv != nil && strings.HasPrefix(name, f.recv.name+".") { + if r.v.structs[f.recv.typ][strings.TrimPrefix(name, f.recv.name+".")] == "bool" { + return kBool + } + } + return kInt + }) + }() + return e +} + +// pure reads a Go expression as a term over the names it mentions. It +// checks nothing yet; pos remembers where each part came from. +func (r *run) pure(e ast.Expr, kind func(string) vkind) *expr { + at := func(x *expr) *expr { x.pos = int(e.Pos()); return x } + switch e := e.(type) { + case *ast.ParenExpr: + return r.pure(e.X, kind) + case *ast.BasicLit: + if c, ok := constInt(e); ok { + return at(num(c)) + } + case *ast.Ident: + switch e.Name { + case "true": + return eTrue + case "false": + return eFalse + } + s := sInt + if kind(e.Name) == kBool { + s = sBool + } else if kind(e.Name) == kSlice { + s = sArr + } + return at(sym(e.Name, s)) + case *ast.SelectorExpr: + if x, ok := e.X.(*ast.Ident); ok { + name := x.Name + "." + e.Sel.Name + s := sInt + if kind(name) == kBool { + s = sBool + } else if kind(name) == kSlice { + s = sArr + } + return at(sym(name, s)) + } + case *ast.UnaryExpr: + x := r.pure(e.X, kind) + switch e.Op { + case token.SUB: + return at(mk(opNeg, x)) + case token.ADD: + return x + case token.NOT: + return at(mk(opNot, x)) + } + case *ast.BinaryExpr: + l, rt := r.pure(e.X, kind), r.pure(e.Y, kind) + switch e.Op { + case token.ADD: + return at(mk(opAdd, l, rt)) + case token.SUB: + return at(mk(opSub, l, rt)) + case token.MUL: + return at(mk(opMul, l, rt)) + case token.QUO: + return at(mk(opDiv, l, rt)) + case token.REM: + return at(mk(opMod, l, rt)) + case token.LAND: + return at(mk(opAnd, l, rt)) + case token.LOR: + return at(mk(opOr, l, rt)) + case token.LSS: + return at(mk(opLt, l, rt)) + case token.LEQ: + return at(mk(opLe, l, rt)) + case token.GTR: + return at(mk(opLt, rt, l)) + case token.GEQ: + return at(mk(opLe, rt, l)) + case token.EQL, token.NEQ: + var c *expr + if isBool(l) || isBool(rt) { + c = mk(opIff, l, rt) + } else { + c = mk(opEq, l, rt) + } + if e.Op == token.NEQ { + c = mk(opNot, c) + } + return at(c) + } + case *ast.IndexExpr: + a := r.pure(e.X, kind) + if a.op == opVar { + a.sort = sArr + return at(mk(opSel, a, r.pure(e.Index, kind))) + } + case *ast.CallExpr: + name := "" + switch f := e.Fun.(type) { + case *ast.Ident: + name = f.Name + case *ast.SelectorExpr: + if x, ok := f.X.(*ast.Ident); ok { + name = x.Name + "." + f.Sel.Name + } + } + if name == "" { + break + } + var args []*expr + for _, x := range e.Args { + args = append(args, r.pure(x, kind)) + } + if name == "len" && len(args) == 1 && args[0].op == opVar { + args[0].sort = sArr + return at(mk(opLen, args[0])) + } + if name == "int" && len(args) == 1 { + return args[0] + } + c := at(&expr{op: opCall, name: name, args: args}) + return c + } + r.no(e, "this expression is outside what litgo prove understands (integers, booleans, slices of integers, fields of structs, calls of verified functions)") + return nil +} + +// value evaluates a Go expression in a state, and states what has to be +// true for the evaluation to be safe: indices in range, divisors not zero, +// the preconditions of what is called. +func (r *run) value(st *state, e ast.Expr) *expr { + p := r.pure(e, func(name string) vkind { + if b := r.binding(st, name); b != nil { + return b.kind + } + return kNone + }) + vals := r.eval(st, p, nil, e) + if len(vals) != 1 { + r.no(e, "a call with %d results cannot be used as a value", len(vals)) + } + return vals[0] +} + +// eval does the work of value. The guards are what is known at this point +// of the expression and not before: the left side of an && while its right +// side is being evaluated. +func (r *run) eval(st *state, e *expr, guards []*expr, at ast.Node) []*expr { + one := func(x *expr) *expr { + vals := r.eval(st, x, guards, at) + if len(vals) != 1 { + r.no(at, "a call with %d results cannot be used as a value", len(vals)) + } + return vals[0] + } + guarded := func(goal *expr) *expr { + if len(guards) == 0 { + return goal + } + return mk(opImp, conj(guards), goal) + } + here := ast.Node(at) + if e.pos != 0 { + here = point(e.pos) + } + switch e.op { + case opNum, opTrue, opFalse: + return []*expr{e} + case opVar: + if strings.ContainsAny(e.name, "#?") { + return []*expr{e} + } + b := r.binding(st, e.name) + if b == nil { + if c, ok := r.v.consts[e.name]; ok { + return []*expr{num(c)} + } + r.no(here, "%s is not a variable the proof can talk about", e.name) + } + if len(b.hist) == 0 { + r.no(here, "%s is a struct, and only its fields have values", e.name) + } + return []*expr{b.hist[len(b.hist)-1]} + case opAnd, opOr: + l := one(e.args[0]) + g := l + if e.op == opOr { + g = not(l) + } + rt := r.eval(st, e.args[1], append(guards[:len(guards):len(guards)], g), at) + return []*expr{mk(e.op, l, rt[0])} + case opSel: + arr, idx := one(e.args[0]), one(e.args[1]) + in := mk(opAnd, mk(opLe, num(0), idx), mk(opLt, idx, mk(opLen, arr))) + r.check(st, guarded(in), nil, here, nil, "the index may be out of range: 0 <= "+e.args[1].String()+" < len("+e.args[0].String()+") is not proved") + r.assume(st, guarded(in)) + return []*expr{mk(opSel, arr, idx)} + case opDiv, opMod: + l, d := one(e.args[0]), one(e.args[1]) + r.check(st, guarded(mk(opNe, d, num(0))), nil, here, nil, "the divisor may be zero: "+e.args[1].String()+" <> 0 is not proved") + r.assume(st, guarded(mk(opNe, d, num(0)))) + return []*expr{mk(e.op, l, d)} + case opCall: + var args []*expr + for _, x := range e.args { + args = append(args, one(x)) + } + return r.call(st, e, args, guards, here) + } + c := *e + c.args = make([]*expr, len(e.args)) + for i, x := range e.args { + c.args[i] = one(x) + } + return []*expr{&c} +} + +// point is a position that stands in for a node. +type point token.Pos + +func (p point) Pos() token.Pos { return token.Pos(p) } +func (p point) End() token.Pos { return token.Pos(p) + 1 } + +// callee finds the declared function a call expression calls. +func (r *run) callee(st *state, c *ast.CallExpr) (*fn, string) { + switch f := c.Fun.(type) { + case *ast.Ident: + if fn := r.v.funcs[f.Name]; fn != nil && fn.recv == nil { + return fn, "" + } + case *ast.SelectorExpr: + x, ok := f.X.(*ast.Ident) + if !ok { + return nil, "" + } + if st == nil { // asked only whether the function calls itself + if r.f.recv != nil && x.Name == r.f.recv.name && r.f.key == r.f.recv.typ+"."+f.Sel.Name { + return r.f, x.Name + } + return nil, "" + } + return r.named(st, x.Name+"."+f.Sel.Name) + } + return nil, "" +} + +// call uses a function by its contract: the precondition is an obligation, +// the postcondition is what is known about the results. A function that is +// one expression is used as that expression, which says more. +func (r *run) call(st *state, e *expr, args []*expr, guards []*expr, at ast.Node) []*expr { + switch e.name { + case "min", "max": + if len(args) != 2 || r.v.funcs[e.name] != nil { + break + } + m := r.fresh(e.name, kInt) + lo, hi := m, args[0] + lo2, hi2 := m, args[1] + if e.name == "max" { + lo, hi, lo2, hi2 = args[0], m, args[1], m + } + r.assume(st, conj([]*expr{mk(opLe, lo, hi), mk(opLe, lo2, hi2), mk(opOr, mk(opEq, m, args[0]), mk(opEq, m, args[1]))})) + return []*expr{m} + } + callee, recv := r.named(st, e.name) + if callee == nil { + r.no(at, "%s has no contract the proof could use; only functions of this package can be called", e.name) + } + if len(args) != len(callee.params) { + r.no(at, "%s takes %d arguments", e.name, len(callee.params)) + } + if !callee.annotated && r.oneLine(callee) == nil { + // Nobody has checked what it does to the slices and structs it can reach. + r.no(at, "%s has no annotations, so nothing is known about what it does; give it a contract, even an empty one like Requires true", e.name) + } + guarded := func(goal *expr) *expr { + if len(guards) == 0 { + return goal + } + return mk(opImp, conj(guards), goal) + } + // The callee's formulas, in the caller's terms. + inst := func(a *annot, results []*expr) *expr { + if a.e == nil { + return eTrue + } + sub := &state{vars: map[string]*binding{}} + sub.push() + for i, p := range callee.params { + if p.kind == kNone { + continue + } + sub.vars[p.name] = &binding{kind: p.kind, typ: p.typ, hist: []*expr{args[i]}} + if results != nil && p.kind != kSlice { + // Primed, a parameter is the callee's own copy when it returned, + // and the caller knows nothing about that. + sub.vars[p.name].hist = []*expr{r.fresh(p.name, p.kind)} + } + } + if callee.recv != nil && callee.recv.kind == kStruct { + sub.vars[callee.recv.name] = &binding{kind: kStruct, typ: callee.recv.typ} + for field := range r.v.structs[callee.recv.typ] { + if b := r.binding(st, recv+"."+field); b != nil { + sub.vars[callee.recv.name+"."+field] = b + } + } + } + for i, p := range callee.results { + if results != nil { + sub.vars[p.name] = &binding{kind: p.kind, hist: []*expr{results[i]}} + } + } + inner := &run{v: r.v, f: callee, entry: map[string]*expr{}} + for i, p := range callee.params { + if p.kind != kNone && p.kind != kStruct { + inner.entry[p.name] = args[i] + } + } + defer func() { + // What goes wrong in the callee's annotation is the callee's problem. + if p := recover(); p != nil { + if _, ok := p.(unsupported); !ok { + panic(p) + } + r.no(at, "the contract of %s cannot be used here", e.name) + } + }() + return inner.formula(sub, a, results != nil) + } + for _, a := range callee.requires { + r.check(st, guarded(inst(a, nil)), nil, at, nil, fmt.Sprintf("this call does not establish what %s requires: %s", callee.key, a.text)) + } + if callee == r.f { + r.descends(st, e, args, guards, at) + } + if body := r.oneLine(callee); body != nil { + if r.depth++; r.depth > 40 { + r.no(at, "%s never stops calling itself", callee.key) + } + defer func() { r.depth-- }() + var formals []string + for _, p := range callee.params { + formals = append(formals, p.name) + } + from := "" + if callee.recv != nil { + from = callee.recv.name + } + return r.eval(st, replace(body, formals, args, from, recv), guards, at) + } + var results []*expr + for _, p := range callee.results { + results = append(results, r.fresh(callee.decl.Name.Name+"."+p.name, p.kind)) + } + for _, a := range callee.ensures { + r.assume(st, guarded(inst(a, results))) + } + for _, a := range callee.exsures { + r.assume(st, guarded(not(inst(a, results)))) + } + return results +} + +// descends checks a recursive call: the measure is smaller than it was on +// entry and was not negative, and no base case has been passed. +func (r *run) descends(st *state, e *expr, args []*expr, guards []*expr, at ast.Node) { + guarded := func(goal *expr) *expr { + if len(guards) == 0 { + return goal + } + return mk(opImp, conj(guards), goal) + } + for i, a := range r.f.measure { + if a.e == nil { + continue + } + sub := &state{vars: map[string]*binding{}} + for j, p := range r.f.params { + if p.kind != kNone && p.kind != kStruct { + sub.vars[p.name] = &binding{kind: p.kind, hist: []*expr{args[j]}} + } + } + m := (&run{v: r.v, f: r.f, entry: map[string]*expr{}}).formula(sub, a, false) + r.check(st, guarded(mk(opLt, m, r.measure0[i])), a, nil, at, "Measure is not shown to get smaller in the recursive call at {ref}") + r.check(st, guarded(mk(opLe, num(0), r.measure0[i])), a, nil, at, "Measure can be negative when the recursive call at {ref} is made") + } + for _, b := range st.base { + r.check(st, guarded(not(b)), nil, at, nil, "this recursive call can be reached in the base case") + } +} + +// block runs the statements of a block, with the annotations between them. +func (r *run) block(st *state, b *ast.BlockStmt) outcomes { + st.push() + depth := len(st.scopes) + var out outcomes + live := []*state{st} + notes := r.blocks[b] + n := 0 + for i := 0; i <= len(b.List); i++ { + var next ast.Stmt + limit := b.Rbrace + if i < len(b.List) { + next = b.List[i] + limit = next.Pos() + } + var pending []*annot + for n < len(notes) && notes[n].pos < limit { + a := notes[n] + n++ + if a.kind == "Invariant" || a.kind == "Variant" { + if isLoop(next) { + pending = append(pending, a) + } + continue // at the end of a loop's body it belongs to that loop + } + for _, s := range live { + r.note(s, a) + } + } + if next == nil { + break + } + var after []*state + for _, s := range live { + o := r.stmt(s, next, pending) + after = append(after, o.normal...) + out.brk = append(out.brk, o.brk...) + out.cont = append(out.cont, o.cont...) + } + live = after + if r.paths += len(live); r.paths > maxPaths*50 || len(live) > maxPaths { + r.no(next, "there are too many paths through it") + } + } + for _, s := range live { + s.popTo(depth - 1) + } + out.normal = live + return out +} + +func isLoop(s ast.Stmt) bool { + switch s.(type) { + case *ast.ForStmt, *ast.RangeStmt: + return true + } + return false +} + +// note handles an annotation that stands where a statement could. +func (r *run) note(st *state, a *annot) { + a.used = true + if a.e == nil { + return + } + switch a.kind { + case "Assert", "InductionHypothesis": + f := r.formula(st, a, false) + r.check(st, f, a, nil, nil, a.kind+" is not proved: "+a.text) + r.assume(st, f) + case "Axiom": + r.v.report(a.where, a.endCol, true, "Axiom is taken on trust, not proved: "+a.text) + r.assume(st, r.formula(st, a, false)) + case "Preserves": + f := r.formula(st, a, false) + r.check(st, f, a, nil, nil, "Preserves does not hold to begin with: "+a.text) + r.assume(st, f) + st.preserved = append(st.preserved, kept{a, len(st.scopes)}) + case "BaseCase": + st.base = append(st.base, r.formula(st, a, false)) + case "Immutable": + default: + r.v.report(a.where, a.endCol, false, a.kind+" does not belong inside a function body") + } +} + +func (r *run) stmt(st *state, s ast.Stmt, pending []*annot) outcomes { + switch s := s.(type) { + case *ast.EmptyStmt: + return outcomes{normal: []*state{st}} + case *ast.BlockStmt: + return r.block(st, s) + case *ast.DeclStmt: + g, ok := s.Decl.(*ast.GenDecl) + if !ok || g.Tok != token.VAR { + r.no(s, "only variables can be declared inside a verified function") + } + for _, spec := range g.Specs { + vs := spec.(*ast.ValueSpec) + for i, n := range vs.Names { + var val *expr + k := kNone + if vs.Type != nil { + k, _ = r.v.kindOf(vs.Type) + } + if i < len(vs.Values) { + val = r.value(st, vs.Values[i]) + if k == kNone { + k = kInt + if isBool(val) { + k = kBool + } + } + } else if k == kInt { + val = num(0) + } else if k == kBool { + val = eFalse + } + if val == nil || k != kInt && k != kBool { + r.no(s, "%s is neither an integer nor a boolean", n.Name) + } + st.declare(n.Name, &binding{kind: k, hist: []*expr{val}}) + } + } + return outcomes{normal: []*state{st}} + case *ast.AssignStmt: + r.assign(st, s) + return outcomes{normal: []*state{st}} + case *ast.IncDecStmt: + one := mk(opAdd, r.value(st, s.X), num(1)) + if s.Tok == token.DEC { + one.op = opSub + } + r.set(st, s.X, one, s) + return outcomes{normal: []*state{st}} + case *ast.ExprStmt: + r.effect(st, s) + return outcomes{normal: []*state{st}} + case *ast.ReturnStmt: + r.ret(st, s) + return outcomes{} + case *ast.BranchStmt: + if s.Label != nil { + r.no(s, "labels are outside what litgo prove understands") + } + switch s.Tok { + case token.BREAK: + return outcomes{brk: []*state{st}} + case token.CONTINUE: + return outcomes{cont: []*state{st}} + } + case *ast.IfStmt: + return r.branch(st, s) + case *ast.ForStmt: + return r.loop(st, &loop{stmt: s, init: s.Init, cond: s.Cond, post: s.Post, body: s.Body}, pending) + case *ast.RangeStmt: + return r.loop(st, &loop{stmt: s, over: s, body: s.Body}, pending) + } + r.no(s, "this statement is outside what litgo prove understands") + return outcomes{} +} + +// effect is a call made for what it does. A call of a verified function is +// checked like any other. A call of anything else is let through if all it +// is handed are numbers, booleans and literals, because then it cannot +// change anything the proof is about. panic ends the path. +func (r *run) effect(st *state, s *ast.ExprStmt) { + c, ok := s.X.(*ast.CallExpr) + if !ok { + r.no(s, "this statement is outside what litgo prove understands") + } + if id, ok := c.Fun.(*ast.Ident); ok && id.Name == "panic" { + r.assume(st, eFalse) + return + } + if callee, _ := r.callee(st, c); callee != nil { + r.eval(st, r.pure(c, func(name string) vkind { + if b := r.binding(st, name); b != nil { + return b.kind + } + return kNone + }), nil, c) + return + } + if sel, ok := c.Fun.(*ast.SelectorExpr); ok { + if x, ok := sel.X.(*ast.Ident); ok && st.vars[x.Name] != nil { + r.no(c, "%s.%s has no contract, and a method may change what it is called on", x.Name, sel.Sel.Name) + } + } + for _, a := range c.Args { + if lit, ok := a.(*ast.BasicLit); ok && lit.Kind != token.INT { + continue + } + if v := r.value(st, a); v.sort == sArr { + r.no(a, "a slice handed to a function without a contract may come back changed") + } + } +} + +func (r *run) assign(st *state, s *ast.AssignStmt) { + ops := map[token.Token]op{token.ADD_ASSIGN: opAdd, token.SUB_ASSIGN: opSub, token.MUL_ASSIGN: opMul, + token.QUO_ASSIGN: opDiv, token.REM_ASSIGN: opMod} + if o, ok := ops[s.Tok]; ok { + bin := &ast.BinaryExpr{X: s.Lhs[0], OpPos: s.TokPos, Y: s.Rhs[0], + Op: map[op]token.Token{opAdd: token.ADD, opSub: token.SUB, opMul: token.MUL, opDiv: token.QUO, opMod: token.REM}[o]} + r.set(st, s.Lhs[0], r.value(st, bin), s) + return + } + if s.Tok != token.ASSIGN && s.Tok != token.DEFINE { + r.no(s, "this assignment is outside what litgo prove understands") + } + var vals []*expr + if len(s.Rhs) == 1 && len(s.Lhs) > 1 { + c, ok := s.Rhs[0].(*ast.CallExpr) + if !ok { + r.no(s, "only a call can give several values at once") + } + vals = r.eval(st, r.pure(c, func(name string) vkind { + if b := r.binding(st, name); b != nil { + return b.kind + } + return kNone + }), nil, c) + } else { + for _, x := range s.Rhs { + vals = append(vals, r.value(st, x)) + } + } + if len(vals) != len(s.Lhs) { + r.no(s, "the two sides of this assignment do not match") + } + for i, l := range s.Lhs { + id, ok := l.(*ast.Ident) + if s.Tok == token.DEFINE && ok && (st.vars[id.Name] == nil || !r.inScope(st, id.Name)) { + k := kInt + if isBool(vals[i]) { + k = kBool + } else if vals[i].sort == sArr { + r.no(s, "slices cannot be copied inside a verified function") + } + st.declare(id.Name, &binding{kind: k, hist: []*expr{vals[i]}}) + continue + } + r.store(st, l, vals[i]) + } + // A parallel assignment is one step: what is preserved is looked at + // once all of it has happened. + r.keep(st, s) +} + +// inScope reports whether a name was declared in the innermost block, which +// is what decides whether := declares it again. +func (r *run) inScope(st *state, name string) bool { + _, ok := st.scopes[len(st.scopes)-1][name] + return ok +} + +// set assigns to a variable, and then looks at what was promised to be +// preserved. +func (r *run) set(st *state, target ast.Expr, val *expr, at ast.Node) { + r.store(st, target, val) + r.keep(st, at) +} + +func (r *run) store(st *state, target ast.Expr, val *expr) { + id, ok := target.(*ast.Ident) + if !ok { + r.no(target, "only variables can be assigned to: an element of a slice or a field may have other names, and the proof would not see them change") + } + if id.Name == "_" { + return + } + b := st.vars[id.Name] + if b == nil || b.kind != kInt && b.kind != kBool { + r.no(target, "%s is not an integer or boolean variable", id.Name) + } + b.hist = append(b.hist, val) +} + +func (r *run) keep(st *state, at ast.Node) { + for _, k := range st.preserved { + f := r.formula(st, k.a, false) + r.check(st, f, k.a, nil, at, "Preserves is broken by the assignment at {ref}: "+k.a.text) + r.assume(st, f) + } +} + +func (r *run) branch(st *state, s *ast.IfStmt) outcomes { + st.push() + depth := len(st.scopes) + var out outcomes + starts := []*state{st} + if s.Init != nil { + starts = r.stmt(st, s.Init, nil).normal + } + for _, st := range starts { + cond := r.value(st, s.Cond) + yes, no := st, st.clone() + r.assume(yes, cond) + r.assume(no, not(cond)) + if r.feasible(yes) { + out.add(r.block(yes, s.Body)) + } + if r.feasible(no) { + switch e := s.Else.(type) { + case nil: + out.normal = append(out.normal, no) + case *ast.BlockStmt: + out.add(r.block(no, e)) + case *ast.IfStmt: + out.add(r.branch(no, e)) + } + } + } + for _, s := range out.normal { + s.popTo(depth - 1) + } + return out +} + +// ret settles a return: the results get their values, and every +// postcondition is an obligation. +func (r *run) ret(st *state, s *ast.ReturnStmt) { + if len(s.Results) > 0 { + var vals []*expr + if len(s.Results) == 1 && len(r.f.results) > 1 { + c, ok := s.Results[0].(*ast.CallExpr) + if !ok { + r.no(s, "only a call can give several values at once") + } + vals = r.eval(st, r.pure(c, func(name string) vkind { + if b := r.binding(st, name); b != nil { + return b.kind + } + return kNone + }), nil, c) + } else { + for _, x := range s.Results { + vals = append(vals, r.value(st, x)) + } + } + if len(vals) != len(r.f.results) { + r.no(s, "this return does not match the results") + } + // The results may be shadowed here; a return assigns the real ones. + for i, p := range r.f.results { + st.vars[p.name] = &binding{kind: p.kind, hist: []*expr{vals[i]}} + } + } + r.leave(st, s.Pos()) +} + +func (r *run) leave(st *state, at token.Pos) { + for _, a := range r.f.ensures { + if a.e != nil { + r.check(st, r.formula(st, a, true), a, nil, point(at), "Ensures is not proved for the return at {ref}: "+a.text) + } + } + for _, a := range r.f.exsures { + if a.e != nil { + r.check(st, not(r.formula(st, a, true)), a, nil, point(at), "Exsures is not ruled out for the return at {ref}: "+a.text) + } + } +} + +// loop is a for statement, or a range statement read as one. +type loop struct { + stmt ast.Stmt + init ast.Stmt + cond ast.Expr + post ast.Stmt + body *ast.BlockStmt + over *ast.RangeStmt +} + +// loop cuts a loop at its invariant. The invariant is proved on entry; +// then everything the loop assigns is forgotten, the invariant is assumed, +// and one iteration has to give it back, with a smaller variant. What +// follows the loop knows the invariant and that the condition is false. +func (r *run) loop(st *state, l *loop, pending []*annot) outcomes { + var invs, vars []*annot + notes := append(append([]*annot{}, pending...), r.trailing(l.body)...) + for _, a := range notes { + a.used = true + if a.e == nil { + continue + } + if a.kind == "Invariant" { + invs = append(invs, a) + } else { + vars = append(vars, a) + } + } + if len(vars) == 0 && l.over == nil { + r.v.report(r.v.position(l.stmt.Pos()), r.v.position(l.stmt.Pos()).Col+3, true, "this loop has no Variant, so nothing shows that it terminates") + } + st.push() + depth := len(st.scopes) + var out outcomes + starts := []*state{st} + var index string // the counter of a range loop + var bound *expr + if l.over != nil { + index, bound = r.rangeStart(st, l.over) + } else if l.init != nil { + starts = r.stmt(st, l.init, nil).normal + } + for _, s0 := range starts { + for _, a := range invs { + r.check(s0, r.formula(s0, a, false), a, nil, l.stmt, "Invariant does not hold when the loop at {ref} is entered: "+a.text) + } + head := s0.clone() + for _, name := range r.assigned(l) { + if b := head.vars[name]; b != nil && (b.kind == kInt || b.kind == kBool) { + b.hist = []*expr{r.fresh(name, b.kind)} + } + } + if index != "" { + b := head.vars[index] + b.hist = []*expr{r.fresh(index, kInt)} + r.assume(head, mk(opAnd, mk(opLe, num(0), b.hist[0]), mk(opLe, b.hist[0], bound))) + } + for _, a := range invs { + r.assume(head, r.formula(head, a, false)) + } + for _, k := range head.preserved { + r.assume(head, r.formula(head, k.a, false)) + } + var cond *expr + switch { + case index != "": + cond = mk(opLt, head.vars[index].hist[0], bound) + case l.cond != nil: + cond = r.value(head, l.cond) + default: + cond = eTrue + } + exit := head.clone() + r.assume(exit, not(cond)) + if r.feasible(exit) { + out.normal = append(out.normal, exit) + } + body := head + r.assume(body, cond) + if !r.feasible(body) { + continue + } + var before []*expr + for _, a := range vars { + m := r.formula(body, a, false) + before = append(before, m) + r.check(body, mk(opLe, num(0), m), a, nil, nil, "Variant can be negative while the loop still runs: "+a.text) + } + if l.over != nil && l.over.Value != nil { + if id, ok := l.over.Value.(*ast.Ident); ok && id.Name != "_" { + body.push() + arr := r.value(body, l.over.X) + body.declare(id.Name, &binding{kind: kInt, hist: []*expr{mk(opSel, arr, body.vars[index].hist[0])}}) + } + } + o := r.block(body, l.body) + out.normal = append(out.normal, o.brk...) + for _, s := range append(o.normal, o.cont...) { + s.popTo(depth) + ends := []*state{s} + if index != "" { + b := s.vars[index] + b.hist = append(b.hist, mk(opAdd, b.hist[len(b.hist)-1], num(1))) + } else if l.post != nil { + ends = r.stmt(s, l.post, nil).normal + } + for _, s := range ends { + for _, a := range invs { + r.check(s, r.formula(s, a, false), a, nil, l.stmt, "Invariant is not maintained by the body of the loop at {ref}: "+a.text) + } + for i, a := range vars { + r.check(s, mk(opLt, r.formula(s, a, false), before[i]), a, nil, l.stmt, "Variant is not shown to decrease in the loop at {ref}: "+a.text) + } + } + } + } + for _, s := range out.normal { + s.popTo(depth - 1) + } + return out +} + +// trailing finds the Invariant and Variant lines written inside a loop's +// body rather than above the loop: the ones no inner loop follows. +func (r *run) trailing(body *ast.BlockStmt) []*annot { + var out []*annot + for _, a := range r.blocks[body] { + if a.kind != "Invariant" && a.kind != "Variant" { + continue + } + var next ast.Stmt + for _, s := range body.List { + if s.Pos() > a.pos { + next = s + break + } + } + if !isLoop(next) { + out = append(out, a) + } + } + return out +} + +// rangeStart declares the counter of a range loop, under the name the +// program gives it if it gives it one, and finds where the counting stops. +func (r *run) rangeStart(st *state, s *ast.RangeStmt) (string, *expr) { + if s.Tok == token.ASSIGN { + r.no(s, "a range loop has to declare its variables with :=") + } + index := "range counter" + if id, ok := s.Key.(*ast.Ident); ok && id.Name != "_" { + index = id.Name + } + for _, name := range r.assigned(&loop{body: s.Body}) { + if name == index { + r.no(s, "the body assigns to the loop's own counter") + } + } + var bound *expr + if id, ok := s.X.(*ast.Ident); ok { + if b := r.binding(st, id.Name); b != nil && b.kind == kSlice { + bound = mk(opLen, b.hist[0]) + } + } + if sel, ok := s.X.(*ast.SelectorExpr); ok && bound == nil { + if x, ok := sel.X.(*ast.Ident); ok { + if b := r.binding(st, x.Name+"."+sel.Sel.Name); b != nil && b.kind == kSlice { + bound = mk(opLen, b.hist[0]) + } + } + } + if bound == nil { + if s.Value != nil { + r.no(s, "only a slice of integers or an integer can be ranged over") + } + bound = r.value(st, s.X) + if isBool(bound) || bound.sort == sArr { + r.no(s, "only a slice of integers or an integer can be ranged over") + } + } + st.declare(index, &binding{kind: kInt, hist: []*expr{num(0)}}) + return index, bound +} + +// assigned lists the variables a loop can change. +func (r *run) assigned(l *loop) []string { + seen := map[string]bool{} + visit := func(n ast.Node) bool { + switch s := n.(type) { + case *ast.AssignStmt: + for _, x := range s.Lhs { + if id, ok := x.(*ast.Ident); ok { + seen[id.Name] = true + } + } + case *ast.IncDecStmt: + if id, ok := s.X.(*ast.Ident); ok { + seen[id.Name] = true + } + case *ast.RangeStmt: + for _, x := range []ast.Expr{s.Key, s.Value} { + if id, ok := x.(*ast.Ident); ok { + seen[id.Name] = true + } + } + } + return true + } + ast.Inspect(l.body, visit) + if l.post != nil { + ast.Inspect(l.post, visit) + } + out := make([]string, 0, len(seen)) + for n := range seen { + out = append(out, n) + } + sort.Strings(out) + return out +} diff --git a/litgo.lit.md b/litgo.lit.md index cb031ed..e39b541 100644 --- a/litgo.lit.md +++ b/litgo.lit.md @@ -1,4 +1,4 @@ - + # litgo: literate Go @@ -10,19 +10,26 @@ written by hand is the README. In a literate program the code is written for a human reader first. You put the code in whatever order explains it best, in pieces small enough to talk about one at a time, and a tool puts the pieces back in the order the compiler wants. -litgo does three things with a document like that: +litgo does four things with a document like that: * **tangle** it into source files, * **weave** it into a PDF or a web page for reading, with typeset math and drawn diagrams, -* **run** it, and translate every compiler error, vet warning, test failure and - panic back to a line and column in the Markdown, so you never have to leave - the document to see what went wrong. +* **prove** it, where the code says what it promises. A literate program is an + argument that the code is right, written for a person. The `//@` annotations + of [VeGo](https://arxiv.org/abs/2608.22630) are the same argument written + for a machine: a contract on a function, an invariant on a loop. litgo checks + them, so the part of the prose that says "this loop keeps `lo*lo <= n`" stops + being something the reader has to take on trust, +* **run** it, and translate every compiler error, vet warning, failed proof, + test failure and panic back to a line and column in the Markdown, so you + never have to leave the document to see what went wrong. ```mermaid flowchart LR doc[litgo.lit.md] -->|tangle| src[Go and Lua source] doc -->|weave| page[PDF or web page] + doc -->|prove| qed[contracts and invariants, checked] src -->|go build| bin[bin/litgo] bin -->|run| doc ``` @@ -33,11 +40,20 @@ but Go, and after that litgo rebuilds itself: ```sh go build -o bin/litgo . # stage 1, from the committed sources -bin/litgo run litgo.lit.md # tangle, vet, test, rebuild bin/litgo +bin/litgo run litgo.lit.md # tangle, prove, vet, test, rebuild bin/litgo ``` +litgo takes its own medicine in a small way. The arithmetic that translates a +column between the Markdown and the tangled file is what every editor feature +in this document depends on, and it is proved, by the verifier in this +document, every time the line above is run. [Two halves of one +map](#two-halves-of-one-map) says what is proved and, just as carefully, what +is only tested. + `litgo run` does what the `run:` directive on the first line of this document -says. If you tangle this document with the binary it produces, you get back +says. (It names the packages rather than saying `./...`, because running two of +the examples leaves two `main` functions in `examples/`, which is fine for +them and not for `go vet`.) If you tangle this document with the binary it produces, you get back exactly the sources that binary was built from. That fixed point is how we know the bootstrap works. @@ -124,6 +140,15 @@ These are the directives: litgo never reformats the code it tangles. Each output line comes from one source line, so any position in the output can be traced back exactly. +Specifications need no directive at all, because they are part of the code: a +comment that starts with `//@` is a VeGo annotation, and goes wherever the Go +it describes goes, through any number of chunks. A function's contract can sit +in the block that introduces the function while the invariant sits three +sections later, in the chunk with the loop, next to the paragraph that explains +it. The verifier sees them joined up, the way the compiler sees the code. +[Proofs](#proofs) is the chapter about them, and +[A tutorial in proofs](#a-tutorial-in-proofs) the tour. + # The document model @@ -998,7 +1023,7 @@ func (o *Output) Map(line, col int) (srcLine, srcCol int, via []int, ok bool) { best := segs[0] for _, s := range segs { if col >= s.OutCol && col < s.OutCol+s.Len { - return s.SrcLine, s.SrcCol + col - s.OutCol, s.Via, true + return s.SrcLine, s.src(col), s.Via, true } if s.OutCol <= col { best = s @@ -1065,7 +1090,7 @@ func (r *Result) Locate(line, col int) (o *Output, outLine, outCol int, ok bool) for n, l := range o.Lines { for _, s := range l.Segs { if s.SrcLine == line && col >= s.SrcCol && col <= s.SrcCol+s.Len { - return o, n, s.OutCol + col - s.SrcCol, true + return o, n, s.out(col), true } } } @@ -1095,7 +1120,7 @@ func (o *Output) Exact(line, col int, end bool) (srcLine, srcCol int, ok bool) { lo, hi = lo+1, hi+1 } if col >= lo && col <= hi { - return s.SrcLine, s.SrcCol + col - s.OutCol, true + return s.SrcLine, s.src(col), true } } } @@ -1146,9 +1171,61 @@ last one, which is where the cursor sits while a word is being typed. `Locate` accepts it so that completion works at the end of a line. `Map` has no use for gaps, because a compiler's column always points at a character, so it sends a column past the end back to the last character. The function that does invert -`Locate` there is `Exact`, asked for the end of a range. The tests in -[Map and Locate undo each other](#map-and-locate-undo-each-other) check all of -this, character by character. +`Locate` there is `Exact`, asked for the end of a range. + +### What is proved, and what is tested + +Each function does two things: it *finds* a segment, and then it *translates* +a column through it. The translation is arithmetic, it is where an off-by-one +would live, and it is the same in all three functions. So it is written once, +as two methods, and `Map`, `Locate` and `Exact` call them. Their contracts are +the invariant of this section, stated about one segment, and +[`litgo prove`](#proofs) proves them: + +```go +// out is where the byte at column col of the source line went in the output. +// +// @ Requires s.SrcCol <= col ^ col <= s.SrcCol + s.Len +func (s Seg) out(col int) (c int) { return s.OutCol + col - s.SrcCol } + +//@ Ensures s.OutCol <= c ^ c <= s.OutCol + s.Len +//@ Ensures s.src(c) = col + +// src is where the byte at column col of the output line came from. +// +// @ Requires s.OutCol <= col ^ col <= s.OutCol + s.Len +func (s Seg) src(col int) (c int) { return s.SrcCol + col - s.OutCol } + +//@ Ensures s.SrcCol <= c ^ c <= s.SrcCol + s.Len +//@ Ensures s.out(c) = col +``` + +(The `// @` with a space is what gofmt makes of `//@` inside a doc comment. +The verifier reads both, and this is the spelling that survives formatting on +save.) + +Read the second `Ensures` of each: `src(out(col)) = col` and +`out(src(col)) = col`, for every segment there could ever be and every column +in it or at its end, and the first `Ensures` says that the result is a column +the other function accepts. That is $\mathrm{Map} \circ \mathrm{Locate}$ and +$\mathrm{Locate} \circ \mathrm{Map}$ being the identity *within a segment*, +as a theorem rather than as a test that passed. + +It is not the whole invariant, and it would be wrong to say it was. That +`Map(Locate(p)) = p` also needs the search in `Map` to arrive at the segment +the search in `Locate` left from, and that is true because of how tangling +lays segments out: on one output line they do not overlap, and two segments +from one source line are either the same text written twice or have a chunk +reference between them. Those are facts about slices of structs that hold +slices, built by a recursive splice. The verifier's fragment is integers and +slices of integers, so it declines, and says so, rather than prove something +about a simplified model that is not the program. And the unqualified +statement "`Map` and `Locate` are inverses" is simply not true, as the table +above says: `Locate(Map(q)) = q` fails for the second copy of a chunk, by +design. So the division of labour is this. The arithmetic is proved, for all +inputs. The search, and the layout it relies on, are tested, in +[Map and Locate undo each other](#map-and-locate-undo-each-other), character +by character, on documents built to have every awkward case. # Editing structure @@ -1800,7 +1877,7 @@ chunks are, and what is wrong. ```go // Item is something an editor can render in place of, or next to, source text. type Item struct { - Kind string `json:"kind"` // math, math-block, mermaid, table, chunk-def, chunk-ref, file + Kind string `json:"kind"` // math, math-block, mermaid, table, chunk-def, chunk-ref, file, vego, vego-math Line int `json:"line"` Col int `json:"col"` EndLine int `json:"end_line"` @@ -1881,6 +1958,8 @@ func (d *Doc) Analyze(o AnalyzeOptions) *Analysis { a.Items = append(a.Items, Item{Kind: "math-block", Line: b.Open, EndLine: min(b.Close, len(d.Lines)-1), Lines: latex.Render(src, latex.Options{Display: true, Upright: o.Upright})}) } + case "go": + a.Items = append(a.Items, d.proofItems(b, o)...) } } a.Items = append(a.Items, d.mathItems(inBlock, o)...) @@ -1949,6 +2028,241 @@ func plural(n int) string { } ``` +## Finding the annotations + +A `//@` comment is a specification ([Proofs](#proofs) has the whole story), and +to a syntax highlighter it is a comment like any other, grey and easy to skip. +It is the most carefully chosen line in the block, so the editor gets told +where the annotations are: the item runs from the `//@` to the end of the line, +and `Name` is the keyword, which an editor can set apart from the formula. + +```go +// ProofKeywords are the words an annotation can start with. One that starts +// with none of them is an Assert. +var ProofKeywords = map[string]bool{"Requires": true, "Ensures": true, "Exsures": true, "Invariant": true, + "Variant": true, "Measure": true, "BaseCase": true, "InductionHypothesis": true, "Assert": true, + "Axiom": true, "Preserves": true, "Predicate": true, "Immutable": true, "Property": true} + +// ProofComment reports whether a line comment is an annotation, and returns +// what follows the marker. The marker is //@. gofmt rewrites that to // @ in +// a doc comment, so that is accepted too, but only in front of a keyword or +// where the annotation above is not finished (continuing), because a comment +// may well start with an @ for reasons of its own. +func ProofComment(text string, continuing bool) (body string, ok bool) { + if strings.HasPrefix(text, "//@") { + return text[3:], true + } + if !strings.HasPrefix(text, "// @") { + return "", false + } + body = text[4:] + word, _, _ := strings.Cut(strings.TrimLeft(body, " \t"), " ") + return body, continuing || ProofKeywords[word] +} + +// proofItems finds the VeGo annotations of a Go block: one item for the +// annotation, and one for its formula, typeset. +func (d *Doc) proofItems(b *Block, o AnalyzeOptions) []Item { + var out []Item + continuing := false + for i := b.First(); i <= b.Last() && i < len(d.Lines); i++ { + line := d.Lines[i] + text := strings.TrimLeft(line, " \t") + body, ok := ProofComment(text, continuing) + if !ok { + continuing = false + continue + } + word, _, _ := strings.Cut(strings.TrimLeft(body, " \t"), " ") + if continuing || !ProofKeywords[word] { + word = "" // a formula by itself is an Assert without the word + } + trimmed := strings.TrimRight(body, " \t") + continuing = strings.HasSuffix(trimmed, "||") || strings.HasSuffix(trimmed, " v") || strings.HasSuffix(trimmed, " in") || + trimmed != "" && strings.ContainsAny(trimmed[len(trimmed)-1:], "^:=<>+-*/%(,[~!&") + out = append(out, Item{Kind: "vego", Line: i, EndLine: i, Col: len(line) - len(text), EndCol: len(line), Name: word}) + formula := strings.TrimLeft(body, " \t") + if word != "" { + formula = strings.TrimLeft(formula[len(word):], " \t") + } + if math := proofMath(formula, o); math != "" { + out = append(out, Item{Kind: "vego-math", Line: i, EndLine: i, Col: len(line) - len(formula), + EndCol: len(strings.TrimRight(line, " \t")), Text: math}) + } + } + return out +} +``` + +A formula is ASCII so that it can be typed, and mathematics so that it can be +read, and those pull in different directions: `lo*lo <= n ^ n < hi*hi` is +what the keyboard has, and $lo^2 \leq n \land n < hi^2$ is what +it means. The prose of a document already gets this treatment, where `$...$` +is drawn as Unicode and turns back into its source under the cursor. So a +formula is translated into the LaTeX it would have been in the prose, and +handed to the same renderer. It comes out looking like the math around it, +italic letters and all, and nothing here knows what a `≀` is. + +The translation is a scan for words and operators. It does not parse, because +it has to be right about a formula that is half typed. The one thing that +needs care is `v`, which is *or* between two operands and a variable +anywhere else. If the renderer leaves a backslash in its answer there was +something it did not know, and the source is better than a guess. + +```go +// proofOps are the operators of a formula, longest first, and their LaTeX. +var proofOps = []struct{ op, tex string }{ + {"<->", `\leftrightarrow `}, {"::=", `\coloneqq `}, {"...", `\infty `}, {"->", `\to `}, {"<-", `\leftarrow `}, + {"<=", `\leq `}, {">=", `\geq `}, {"<>", `\neq `}, {"!=", `\neq `}, {"==", `= `}, {"&&", `\land `}, {"||", `\lor `}, + {"^", `\land `}, {"~", `\lnot `}, {"!", `\lnot `}, {"*", `\cdot `}, {"%", `\bmod `}, +} + +var proofWords = map[string]string{"Forall": `\forall `, "Exists": `\exists `, "Unique": `\exists! `, "in": `\in `, + "true": `\text{true}`, "false": `\text{false}`, "len": `\text{len}`} + +// proofPower counts the factors of a product of one name with itself, +// x*x*x, where the first x ends at j, and says where the product ends. A +// name that is called or indexed is not the same factor. +func proofPower(formula string, j int, word string) (n, end int) { + n, end = 1, j + for { + k := end + for k < len(formula) && formula[k] == ' ' { + k++ + } + if k >= len(formula) || formula[k] != '*' { + return n, end + } + for k++; k < len(formula) && formula[k] == ' '; k++ { + } + if !strings.HasPrefix(formula[k:], word) { + return n, end + } + k += len(word) + if k < len(formula) { + if c := formula[k]; c == '_' || c == '.' || c == '\'' || c == '(' || c == '[' || + c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' { + return n, end + } + } + n, end = n+1, k + } +} + +// proofClose finds the ) that closes the ( at i, or returns 0. +func proofClose(formula string, i int) int { + depth := 0 + for k := i; k < len(formula); k++ { + switch formula[k] { + case '(': + depth++ + case ')': + if depth--; depth == 0 { + return k + } + } + } + return 0 +} + +// proofMath typesets a formula the way the math in the prose is typeset. +func proofMath(formula string, o AnalyzeOptions) string { + var tex strings.Builder + operand := false // was the last thing something an operator can follow + quantifiers := 0 // that still wait for the colon before their body + powers := map[int][2]int{} // by the ) of a repeated factor: how often, and where the product ends + last := "" + for i := 0; i < len(formula); { + c := formula[i] + switch { + case c == ' ' || c == '\t': + i++ + case c == '_' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z': + j := i + for j < len(formula) && (formula[j] == '_' || formula[j] == '.' && j+1 < len(formula) && formula[j+1] != '.' || + formula[j] >= 'a' && formula[j] <= 'z' || formula[j] >= 'A' && formula[j] <= 'Z' || formula[j] >= '0' && formula[j] <= '9') { + j++ + } + word := formula[i:j] + for j < len(formula) && formula[j] == '\'' { + j++ + } + primes := formula[i+len(word) : j] + switch { + case word == "v" && primes == "" && operand: + tex.WriteString(`\lor `) + operand = false + case word == "Z" && last == "in": + tex.WriteString(`\mathbb{Z}`) + operand = true + case proofWords[word] != "" && primes == "": + if word == "Forall" || word == "Exists" || word == "Unique" { + quantifiers++ + } + tex.WriteString(proofWords[word]) + operand = word == "true" || word == "false" + default: + tex.WriteString(strings.ReplaceAll(word, "_", `\_`) + primes) + // x*x is a square, and is read more easily as one. After a / or a % + // it is not: a/x*x is (a/x)*x. + if n, end := proofPower(formula, j, word); n > 1 && primes == "" && last != "/" && last != "%" { + fmt.Fprintf(&tex, "^{%d}", n) + j = end + } + tex.WriteString(" ") + operand = true + } + last = word + i = j + default: + found := false + for _, op := range proofOps { + if strings.HasPrefix(formula[i:], op.op) { + tex.WriteString(op.tex) + i += len(op.op) + found, operand, last = true, op.op == "...", op.op + break + } + } + if !found && c == ':' && quantifiers > 0 { + // The colon of a quantifier gets room; the one in A[a:b) does not. + quantifiers-- + tex.WriteString(`\;:\;`) + found, operand, last = true, false, ":" + i++ + } + if c == '(' && !operand && last != "/" && last != "%" { + // (r+1)*(r+1) is a square too. Remember it until its first ) comes by. + if close := proofClose(formula, i); close > 0 { + if n, end := proofPower(formula, close+1, formula[i:close+1]); n > 1 { + powers[close] = [2]int{n, end} + } + } + } + if pw, ok := powers[i]; ok && c == ')' { + fmt.Fprintf(&tex, ")^{%d}", pw[0]) + found, operand, last = true, true, ")" + i = pw[1] + } + if !found { + tex.WriteByte(c) + operand = c == ')' || c == ']' || c == '|' || c >= '0' && c <= '9' + last = string(c) + i++ + } + } + } + if tex.Len() == 0 { + return "" + } + out := latex.Render(tex.String(), latex.Options{Upright: o.Upright}) + if len(out) != 1 || strings.Contains(out[0], `\`) { + return "" + } + return out[0] +} +``` + ## Finding the math Inline math follows pandoc's rules. The opening `$` has to touch the formula on @@ -2718,6 +3032,42 @@ func TestAnalyzeExample(t *testing.T) { } } +func TestProofMath(t *testing.T) { + for formula, want := range map[string]string{ + "lo*lo <= n ^ n < hi*hi": "π‘™π‘œΒ² ≀ 𝑛 ∧ 𝑛 < β„Žπ‘–Β²", + "x * x*x >= 0 ^ y*x*x = a/x*x": "π‘₯Β³ β‰₯ 0 ∧ 𝑦 Β· π‘₯Β² = π‘Ž/π‘₯ Β· π‘₯", + "n < (r+1)*(r+1) ^ (i+1)*i = f(x)*(x)": "𝑛 < (π‘Ÿ + 1)Β² ∧ (𝑖 + 1) Β· 𝑖 = 𝑓(π‘₯) Β· (π‘₯)", + "Forall k in [0, n) : (k)*(k) >= 0": "βˆ€π‘˜ ∈ [0, 𝑛) : (π‘˜)Β² β‰₯ 0", + "f*f(x) + A*A[i] + x*xs + x'*x'": "𝑓 Β· 𝑓(π‘₯) + 𝐴 Β· 𝐴[𝑖] + π‘₯ Β· π‘₯𝑠 + π‘₯β€² Β· π‘₯β€²", + "at = -1 v 0 <= at < |A|": "π‘Žπ‘‘ = βˆ’1 ∨ 0 ≀ π‘Žπ‘‘ < |𝐴|", + "v >= 0 v v' <> v": "𝑣 β‰₯ 0 ∨ 𝑣′ β‰  𝑣", + "~found -> x <> A[:]": "Β¬π‘“π‘œπ‘’π‘›π‘‘ β†’ π‘₯ β‰  𝐴[:]", + "Forall k in [0, |A|) : A[k] <> x": "βˆ€π‘˜ ∈ [0, |𝐴|) : 𝐴[π‘˜] β‰  π‘₯", + "Exists k in Z : a <-> b && c || !d": "βˆƒπ‘˜ ∈ β„€ : π‘Ž ↔ 𝑏 ∧ 𝑐 ∨ ¬𝑑", + "Sorted(A) ::= A[0:n) <= m": "π‘†π‘œπ‘Ÿπ‘‘π‘’π‘‘(𝐴) ≔ 𝐴[0:𝑛) ≀ π‘š", + "q * d + A[i] % d = A[i] ^ ret_0 == 1": "π‘ž Β· 𝑑 + 𝐴[𝑖] mod 𝑑 = 𝐴[𝑖] ∧ π‘Ÿπ‘’π‘‘_0 = 1", + "s.OutCol <= c ^ c <= s.OutCol + s.Len": "𝑠.π‘‚π‘’π‘‘πΆπ‘œπ‘™ ≀ 𝑐 ∧ 𝑐 ≀ 𝑠.π‘‚π‘’π‘‘πΆπ‘œπ‘™ + 𝑠.𝐿𝑒𝑛", + "Forall k in [0, ...) : k >= 0": "βˆ€π‘˜ ∈ [0, ∞) : π‘˜ β‰₯ 0", + "": "", + } { + if got := proofMath(formula, AnalyzeOptions{}); got != want { + t.Errorf("proofMath(%q) = %q, want %q", formula, got, want) + } + } + // The item covers the formula and nothing else, so that the keyword stays. + d := Parse("t.lit.md", []byte("```go\n\t//@ Invariant lo <= hi \n\t//@ a ^ b\n\t// @ Ensures b\n\t//@ c\n```\n")) + var got []string + for _, it := range d.Analyze(AnalyzeOptions{}).Items { + if it.Kind == "vego-math" { + got = append(got, fmt.Sprintf("%d:%d-%d %s", it.Line, it.Col, it.EndCol, it.Text)) + } + } + want := []string{"1:15-23 π‘™π‘œ ≀ β„Žπ‘–", "2:5-10 π‘Ž ∧ 𝑏", "3:14-15 𝑏", "4:7-8 𝑐"} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Errorf("items = %q, want %q", got, want) + } +} + func TestTables(t *testing.T) { src := "| Key | Does | Cost |\n|---|:-:|--:|\n| `\\r` | **run** it, see [docs](x) | $x^2$ |\n| a \\| b | short |\n\nnot | a table\n\n```go\n// | a | b |\n// |---|---|\n```\n" d := Parse("t.lit.md", []byte(src)) @@ -6329,4945 +6679,10785 @@ func TestWeaveForHTML(t *testing.T) { } ``` -# The command +# Proofs - - - +A test says that a program did the right thing the times somebody looked. A +proof says that it does the right thing every time. For a long while only the +first was practical, and the reason was never the checking, which a machine +does. It was writing down *what* to check: the precondition, the invariant of +every loop, the quantity that shrinks. That is the part a coding agent is good +at proposing and a person is good at reading, and a literate program is the +natural place for it, because the argument for why a loop is right is exactly +what the prose around the loop was trying to say anyway. -Half of the commands are for people. The other half are the editor protocol, -where the document arrives on standard input and JSON goes out. The document -comes in on standard input because the buffer being edited is rarely the same -as the file on disk. +[VeGo](https://arxiv.org/abs/2608.22630) (Verified Go, by Tina Massoudi and +Chris Dutchyn) is a notation for that argument. The specifications are +comments that start with `//@`, so an annotated program is still plain Go: it +compiles, runs and formats the way it did before. litgo reads those comments +and checks them. + ```go -const usage = `litgo β€” literate Go - - litgo tangle [--force] [--stdout] FILE.lit.md... write the Go source - litgo run [--vet] [--timeout 30s] FILE [-- ARGS] tangle, compile and run - litgo weave [--to pdf|html|typ|md] [-o OUT] FILE write a PDF through Typst, or a web page - litgo check FILE.lit.md... chunk, syntax and type errors; writes nothing - litgo cat FILE.lit.md print with math and diagrams rendered - litgo fmt [-w] FILE.lit.md gofmt the code blocks - -Editor protocol (document on stdin, JSON on stdout): - litgo analyze [--path P] [--upright] - litgo untangle --start L --end L [--start-col C --end-col C] [--name N] - litgo rename --from NAME --to NAME - litgo run --json FILE one JSON event per line - litgo lsp [--gopls PATH|off] language server on stdin/stdout: errors as you type, - and gopls for the Go in the code blocks - -Positions in the editor protocol are 0-based; columns are bytes. -` - -const VERSION = "0.1.6" - -func main() { - if len(os.Args) < 2 { - fmt.Fprint(os.Stderr, usage) - os.Exit(2) - } - cmd, args := os.Args[1], os.Args[2:] - switch cmd { - case "tangle": - os.Exit(cmdTangle(args)) - case "run": - os.Exit(cmdRun(args)) - case "weave": - os.Exit(cmdWeave(args)) - case "check": - os.Exit(cmdCheck(args)) - case "cat": - os.Exit(cmdCat(args)) - case "fmt": - os.Exit(cmdFmt(args)) - case "analyze": - os.Exit(cmdAnalyze(args)) - case "untangle": - os.Exit(cmdUntangle(args)) - case "version": - os.Exit(cmdVersion(args)) - case "rename": - os.Exit(cmdRename(args)) - case "lsp": - fs := flag.NewFlagSet("lsp", flag.ExitOnError) - gopls := fs.String("gopls", "", "the gopls to put behind the server; found when empty, none when \"off\"") - fs.Parse(args) - if err := lsp.Serve(os.Stdin, os.Stdout, *gopls); err != nil { - fmt.Fprintln(os.Stderr, "litgo lsp:", err) - os.Exit(1) - } - case "help", "-h", "--help": - fmt.Print(usage) - default: - fmt.Fprintf(os.Stderr, "litgo: unknown command %q\n\n%s", cmd, usage) - os.Exit(2) +//@ Requires n >= 0 ^ d > 0 +func Divide(n, d int) (q, r int) { + r = n + //@ Invariant n = q*d + r ^ r >= 0 + //@ Variant r + for r >= d { + q, r = q+1, r-d } + return q, r } +//@ Ensures n = q*d + r ^ 0 <= r < d ``` +```mermaid +flowchart LR + doc[prog.lit.md] -->|tangle, in memory| go[annotated Go] + go -->|walk every path| vcs[proof obligations] + vcs -->|refute| prover[arithmetic prover] + prover -->|not proved, at a line of the Markdown| doc +``` + +The paper's own checker, `vegop`, has not been published, so litgo has one of +its own, in this chapter. It follows the paper's design in the ways that +matter: it runs natively in Go with no SMT solver behind it, it works through +a function one assignment at a time, giving every assignment a new name the +way SSA form does, and it settles its verification conditions with +Fourier-Motzkin elimination over linear integer arithmetic. + +It is a deductive verifier, and two properties are worth stating plainly. + +* **It is sound and incomplete, on purpose.** "Proved" means that the + hypotheses and the negated goal were shown to have no model. Everything the + prover does on the way (instantiating a quantifier at a few terms, treating + `x*y` as an unknown of its own, giving up when a budget runs out) can only + make it fail to find a proof, never find a wrong one. So the two answers are + *proved* and *not proved*, and "not proved" sometimes means "true, but say + more in the invariant". +* **It refuses what it does not understand.** The fragment is integers, + booleans, slices of integers that are read but not written, integer fields + of structs, `if`, `for`, `range`, `break`, `continue`, `return`, and calls + of other annotated functions. A function with annotations that steps outside + that fragment is reported as *not verified*, with the statement that was too + much. An unchecked contract is worse than none. + +What a proof covers: every `Ensures` on every path to every `return`; every +`Invariant`, on entry and around the loop; termination, where a `Variant` or a +`Measure` is given; every index within its slice and every divisor not zero; +the `Requires` of every call. What it does not cover: integers are the +mathematical ones, so overflow is out of sight, as it is in the paper; and a +`nil` pointer to a struct is not considered. + +A function without annotations is not looked at, so a program can be verified +one function at a time. + +## What there is to say + + + + + +| Annotation | Where | Meaning | +|:--|:--|:--| +| `Requires P` | above a function | The callers owe `P`; the body may assume it. | +| `Ensures Q` | under the closing brace | Holds at every `return`. A parameter means its value on entry, the results their final values. | +| `Exsures X` | under the closing brace | A state that no `return` can be in: `~X` is proved. | +| `Invariant I` | above a loop, or at the end of its body | Holds on entry and after every iteration, and is all that is known after the loop, with the negated condition. | +| `Variant e` | with the invariant | An integer that is never negative while the loop runs and gets smaller every time round. | +| `Measure e` | above a recursive function, or in it | The same, for recursion: smaller at every recursive call. | +| `BaseCase c` | in a body | No recursive call is reachable once `c` has held. | +| `InductionHypothesis P`, `Assert P`, or just `P` | in a body | Proved there, and known from there on. | +| `Preserves P` | in a body | Proved there and again after every assignment until the block ends. A loop inside the block may assume it. | +| `Axiom P` | in a body | Assumed without proof, with a warning every time. | +| `Predicate N(a, b) ::= P` | anywhere | A name for a formula. | + +The paper's `.vgo` dialect also has a `while` keyword, a `skip` statement and +functions declared inside functions. None of those is Go, and a litgo block +has to be Go, so they are left out; `for cond {}`, an empty statement and a +separate function say the same things. `Property` and contracts on interface +methods are read but not checked, and say so. + ```go -func load(path string) (*lit.Doc, error) { - abs, err := filepath.Abs(path) - if err != nil { - return nil, err - } - src, err := os.ReadFile(abs) - if err != nil { - return nil, err - } - return lit.Parse(abs, src), nil +// Marker starts a comment that is a VeGo annotation. +const Marker = "//@" + +// Source is a Go file to verify. +type Source struct { + Path string + Text []byte } -func stdinDoc(path string) *lit.Doc { - src, _ := io.ReadAll(os.Stdin) - if path == "" { - path = "stdin.lit.md" - } - return lit.Parse(path, src) +// Pos is a 0-based position in a Source. +type Pos struct { + Path string + Line, Col int } -func printJSON(v any) int { - enc := json.NewEncoder(os.Stdout) - enc.SetEscapeHTML(false) - if err := enc.Encode(v); err != nil { - fmt.Fprintln(os.Stderr, "litgo:", err) - return 1 - } - return 0 +// Finding is something the verifier has to say. Ref, when it has a Path, is a +// second position; the message mentions it as {ref}. +type Finding struct { + Pos + EndCol int + Warning bool + Message string + Ref Pos } -func jsonError(err error) int { - printJSON(map[string]string{"error": err.Error()}) - return 1 +// Summary counts what a verification did. +type Summary struct { + Functions int // functions with annotations + Proved int // of which every obligation was proved + Obligations int + Names []string // the proved functions } +``` -func report(diags []lit.Diag) (errors int) { - r := &reporter{seen: map[string]bool{}} - for _, d := range diags { - r.diag(d) - if d.Severity == lit.SevError { - errors++ - } - } - return errors +An annotation is one `//@` comment. A formula that is too long for a line goes +on in the next comment, under the rule Go uses for semicolons: a line that ends +in something that cannot end a formula (an operator, an opening bracket) is not +finished. + +```go +// annot is one annotation: a //@ comment, with the lines that continue it. +type annot struct { + kind string + text string // the formula + e *expr + pos token.Pos + where Pos + endCol int + textCol int // where the formula starts on the line + first int // how much of the formula is on that line + lines int // how many comment lines it takes + used bool +} + +// Annotated reports whether a Go file has annotations at all, which is what +// decides whether verifying it is worth the time. +func Annotated(text []byte) bool { + return strings.Contains(string(text), Marker) || strings.Contains(string(text), "// @ ") } ``` +`Prove` is the door the rest of litgo comes in by. It takes what a document +tangles to, in memory, and answers in the document's own lines, which is the +source map's job as always. A position mentioned inside a message (the +`return` a postcondition failed for, say) is translated too. A document's Go +files are verified a directory at a time, because a contract in one file is +used by calls in another. + ```go -func cmdTangle(args []string) int { - fs := flag.NewFlagSet("tangle", flag.ExitOnError) - force := fs.Bool("force", false, "overwrite files litgo did not generate") - stdout := fs.Bool("stdout", false, "print instead of writing") - fs.Parse(args) - if fs.NArg() == 0 { - fmt.Fprintln(os.Stderr, "litgo tangle: no input files") - return 2 +// Prove verifies the annotated Go a document tangles to, and reports at the +// document's own lines. Nothing is written, so it works on a buffer. +func Prove(res *lit.Result) ([]lit.Diag, Summary) { + var total Summary + if res.HasErrors() { + return nil, total } - status := 0 - for _, path := range fs.Args() { - doc, err := load(path) - if err != nil { - fmt.Fprintln(os.Stderr, "litgo:", err) - status = 1 + packages := map[string][]*lit.Output{} + var dirs []string + for _, o := range res.Files { + if o.File.Lang != "go" || strings.HasSuffix(o.Path, "_test.go") { continue } - res := doc.Tangle() - if report(res.Diags) > 0 { - status = 1 + dir := filepath.Dir(o.Path) + if packages[dir] == nil { + dirs = append(dirs, dir) + } + packages[dir] = append(packages[dir], o) + } + var diags []lit.Diag + seen := map[string]bool{} + for _, dir := range dirs { + var sources []Source + outputs := map[string]*lit.Output{} + annotated := false + for _, o := range packages[dir] { + text := o.Bytes() + annotated = annotated || Annotated(text) + sources = append(sources, Source{o.Path, text}) + outputs[o.Path] = o + } + if !annotated { continue } - for _, o := range res.Files { - if *stdout { - if len(res.Files) > 1 { - fmt.Printf("==> %s <==\n", relPath(o.Path)) - } - os.Stdout.Write(o.Bytes()) + findings, sum := Verify(sources) + total.Functions += sum.Functions + total.Proved += sum.Proved + total.Obligations += sum.Obligations + total.Names = append(total.Names, sum.Names...) + for _, f := range findings { + o := outputs[f.Path] + line, col, _, ok := o.Map(f.Line, f.Col) + if !ok { continue } - if err := writeTangled(o, *force); err != nil { - fmt.Fprintln(os.Stderr, "litgo:", err) - status = 1 - continue + d := lit.Diag{File: res.Doc.Path, Line: line, Col: col, Severity: lit.SevError, Message: f.Message, Source: "vego"} + if f.Warning { + d.Severity = lit.SevWarning + } + if f.EndCol > f.Col { + d.EndCol = col + f.EndCol - f.Col + } + if f.Ref.Path != "" { + if l, _, _, ok := outputs[f.Ref.Path].Map(f.Ref.Line, f.Ref.Col); ok { + d.Message = strings.ReplaceAll(d.Message, "{ref}", fmt.Sprintf("line %d", l+1)) + } + } + // A chunk that is used twice is verified twice, and fails twice. + key := fmt.Sprintf("%d:%d:%s", d.Line, d.Col, d.Message) + if !seen[key] { + seen[key] = true + diags = append(diags, d) } - fmt.Fprintf(os.Stderr, "litgo: %s β†’ %s (%d lines)\n", relPath(doc.Path), relPath(o.Path), len(o.Lines)) } } - return status + return diags, total } +``` -func cmdRun(args []string) int { - fs := flag.NewFlagSet("run", flag.ExitOnError) - o := runOptions{} - fs.BoolVar(&o.json, "json", false, "emit one JSON event per line") - fs.BoolVar(&o.force, "force", false, "overwrite files litgo did not generate") - fs.BoolVar(&o.vet, "vet", false, "also run go vet") - fs.DurationVar(&o.timeout, "timeout", 0, "kill the program after this long") - fs.Parse(args) - if fs.NArg() == 0 { - fmt.Fprintln(os.Stderr, "litgo run: no input file") - return 2 +`Verify` is the same thing without the Markdown, for one package of Go. A +function counts as proved when verifying it reported no error. Warnings (an +`Axiom`, a loop without a `Variant`) do not take a proof away, they qualify +it. + +```go +// Verify checks every annotated function of one package. +func Verify(sources []Source) ([]Finding, Summary) { + v := &verifier{fset: token.NewFileSet(), funcs: map[string]*fn{}, structs: map[string]map[string]string{}, + consts: map[string]int64{}, preds: map[string]*predicate{}, reported: map[string]bool{}} + for _, s := range sources { + f, err := parser.ParseFile(v.fset, s.Path, s.Text, parser.ParseComments|parser.SkipObjectResolution) + if err != nil { + continue // the syntax check has more to say about it than we do + } + v.files = append(v.files, f) } - o.args = fs.Args()[1:] - if len(o.args) > 0 && o.args[0] == "--" { - o.args = o.args[1:] - } - return runFile(fs.Arg(0), o) -} - -func cmdWeave(args []string) int { - fs := flag.NewFlagSet("weave", flag.ExitOnError) - to := fs.String("to", "", "pdf, html, typ (Typst source) or md (woven Markdown); default pdf, or what -o ends in") - out := fs.String("o", "", "output file") - fs.Parse(args) - if fs.NArg() != 1 { - fmt.Fprintln(os.Stderr, "litgo weave: expected one input file") - return 2 + for _, f := range v.files { + v.declarations(f) } - doc, err := load(fs.Arg(0)) - if err != nil { - fmt.Fprintln(os.Stderr, "litgo:", err) - return 1 + for _, f := range v.files { + v.functions(f) } - format := *to - if format == "" { - format = strings.TrimPrefix(filepath.Ext(*out), ".") + for _, f := range v.files { + v.annotations(f) } - if format == "" { - format = "pdf" + var sum Summary + for _, f := range v.ordered { + if !f.annotated { + continue + } + sum.Functions++ + before := v.errors + v.verify(f) + sum.Obligations += f.obligations + if v.errors == before && !f.broken { + sum.Proved++ + sum.Names = append(sum.Names, f.key) + } } - if *out == "" { - ext := map[string]string{"md": "woven.md"}[format] - if ext == "" { - ext = format + for _, a := range v.all { + if !a.used && a.e != nil { + v.report(a.where, a.endCol, false, a.kind+" is not attached to anything: Requires goes above a function, Ensures below it, Invariant and Variant at a loop") } - *out = strings.TrimSuffix(strings.TrimSuffix(doc.Path, ".md"), ".lit") + "." + ext } - switch format { - case "pdf": - err = weave.PDF(doc, *out) - case "html": - err = weave.HTML(doc, *out) - case "typ": - var src string - if src, err = weave.Typst(doc); err == nil { - err = os.WriteFile(*out, []byte(src), 0o644) + sort.SliceStable(v.findings, func(i, j int) bool { + a, b := v.findings[i], v.findings[j] + if a.Path != b.Path { + return a.Path < b.Path } - case "md": - err = os.WriteFile(*out, []byte(weave.Markdown(doc)), 0o644) - default: - err = fmt.Errorf("cannot weave to %q: pdf, html, typ and md are what there is", format) + return a.Line < b.Line + }) + return v.findings, sum +} +``` + +The verifier is syntactic. It never runs the type checker, because the types +it can reason about are few enough to read off the declarations: the integer +types, `bool`, `[]int`, and structs of the package, of which it uses the +integer and boolean fields. + +```go +type vkind int + +const ( + kNone vkind = iota + kInt + kBool + kSlice + kStruct +) + +type param struct { + name string + kind vkind + typ string // the struct's name, for kStruct +} + +// fn is a function of the package, with its contract. +type fn struct { + key string // name, or Type.name for a method + decl *ast.FuncDecl + recv *param + params []param + results []param + requires []*annot + ensures []*annot + exsures []*annot + measure []*annot + inside []*annot // the annotations in the body + annotated bool + broken bool // an annotation of it does not parse + obligations int +} + +type verifier struct { + fset *token.FileSet + files []*ast.File + funcs map[string]*fn + ordered []*fn + structs map[string]map[string]string + consts map[string]int64 + preds map[string]*predicate + all []*annot + findings []Finding + reported map[string]bool + errors int + fresh int + parseFresh int +} +``` + +```go +func (v *verifier) position(p token.Pos) Pos { + pp := v.fset.Position(p) + return Pos{pp.Filename, pp.Line - 1, pp.Column - 1} +} + +func (v *verifier) report(at Pos, endCol int, warning bool, msg string) { + v.reportRef(at, endCol, warning, msg, Pos{}) +} + +// reportRef reports once per place and message: a contract that fails on +// three paths is still one problem. +func (v *verifier) reportRef(at Pos, endCol int, warning bool, msg string, ref Pos) { + key := fmt.Sprintf("%s:%d:%d:%s", at.Path, at.Line, at.Col, msg) + if v.reported[key] { + return } - if err != nil { - fmt.Fprintln(os.Stderr, "litgo:", err) - return 1 + v.reported[key] = true + if !warning { + v.errors++ } - abs, _ := filepath.Abs(*out) - fmt.Println(abs) - return 0 + v.findings = append(v.findings, Finding{Pos: at, EndCol: endCol, Warning: warning, Message: msg, Ref: ref}) } +``` -func cmdCheck(args []string) int { - status, cache := 0, &check.Cache{} - for _, path := range args { - doc, err := load(path) - if err != nil { - fmt.Fprintln(os.Stderr, "litgo:", err) - status = 1 +```go +// kindOf reads a type the way the verifier sees it: integers, booleans, +// slices of integers, and structs declared in the package. +func (v *verifier) kindOf(t ast.Expr) (vkind, string) { + switch t := t.(type) { + case *ast.Ident: + switch t.Name { + case "int", "int8", "int16", "int32", "int64": + return kInt, "" + case "bool": + return kBool, "" + } + if _, ok := v.structs[t.Name]; ok { + return kStruct, t.Name + } + case *ast.StarExpr: + if k, name := v.kindOf(t.X); k == kStruct { + return k, name + } + case *ast.ArrayType: + if k, _ := v.kindOf(t.Elt); t.Len == nil && k == kInt { + return kSlice, "" + } + case *ast.ParenExpr: + return v.kindOf(t.X) + } + return kNone, "" +} + +// declarations notes the structs and the integer constants. +func (v *verifier) declarations(f *ast.File) { + for _, d := range f.Decls { + g, ok := d.(*ast.GenDecl) + if !ok { continue } - res := doc.Tangle() - diags := append(res.Diags, res.SyntaxCheck()...) - if len(diags) == len(res.Diags) { - diags = append(diags, check.Types(res, cache)...) - } - if report(diags) > 0 { - status = 1 + for _, spec := range g.Specs { + switch s := spec.(type) { + case *ast.TypeSpec: + if st, ok := s.Type.(*ast.StructType); ok { + fields := map[string]string{} + for _, fl := range st.Fields.List { + for _, n := range fl.Names { + fields[n.Name] = typeText(fl.Type) + } + } + v.structs[s.Name.Name] = fields + } + case *ast.ValueSpec: + if g.Tok != token.CONST { + continue + } + for i, n := range s.Names { + if i < len(s.Values) { + if c, ok := constInt(s.Values[i]); ok { + v.consts[n.Name] = c + } + } + } + } } } - return status } -``` -`cat` prints a document the way the editor shows it, so it is the quickest way -to see what the renderers do. +// functions notes the functions, once every struct they may mention is known. +func (v *verifier) functions(f *ast.File) { + for _, d := range f.Decls { + fd, ok := d.(*ast.FuncDecl) + if !ok || fd.Body == nil { + continue + } + f := &fn{key: fd.Name.Name, decl: fd} + if fd.Recv != nil && len(fd.Recv.List) == 1 { + r := fd.Recv.List[0] + k, typ := v.kindOf(r.Type) + name := "_" + if len(r.Names) == 1 { + name = r.Names[0].Name + } + f.recv = ¶m{name, k, typ} + f.key = strings.TrimPrefix(typeText(r.Type), "*") + "." + f.key + } + f.params = v.fields(fd.Type.Params, "") + f.results = v.fields(fd.Type.Results, "ret_") + v.funcs[f.key] = f + v.ordered = append(v.ordered, f) + } +} -```go -// cmdCat prints the document the way the editor shows it: math as Unicode, -// diagrams drawn, chunk syntax prettified. -func cmdCat(args []string) int { - if len(args) != 1 { - fmt.Fprintln(os.Stderr, "litgo cat: expected one input file") - return 2 +func (v *verifier) fields(list *ast.FieldList, unnamed string) []param { + var out []param + if list == nil { + return nil } - doc, err := load(args[0]) - if err != nil { - fmt.Fprintln(os.Stderr, "litgo:", err) - return 1 + for _, fl := range list.List { + k, typ := v.kindOf(fl.Type) + if len(fl.Names) == 0 { + out = append(out, param{fmt.Sprintf("%s%d", unnamed, len(out)), k, typ}) + } + for _, n := range fl.Names { + out = append(out, param{n.Name, k, typ}) + } } - a := doc.Analyze(lit.AnalyzeOptions{}) - blocks := map[int]lit.Item{} - inline := map[int][]lit.Item{} - for _, it := range a.Items { - if it.Kind == "table" { - continue // lining up a table is the editor's business - } else if len(it.Lines) > 0 { - blocks[it.Line] = it - } else { - inline[it.Line] = append(inline[it.Line], it) + return out +} + +func typeText(t ast.Expr) string { + switch t := t.(type) { + case *ast.Ident: + return t.Name + case *ast.StarExpr: + return "*" + typeText(t.X) + case *ast.ArrayType: + if t.Len == nil { + return "[]" + typeText(t.Elt) } } - for i := 0; i < len(doc.Lines); i++ { - if it, ok := blocks[i]; ok { - for _, l := range it.Lines { - fmt.Println(" " + l) + return "?" +} + +func constInt(e ast.Expr) (int64, bool) { + switch e := e.(type) { + case *ast.BasicLit: + if e.Kind == token.INT { + var v int64 + if _, err := fmt.Sscan(e.Value, &v); err == nil { + return v, true } - i = it.EndLine - continue } - line := doc.Lines[i] - items := inline[i] - sort.Slice(items, func(a, b int) bool { return items[a].Col < items[b].Col }) - for k := len(items) - 1; k >= 0; k-- { // right to left keeps columns valid - it := items[k] - line = line[:it.Col] + it.Text + line[min(it.EndCol, len(line)):] + case *ast.UnaryExpr: + if c, ok := constInt(e.X); ok && e.Op == token.SUB { + return -c, true } - fmt.Println(line) + case *ast.ParenExpr: + return constInt(e.X) } - return 0 + return 0, false } ``` -`go/format` accepts fragments (a list of statements as well as a whole file), -so most blocks can be formatted right where they are. A block that only parses -after its chunks are spliced in gets left alone. +Where an annotation stands decides whose it is. `Requires` belongs to the +function below it, and `Ensures` to the function above it, which is where VeGo +puts it: a postcondition reads best after the code, the way a conclusion does. +An `Ensures` written in the doc comment of a function, with its `Requires`, is +accepted too. Annotations inside a body wait there until the walk through the +body reaches them. + +gofmt has opinions about comments, and an annotation has to survive them, +because code gets formatted on save whether anyone thinks about annotations or +not. There are two. In a doc comment, gofmt puts a space after the slashes, so +`//@ Requires` above a function comes back as `// @ Requires`. And it puts a +blank line between a closing brace and a comment under it, so an `Ensures` +comes loose from its function. Neither changes what the annotation means +here: `// @` followed by a keyword is read as `//@`, and an `Ensures` looks +upward past blank lines. (`lit.ProofComment`, which decides what is an +annotation, is in [Finding the annotations](#finding-the-annotations), because +the editor's highlighting asks the same question.) ```go -func cmdFmt(args []string) int { - fs := flag.NewFlagSet("fmt", flag.ExitOnError) - write := fs.Bool("w", false, "write the result back instead of printing it") - fs.Parse(args) - if fs.NArg() != 1 { - fmt.Fprintln(os.Stderr, "litgo fmt: expected one input file") - return 2 +// annotations reads a file's //@ comments, parses them, and hands each to the +// function it belongs to. +func (v *verifier) annotations(f *ast.File) { + var list []*annot + for _, group := range f.Comments { + var open *annot // an annotation whose formula is not finished + for _, c := range group.List { + body, ok := lit.ProofComment(c.Text, open != nil) + if !ok { + open = nil + continue + } + if open != nil { + open.text += " " + strings.TrimSpace(body) + open.lines++ + if !continues(body) { + open = nil + } + continue + } + rest := strings.TrimLeft(body, " \t") + a := &annot{kind: "Assert", pos: c.Pos(), where: v.position(c.Pos()), lines: 1} + a.endCol = a.where.Col + len(c.Text) + word, after, _ := strings.Cut(rest, " ") + if lit.ProofKeywords[word] { + a.kind = word + rest = strings.TrimLeft(after, " \t") + } + a.text = strings.TrimSpace(rest) + a.first = len(a.text) + a.textCol = a.where.Col + len(c.Text) - len(rest) + list = append(list, a) + if continues(body) { + open = a + } + } } - doc, err := load(fs.Arg(0)) - if err != nil { - fmt.Fprintln(os.Stderr, "litgo:", err) - return 1 + for _, a := range list { + v.parse(a) } - lines := append([]string{}, doc.Lines...) - for i := len(doc.Blocks) - 1; i >= 0; i-- { - b := doc.Blocks[i] - if !b.Tangled() || b.Lang != "go" || b.Last() < b.First() || b.Strip > 0 { + v.all = append(v.all, list...) + + var decls []*fn + for _, fd := range v.ordered { + if v.fset.File(fd.decl.Pos()) == v.fset.File(f.Pos()) { + decls = append(decls, fd) + } + } + for _, a := range list { + if a.kind == "Predicate" { + a.used = true continue } - // format.Source accepts fragments; blocks that only make sense once - // their chunks are spliced in fail to parse and are left alone. So are - // blocks with inline references: gofmt would tidy away the comma that - // precedes a reference standing for the rest of an argument list. - if !b.Verbatim && inlineRefs(doc.Lines[b.First():b.Last()+1]) { + if v.inType(f, a) { + v.report(a.where, a.endCol, true, "litgo prove does not check contracts on interface methods") + a.used = true continue } - src := strings.Join(doc.Lines[b.First():b.Last()+1], "\n") + "\n" - out, err := format.Source([]byte(src)) - if err != nil { + var inside, before, after *fn + for _, fd := range decls { + switch { + case a.pos > fd.decl.Body.Lbrace && a.pos < fd.decl.Body.Rbrace: + inside = fd + case fd.decl.End() <= a.pos: + before = fd + case a.pos < fd.decl.Pos() && after == nil: + after = fd + } + } + // An Ensures belongs to the function above it, unless it is part of + // the doc comment of the function below and does not touch the one above. + if after != nil && after.decl.Doc != nil && a.pos >= after.decl.Doc.Pos() && a.pos < after.decl.Doc.End() && + !(before != nil && v.adjacent(before, a, list)) { + before = nil + } + var owner *fn + switch a.kind { + case "Requires": + if owner = after; inside != nil { + owner = nil + } + if owner != nil { + owner.requires = append(owner.requires, a) + } + case "Ensures", "Exsures": + if inside == nil { + if owner = before; owner == nil { + owner = after + } + } + if owner != nil && a.kind == "Ensures" { + owner.ensures = append(owner.ensures, a) + } else if owner != nil { + owner.exsures = append(owner.exsures, a) + } + case "Measure": + if owner = inside; owner == nil { + owner = after + } + if owner != nil { + owner.measure = append(owner.measure, a) + } + case "Property": + v.report(a.where, a.endCol, true, "litgo prove does not check Property annotations") + a.used = true continue + default: + if owner = inside; owner != nil { + owner.inside = append(owner.inside, a) + } } - formatted := strings.Split(strings.TrimRight(string(out), "\n"), "\n") - lines = append(append(append([]string{}, lines[:b.First()]...), formatted...), lines[b.Last()+1:]...) - } - text := strings.Join(lines, "\n") + "\n" - if *write { - if err := os.WriteFile(doc.Path, []byte(text), 0o644); err != nil { - fmt.Fprintln(os.Stderr, "litgo:", err) - return 1 + if owner != nil { + owner.annotated = true + owner.broken = owner.broken || a.e == nil + if a.kind != "Requires" && a.kind != "Ensures" && a.kind != "Exsures" && a.kind != "Measure" { + continue // the body marks its own as used, when it reaches them + } + a.used = true } - return 0 } - fmt.Print(text) - return 0 } ``` ```go -func inlineRefs(lines []string) bool { - for i, l := range lines { - for _, r := range lit.RefsIn(l, i) { - if r.Inline { - return true - } +// inType reports whether an annotation is written inside a type declaration, +// which is where VeGo puts the contract of an interface method. +func (v *verifier) inType(f *ast.File, a *annot) bool { + for _, d := range f.Decls { + if g, ok := d.(*ast.GenDecl); ok && g.Tok == token.TYPE && a.pos > g.Pos() && a.pos < g.End() { + return true } } return false } -func cmdAnalyze(args []string) int { - fs := flag.NewFlagSet("analyze", flag.ExitOnError) - path := fs.String("path", "", "the document's file name") - upright := fs.Bool("upright", false, "do not italicise math letters") - fs.Parse(args) - return printJSON(stdinDoc(*path).Analyze(lit.AnalyzeOptions{Upright: *upright})) +// adjacent reports whether nothing but annotations lies between the end of a +// function and an annotation below it. +func (v *verifier) adjacent(f *fn, a *annot, list []*annot) bool { + line := v.position(f.decl.End()).Line + for _, b := range list { + if b.where.Line == line+1 && b.pos > f.decl.End() { + line += b.lines + if b == a { + return true + } + } + } + return false } -func cmdUntangle(args []string) int { - fs := flag.NewFlagSet("untangle", flag.ExitOnError) - path := fs.String("path", "", "the document's file name") - start := fs.Int("start", -1, "first selected line") - end := fs.Int("end", -1, "last selected line") - startCol := fs.Int("start-col", 0, "first selected byte (charwise)") - endCol := fs.Int("end-col", -1, "byte after the selection (charwise)") - name := fs.String("name", "", "chunk name; suggested when empty") - fs.Parse(args) - res, err := stdinDoc(*path).Untangle(lit.Selection{StartLine: *start, EndLine: *end, StartCol: *startCol, EndCol: *endCol}, *name) - if err != nil { - return jsonError(err) +func (v *verifier) parse(a *annot) { + var err error + switch a.kind { + case "Predicate": + var p *predicate + if p, err = parsePredicate(a.text, &v.parseFresh); err == nil { + v.preds[p.name] = p + a.e = p.body + } + case "Immutable": + a.e = eTrue + default: + a.e, err = parseFormula(a.text, &v.parseFresh) + number := a.kind == "Variant" || a.kind == "Measure" + if err == nil && a.e.op != opCall && isBool(a.e) == number { + a.e = nil + err = &syntaxError{0, "this has to be a formula, something that is true or false"} + if number { + err = &syntaxError{0, "this has to be an integer expression, something that can get smaller"} + } + } + } + if se, ok := err.(*syntaxError); ok { + at := a.where + at.Col = a.textCol + min(se.pos, a.first) + v.report(at, a.endCol, false, "in this "+a.kind+": "+se.msg) } - return printJSON(res) } +``` -func cmdVersion(args []string) int { - fmt.Printf("%s\n", VERSION) - return 0 -} +## The language of formulas -func cmdRename(args []string) int { - fs := flag.NewFlagSet("rename", flag.ExitOnError) - path := fs.String("path", "", "the document's file name") - from := fs.String("from", "", "current chunk name") - to := fs.String("to", "", "new chunk name") - fs.Parse(args) - edits, err := stdinDoc(*path).Rename(*from, *to) - if err != nil { - return jsonError(err) - } - return printJSON(map[string]any{"edits": edits, "name": lit.NormName(*to)}) -} -``` + + + -# The feedback loop +Formulas are written the way they are on a blackboard, in ASCII: - - - +| | | +|:--|:--| +| `^` `v` `~` `->` `<-` `<->` | and, or, not, implies, is implied by, if and only if | +| `=` `<>` `<` `<=` `>` `>=` | comparisons, which chain: `0 <= i < n` (`==` and `!=` work too) | +| `+` `-` `*` `/` `%` | integer arithmetic, with Go's division | +| `len(A)`, `A[i]` | the length and the elements of a slice | +| `Forall k in [a, b) : P`, `Exists`, `Unique` | quantifiers over an interval, with `[` `]` for an end that is included and `(` `)` for one that is not, `...` for no end at all, or over all of `Z` | +| `A[a:b) <= m`, `m in A[a:b)` | every element of the range is at most `m`; some element is `m` | +| `x'` | `x` after the assignment, where `x` is before it | -`litgo run` is the reason everything above exists. It tangles, compiles and -runs the program, and reports every position in terms of the document. +The length of a slice can also be written `|A|`, and Go's `&&`, `||` and `!` +are read as `^`, `v` and `~`, for the fingers that type them anyway. -```mermaid -sequenceDiagram - participant E as editor - participant L as litgo run - participant G as go build - participant P as program - E->>L: pool.lit.md - L->>G: pool.go - G-->>L: pool.go:31:14: undefined: jobb - L-->>E: pool.lit.md:67:15: undefined: jobb - L->>P: start - P-->>L: output, or a panic with a stack trace - L-->>E: output, positions translated -``` +One type serves for terms, for formulas, and later for the symbolic values of +program variables, so that a Go expression and an annotation can meet in the +same formula. -Progress goes to a person as text, or to an editor as one JSON event per line. -The events are `tangle`, `diagnostic`, `build`, `output` and `exit`. +```go +type op int -The reporter has no lock, so only one goroutine is allowed to use it. That is -the goroutine that runs `runFile`. When a program's output has to be read from -two pipes at once, the goroutines that read the pipes send their lines to that -one goroutine over a channel, and it does all of the reporting. +const ( + opNum op = iota + opVar + opTrue + opFalse + opAdd + opSub + opMul + opDiv + opMod + opNeg + opSel // args: array, index + opLen // args: array + opCall // name(args): a predicate, or a Go function that is one expression + opRange // name[lo:hi) as one side of a comparison; args: lo, hi (nil when omitted) + opLt + opLe + opEq + opNe + opIn // x in A[lo:hi) + opAnd + opOr + opNot + opImp + opIff + opForall // name is the bound variable, args[0] the body + opExists +) -```go -// reporter carries the run's progress to a person (plain text) or to an -// editor (one JSON event per line). It belongs to one goroutine. -type reporter struct { - json bool - seen map[string]bool -} +type sort_ int -func (r *reporter) emit(event string, fields map[string]any) { - if !r.json { - return - } - fields["event"] = event - b, _ := json.Marshal(fields) - fmt.Println(string(b)) -} +const ( + sInt sort_ = iota + sBool + sArr +) -func (r *reporter) diag(d lit.Diag) { - key := fmt.Sprintf("%s:%d:%d:%s", d.File, d.Line, d.Col, d.Message) - if r.seen[key] { - return - } - r.seen[key] = true - if r.json { - b, _ := json.Marshal(d) - var fields map[string]any - json.Unmarshal(b, &fields) - r.emit("diagnostic", fields) - return - } - sev := map[int]string{lit.SevError: "error", lit.SevWarning: "warning", lit.SevInfo: "note"}[d.Severity] - fmt.Fprintf(os.Stderr, "%s:%d:%d: %s: %s\n", relPath(d.File), d.Line+1, d.Col+1, sev, d.Message) +// expr is a term or a formula. Annotations parse to it, Go expressions are +// translated to it, and the symbolic values of program variables are made of it. +type expr struct { + op op + val int64 + name string + primes int + sort sort_ + args []*expr + pos int // byte offset in the annotation's text + open [2]bool // opRange: is the low, the high end exclusive } +``` -func (r *reporter) output(stream, text string) { - if r.json { - r.emit("output", map[string]any{"stream": stream, "text": text}) - return +```go +func num(v int64) *expr { return &expr{op: opNum, val: v} } +func sym(name string, s sort_) *expr { return &expr{op: opVar, name: name, sort: s} } +func mk(o op, args ...*expr) *expr { return &expr{op: o, args: args} } + +var ( + eTrue = &expr{op: opTrue} + eFalse = &expr{op: opFalse} +) + +func and(a, b *expr) *expr { + if a.op == opTrue { + return b } - if stream == "stderr" { - fmt.Fprintln(os.Stderr, text) - } else { - fmt.Println(text) + if b.op == opTrue { + return a } + return mk(opAnd, a, b) } -func (r *reporter) status(format string, args ...any) { - if !r.json { - fmt.Fprintf(os.Stderr, "litgo: "+format+"\n", args...) +func not(a *expr) *expr { return mk(opNot, a) } + +func conj(list []*expr) *expr { + out := eTrue + for _, e := range list { + out = and(out, e) } + return out } -func relPath(p string) string { - if wd, err := os.Getwd(); err == nil { - if rel, err := filepath.Rel(wd, p); err == nil && !strings.HasPrefix(rel, "..") { - return rel - } +// isBool says whether e is a formula rather than an integer term. +func isBool(e *expr) bool { + switch e.op { + case opTrue, opFalse, opLt, opLe, opEq, opNe, opIn, opAnd, opOr, opNot, opImp, opIff, opForall, opExists: + return true + case opVar: + return e.sort == sBool } - return p + return false } ``` -## Translating positions - -Anything that looks like `file.go:12:5` and names a file litgo tangled gets -rewritten, wherever it shows up. That can be in a compiler error, in the middle -of an error message that mentions a second position, or in a stack trace. - ```go -// remapper rewrites positions in tangled files back to the literate source. -type remapper struct { - dir string - outputs map[string]*mapped // by absolute output path +func (e *expr) String() string { + bin := map[op]string{opAdd: "+", opSub: "-", opMul: "*", opDiv: "/", opMod: "%", opLt: "<", opLe: "<=", + opEq: "=", opNe: "<>", opAnd: "^", opOr: "v", opImp: "->", opIff: "<->", opIn: "in"} + switch e.op { + case opNum: + return strconv.FormatInt(e.val, 10) + case opVar: + return e.name + strings.Repeat("'", e.primes) + case opTrue: + return "true" + case opFalse: + return "false" + case opNeg: + return "-" + e.args[0].String() + case opNot: + return "~" + e.args[0].String() + case opSel: + return e.args[0].String() + "[" + e.args[1].String() + "]" + case opLen: + return "|" + e.args[0].String() + "|" + case opCall: + parts := make([]string, len(e.args)) + for i, a := range e.args { + parts[i] = a.String() + } + return e.name + "(" + strings.Join(parts, ", ") + ")" + case opRange: + return e.name + "[:]" + case opForall, opExists: + q := "Forall " + if e.op == opExists { + q = "Exists " + } + return "(" + q + e.name + " : " + e.args[0].String() + ")" + } + return "(" + e.args[0].String() + " " + bin[e.op] + " " + e.args[1].String() + ")" } +``` -// mapped is a tangled file together with the document it came from. -type mapped struct { - *lit.Output - doc *lit.Doc -} +Everything that is done to a formula (giving names their values, unfolding a +predicate, instantiating a quantifier) is a substitution, and the one thing a +substitution must not do is touch a variable that a quantifier binds. -var posRe = regexp.MustCompile(`((?:[A-Za-z]:)?[\w./\\-]*\.go):(\d+)(?::(\d+))?`) +```go +// rewrite rebuilds e bottom-up. f sees every variable that is not bound by a +// quantifier inside e, and returns its replacement or nil to keep it. +func rewrite(e *expr, f func(v *expr) *expr) *expr { + return rewriteBound(e, f, nil) +} -// lookup finds the tangled file a tool is talking about. Compilers name -// files relative to where they ran; `go test` names them relative to the -// package, so a bare file name is matched if it is unambiguous. -func (m *remapper) lookup(path string) *mapped { - abs := path - if !filepath.IsAbs(abs) { - abs = filepath.Join(m.dir, abs) - } - if o := m.outputs[filepath.Clean(abs)]; o != nil || filepath.Base(path) != path { - return o +func rewriteBound(e *expr, f func(v *expr) *expr, bound []string) *expr { + if e == nil { + return nil } - var found *mapped - for p, o := range m.outputs { - if filepath.Base(p) == path { - if found != nil { - return nil + switch e.op { + case opNum, opTrue, opFalse: + return e + case opVar: + for _, b := range bound { + if b == e.name && e.primes == 0 { + return e } - found = o } + if r := f(e); r != nil { + return r + } + return e } - return found + c := *e + c.args = make([]*expr, len(e.args)) + if e.op == opForall || e.op == opExists { + bound = append(bound[:len(bound):len(bound)], e.name) + } + for i, a := range e.args { + c.args[i] = rewriteBound(a, f, bound) + } + return &c } -// line rewrites every `file.go:line:col` in a line of tool output and returns -// the first position it translated. -func (m *remapper) line(s string) (string, *lit.Diag) { - var first *lit.Diag - out := posRe.ReplaceAllStringFunc(s, func(match string) string { - g := posRe.FindStringSubmatch(match) - res := m.lookup(g[1]) - if res == nil { - return match - } - ln, _ := strconv.Atoi(g[2]) - col := 1 - if g[3] != "" { - col, _ = strconv.Atoi(g[3]) - } - srcLine, srcCol, _, ok := res.Map(ln-1, col-1) - if !ok { - return match - } - if first == nil { - first = &lit.Diag{File: res.doc.Path, Line: srcLine, Col: srcCol} - } - if g[3] == "" { - return fmt.Sprintf("%s:%d", relPath(res.doc.Path), srcLine+1) +// subst replaces the free variable name by r. +func subst(e *expr, name string, r *expr) *expr { + return rewrite(e, func(v *expr) *expr { + if v.name == name && v.primes == 0 { + return r } - return fmt.Sprintf("%s:%d:%d", relPath(res.doc.Path), srcLine+1, srcCol+1) + return nil }) - return out, first +} + +// A predicate is a named formula with parameters; using it means its body. +type predicate struct { + name string + params []string + body *expr } ``` +### Reading a formula + +The letter `v` is *or* when it stands where an operator can stand, and a +variable anywhere else, so a program can still have a variable called `v`. A +name may have dots in it, which is how `s.Len` is one name, and primes after +it. + ```go -var compilerLineRe = regexp.MustCompile(`^(?:\./)?([^\s:]+\.go):(\d+)(?::(\d+))?:\s+(.*)$`) +type lexeme struct { + kind string // "num", "id", "op", "eof" + text string + pos int +} -// toolOutput turns compiler or vet output into diagnostics, and reports -// whatever it could not place (linker errors, package-level complaints). -func (m *remapper) toolOutput(out []byte, sev int, source string, r *reporter) { - var last *lit.Diag - flush := func() { - if last != nil { - r.diag(*last) - last = nil - } - } - for _, raw := range strings.Split(strings.TrimRight(string(out), "\n"), "\n") { - if raw == "" || strings.HasPrefix(raw, "# ") { - continue - } - if g := compilerLineRe.FindStringSubmatch(raw); g != nil { - flush() - if _, pos := m.line(g[1] + ":" + g[2] + ":" + firstNonEmpty(g[3], "1")); pos != nil { - msg, _ := m.line(g[4]) - pos.Severity, pos.Message, pos.Source = sev, msg, source - last = pos - continue - } - } - if last != nil && (strings.HasPrefix(raw, "\t") || strings.HasPrefix(raw, " ")) { - cont, _ := m.line(strings.TrimSpace(raw)) - last.Message += "\n" + cont - continue - } - flush() - text, _ := m.line(raw) - r.output("stderr", text) - } - flush() +type syntaxError struct { + pos int + msg string } -func firstNonEmpty(a, b string) string { - if a != "" { - return a +func (e *syntaxError) Error() string { return e.msg } + +var operators = []string{"<->", "::=", "...", "->", "<-", "<=", ">=", "<>", "==", "!=", "&&", "||", + "^", "~", "!", "=", "<", ">", "+", "-", "*", "/", "%", "(", ")", "[", "]", ",", ":", "|"} + +func lex(s string) ([]lexeme, error) { + var out []lexeme + for i := 0; i < len(s); { + c := s[i] + switch { + case c == ' ' || c == '\t': + i++ + case c >= '0' && c <= '9': + j := i + for j < len(s) && s[j] >= '0' && s[j] <= '9' { + j++ + } + out = append(out, lexeme{"num", s[i:j], i}) + i = j + case c == '_' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z': + j := i + for j < len(s) && (s[j] == '_' || s[j] == '.' && j+1 < len(s) && s[j+1] != '.' || s[j] >= 'a' && s[j] <= 'z' || s[j] >= 'A' && s[j] <= 'Z' || s[j] >= '0' && s[j] <= '9') { + j++ + } + for j < len(s) && s[j] == '\'' { + j++ + } + out = append(out, lexeme{"id", s[i:j], i}) + i = j + default: + found := false + for _, o := range operators { + if strings.HasPrefix(s[i:], o) { + out = append(out, lexeme{"op", o, i}) + i += len(o) + found = true + break + } + } + if !found { + return nil, &syntaxError{i, fmt.Sprintf("unexpected %q", string(c))} + } + } } - return b + return append(out, lexeme{"eof", "", len(s)}), nil } ``` ```go -// writeTangled writes a file unless that would clobber a hand-written one. -func writeTangled(o *lit.Output, force bool) error { - if old, err := os.ReadFile(o.Path); err == nil { - if bytes.Equal(old, o.Bytes()) { - return nil // leave the timestamp alone so builds stay cached - } - if !force && !lit.Generated(old) && len(bytes.TrimSpace(old)) > 0 && lit.CommentsFor(o.File.Lang).Line != "" { - return fmt.Errorf("%s exists and was not generated by litgo; use --force to overwrite it, or tangle to another ", relPath(o.Path)) - } +// continues reports whether an annotation line ends in a token that cannot +// end a formula, which is how a formula says it goes on on the next line. +func continues(text string) bool { + toks, err := lex(text) + if err != nil || len(toks) < 2 { + return false } - if err := os.MkdirAll(filepath.Dir(o.Path), 0o755); err != nil { - return err + last := toks[len(toks)-2] + if last.kind == "id" { + return last.text == "v" || last.text == "in" } - return os.WriteFile(o.Path, o.Bytes(), 0o644) + if last.kind != "op" { + return false + } + switch last.text { + case ")", "]", "...", "|": + return false + } + return true } ``` -## The pipeline +The parser is recursive descent, one function for each level of binding +strength, loosest first: `<->`, then `->`, `v`, `^`, `~`, the comparisons, and +the arithmetic. A syntax error is a panic that carries its column and is +caught at the top, so that the error lands under the character that caused +it. ```go -type runOptions struct { - json bool - force bool - vet bool - timeout time.Duration - args []string +type reader struct { + toks []lexeme + i int + fresh *int } -``` -```go -// runFile is the feedback loop: tangle, compile, run, with every position in -// every message pointing at the .lit.md source. -func runFile(path string, o runOptions) int { - r := &reporter{json: o.json, seen: map[string]bool{}} - // <> +func (p *reader) peek() lexeme { return p.toks[p.i] } +func (p *reader) next() lexeme { + t := p.toks[p.i] + if t.kind != "eof" { + p.i++ + } + return t +} - m := &remapper{dir: dir, outputs: map[string]*mapped{}} - // <> - // <> +func (p *reader) is(text string) bool { + t := p.peek() + return t.kind == "op" && t.text == text +} - // <> +func (p *reader) isWord(text string) bool { + t := p.peek() + return t.kind == "id" && t.text == text +} - // <> - return execute(r, m, exec.Command(bin, o.args...), dir, o.timeout, false) +func (p *reader) fail(t lexeme, format string, args ...any) { + panic(&syntaxError{t.pos, fmt.Sprintf(format, args...)}) } -``` - -```go -abs, err := filepath.Abs(path) -if err != nil { - return fail(r, err) +func (p *reader) expect(text string) { + if t := p.next(); t.kind != "op" || t.text != text { + what := t.text + if t.kind == "eof" { + what = "the end of the annotation" + } + p.fail(t, "expected %q, found %s", text, what) + } } -dir := filepath.Dir(abs) -src, err := os.ReadFile(abs) -if err != nil { - return fail(r, err) + +// parseFormula parses a whole annotation body. +func parseFormula(text string, fresh *int) (e *expr, err error) { + toks, err := lex(text) + if err != nil { + return nil, err + } + p := &reader{toks: toks, fresh: fresh} + defer func() { + if r := recover(); r != nil { + se, ok := r.(*syntaxError) + if !ok { + panic(r) + } + e, err = nil, se + } + }() + e = p.iff() + if t := p.peek(); t.kind != "eof" { + p.fail(t, "unexpected %q", t.text) + } + return e, nil } -doc := lit.Parse(abs, src) -// A custom run command may build a whole package, so tangle the siblings too. -docs := []*lit.Doc{doc} -if doc.Run != "" { - siblings, _ := filepath.Glob(filepath.Join(dir, "*.lit.md")) - for _, s := range siblings { - if s == abs { - continue +// parsePredicate parses "Name(a, b) ::= body". +func parsePredicate(text string, fresh *int) (pr *predicate, err error) { + toks, err := lex(text) + if err != nil { + return nil, err + } + p := &reader{toks: toks, fresh: fresh} + defer func() { + if r := recover(); r != nil { + se, ok := r.(*syntaxError) + if !ok { + panic(r) + } + pr, err = nil, se } - if b, err := os.ReadFile(s); err == nil { - docs = append(docs, lit.Parse(s, b)) + }() + name := p.next() + if name.kind != "id" { + p.fail(name, "a predicate needs a name") + } + pr = &predicate{name: name.text} + p.expect("(") + for !p.is(")") { + t := p.next() + if t.kind != "id" { + p.fail(t, "expected a parameter name") } + pr.params = append(pr.params, t.text) + if !p.is(")") { + p.expect(",") + } + } + p.expect(")") + p.expect("::=") + pr.body = p.iff() + if t := p.peek(); t.kind != "eof" { + p.fail(t, "unexpected %q", t.text) } + return pr, nil } ``` -Every file that gets written is registered with the remapper. If the document's -structure has problems, the run stops here, because there is no point compiling -half a program. - - ```go -ok := true -for _, d := range docs { - res := d.Tangle() - for _, dg := range res.Diags { - r.diag(dg) +func (p *reader) iff() *expr { + l := p.imp() + for p.is("<->") { + t := p.next() + l = &expr{op: opIff, args: []*expr{l, p.imp()}, pos: t.pos} } - if res.HasErrors() { - ok = false - continue + return l +} + +func (p *reader) imp() *expr { + l := p.or() + if p.is("->") { + t := p.next() + return &expr{op: opImp, args: []*expr{l, p.imp()}, pos: t.pos} } - for _, out := range res.Files { - if err := writeTangled(out, o.force); err != nil { - r.diag(lit.Diag{File: d.Path, Line: max(out.File.Line, 0), Severity: lit.SevError, Message: err.Error(), Source: "litgo"}) - ok = false - continue - } - m.outputs[filepath.Clean(out.Path)] = &mapped{out, d} + if p.is("<-") { + t := p.next() + return &expr{op: opImp, args: []*expr{p.imp(), l}, pos: t.pos} } - if n := len(res.Files); n == 1 { - r.emit("tangle", map[string]any{"file": d.Path, "out": res.Files[0].Path}) - r.status("tangled %s β†’ %s", relPath(d.Path), relPath(res.Files[0].Path)) - } else if n > 1 { - r.emit("tangle", map[string]any{"file": d.Path, "out": fmt.Sprintf("%d files", n)}) - r.status("tangled %s β†’ %d files", relPath(d.Path), n) + return l +} + +func (p *reader) or() *expr { + l := p.and() + for p.isWord("v") || p.is("||") { + t := p.next() + l = &expr{op: opOr, args: []*expr{l, p.and()}, pos: t.pos} } + return l } -if !ok { - r.emit("build", map[string]any{"ok": false, "ms": 0, "stage": "tangle"}) - return 1 + +func (p *reader) and() *expr { + l := p.neg() + for p.is("^") || p.is("&&") { + t := p.next() + l = &expr{op: opAnd, args: []*expr{l, p.neg()}, pos: t.pos} + } + return l +} + +func (p *reader) neg() *expr { + if p.is("~") || p.is("!") { + t := p.next() + return &expr{op: opNot, args: []*expr{p.neg()}, pos: t.pos} + } + if p.isWord("Forall") || p.isWord("Exists") || p.isWord("Unique") { + return p.quantifier() + } + return p.comparison() } ``` -This document takes the first branch. Its `run:` directive vets, tests and -rebuilds litgo. +A chain of comparisons is the conjunction of its links. A comparison with a +range of a slice on one side is a quantifier in disguise, and the parser takes +the disguise off: `A[0:i) <= m` becomes `Forall k : 0 <= k < i -> A[k] <= m`, +and `m in A[0:i)` becomes an `Exists`. - ```go -// A document that writes several files is a package or a module, and -// only it knows how it is built -command := doc.Run -if command == "" && len(doc.Files) > 1 { - command = "go build ./..." +var comparisons = map[string]bool{"=": true, "==": true, "<>": true, "!=": true, "<": true, "<=": true, ">": true, ">=": true} + +// comparison parses a chain: 0 <= i < n means 0 <= i and i < n. +func (p *reader) comparison() *expr { + l := p.sum() + var out *expr + for { + t := p.peek() + in := t.kind == "id" && t.text == "in" + if !in && !(t.kind == "op" && comparisons[t.text]) { + break + } + p.next() + r := p.sum() + c := p.compare(t, l, r) + if out == nil { + out = c + } else { + out = &expr{op: opAnd, args: []*expr{out, c}, pos: t.pos} + } + l = r + } + if out == nil { + if l.op == opRange { + p.fail(p.peek(), "a range like %s[a:b) can only be compared with something", l.name) + } + return l + } + return out } -if command != "" { - r.status("$ %s", command) - return execute(r, m, exec.Command("sh", "-c", command), dir, o.timeout, true) + +func (p *reader) compare(t lexeme, l, r *expr) *expr { + if l.op == opRange || r.op == opRange { + return p.overRange(t, l, r) + } + switch t.text { + case "=", "==": + if isBool(l) || isBool(r) { + return &expr{op: opIff, args: []*expr{l, r}, pos: t.pos} + } + return &expr{op: opEq, args: []*expr{l, r}, pos: t.pos} + case "<>", "!=": + if isBool(l) || isBool(r) { + return &expr{op: opNot, args: []*expr{{op: opIff, args: []*expr{l, r}, pos: t.pos}}, pos: t.pos} + } + return &expr{op: opNe, args: []*expr{l, r}, pos: t.pos} + case "<": + return &expr{op: opLt, args: []*expr{l, r}, pos: t.pos} + case "<=": + return &expr{op: opLe, args: []*expr{l, r}, pos: t.pos} + case ">": + return &expr{op: opLt, args: []*expr{r, l}, pos: t.pos} + case ">=": + return &expr{op: opLe, args: []*expr{r, l}, pos: t.pos} + } + p.fail(t, "%q needs a range like A[a:b) on its right", t.text) + return nil } -def := doc.Files[0] -main := m.outputs[filepath.Clean(doc.OutPath(def))] -if def.Lang != "go" || main == nil { - r.status("no run step for tangler %q; add ", def.Lang) - r.emit("build", map[string]any{"ok": true, "ms": 0}) - return 0 + +// overRange turns a comparison with A[lo:hi) into a quantifier over a fresh +// index: an order comparison holds for all elements, = and in for some. +func (p *reader) overRange(t lexeme, l, r *expr) *expr { + if l.op == opRange && r.op == opRange { + p.fail(t, "only one side of a comparison can be a range") + } + rng := r + if l.op == opRange { + rng = l + } + *p.fresh++ + idx := &expr{op: opVar, name: fmt.Sprintf("_idx%d", *p.fresh)} + elem := &expr{op: opSel, args: []*expr{{op: opVar, name: rng.name, sort: sArr}, idx}} + lo, hi := rng.args[0], rng.args[1] + if lo == nil { + lo = num(0) + } + if hi == nil { + hi = &expr{op: opLen, args: []*expr{{op: opVar, name: rng.name, sort: sArr}}} + } + within := mk(opAnd, mk(bound(rng.open[0]), lo, idx), mk(bound(rng.open[1]), idx, hi)) + if rng.args[1] == nil { + within.args[1] = mk(opLt, idx, hi) + } + if t.text == "in" || t.text == "=" || t.text == "==" { + other := l + if l.op == opRange { + other = r + } + return &expr{op: opExists, name: idx.name, args: []*expr{mk(opAnd, within, mk(opEq, other, elem))}, pos: t.pos} + } + if t.text == "<>" || t.text == "!=" { + other := l + if l.op == opRange { + other = r + } + return &expr{op: opForall, name: idx.name, args: []*expr{mk(opImp, within, mk(opNe, other, elem))}, pos: t.pos} + } + var c *expr + if l.op == opRange { + c = p.compare(t, elem, r) + } else { + c = p.compare(t, l, elem) + } + return &expr{op: opForall, name: idx.name, args: []*expr{mk(opImp, within, c)}, pos: t.pos} +} + +func bound(open bool) op { + if open { + return opLt + } + return opLe } ``` -Building and running are separate steps. That way litgo can tell a compile -error apart from a program that exits non-zero, and it can time both steps. +```go +func (p *reader) sum() *expr { + l := p.product() + for p.is("+") || p.is("-") { + t := p.next() + o := opAdd + if t.text == "-" { + o = opSub + } + l = &expr{op: o, args: []*expr{l, p.product()}, pos: t.pos} + } + return l +} + +func (p *reader) product() *expr { + l := p.unary() + for p.is("*") || p.is("/") || p.is("%") { + t := p.next() + o := map[string]op{"*": opMul, "/": opDiv, "%": opMod}[t.text] + l = &expr{op: o, args: []*expr{l, p.unary()}, pos: t.pos} + } + return l +} + +func (p *reader) unary() *expr { + if p.is("-") { + t := p.next() + return &expr{op: opNeg, args: []*expr{p.unary()}, pos: t.pos} + } + return p.primary() +} + +func (p *reader) primary() *expr { + t := p.next() + switch { + case t.kind == "num": + v, err := strconv.ParseInt(t.text, 10, 64) + if err != nil { + p.fail(t, "%s is too large", t.text) + } + return &expr{op: opNum, val: v, pos: t.pos} + case t.kind == "op" && t.text == "(": + e := p.iff() + p.expect(")") + return e + case t.kind == "op" && t.text == "|": + a := p.next() + if a.kind != "id" { + p.fail(a, "|…| takes the name of a slice") + } + p.expect("|") + return &expr{op: opLen, args: []*expr{{op: opVar, name: a.text, sort: sArr, pos: a.pos}}, pos: t.pos} + case t.kind == "id": + return p.named(t) + case t.kind == "eof": + p.fail(t, "the formula stops short") + } + p.fail(t, "unexpected %q", t.text) + return nil +} +``` - ```go -tmp, err := os.MkdirTemp("", "litgo-run") -if err != nil { - return fail(r, err) +// named parses what starts with an identifier: a variable, true, len(A), +// A[i], a range A[a:b), or P(x, y). +func (p *reader) named(t lexeme) *expr { + name := strings.TrimRight(t.text, "'") + primes := len(t.text) - len(name) + switch name { + case "true": + return &expr{op: opTrue, pos: t.pos} + case "false": + return &expr{op: opFalse, pos: t.pos} + case "Forall", "Exists", "Unique", "in": + p.fail(t, "%s cannot be used as a name", name) + } + v := &expr{op: opVar, name: name, primes: primes, pos: t.pos} + if p.is("[") { + p.next() + if r := p.rangeAfter(v, false); r != nil { + return r + } + idx := p.sum() + if p.is(":") { + p.next() + return p.rangeEnd(v, idx, false) + } + p.expect("]") + v.sort = sArr + return &expr{op: opSel, args: []*expr{v, idx}, pos: t.pos} + } + if p.is("(") { + p.next() + if r := p.rangeAfter(v, true); r != nil { + return r + } + var args []*expr + for !p.is(")") { + a := p.iff() + if len(args) == 0 && p.is(":") { + p.next() + return p.rangeEnd(v, a, true) + } + args = append(args, a) + if !p.is(")") { + p.expect(",") + } + } + p.expect(")") + if name == "len" && len(args) == 1 && args[0].op == opVar { + args[0].sort = sArr + return &expr{op: opLen, args: args, pos: t.pos} + } + return &expr{op: opCall, name: name, args: args, pos: t.pos} + } + return v } -defer os.RemoveAll(tmp) -bin := filepath.Join(tmp, "prog") -start := time.Now() -build := exec.Command("go", /*<>*/) -build.Dir = dir -out, err := build.CombinedOutput() -ms := time.Since(start).Milliseconds() -m.toolOutput(out, lit.SevError, "go build", r) -if err != nil { - r.emit("build", map[string]any{"ok": false, "ms": ms, "stage": "compile"}) - r.status("build failed (%d ms)", ms) - return 1 +// rangeAfter handles A[:b) and A[:], where the low end is left out. +func (p *reader) rangeAfter(v *expr, open bool) *expr { + if !p.is(":") { + return nil + } + p.next() + return p.rangeEnd(v, nil, open) +} + +func (p *reader) rangeEnd(v, lo *expr, open bool) *expr { + r := &expr{op: opRange, name: v.name, args: []*expr{lo, nil}, pos: v.pos} + r.open[0] = open + if !p.is("]") && !p.is(")") { + r.args[1] = p.sum() + } + t := p.next() + switch { + case t.kind == "op" && t.text == ")": + r.open[1] = true + case t.kind == "op" && t.text == "]": + default: + p.fail(t, "a range ends in ] or )") + } + return r } -r.emit("build", map[string]any{"ok": true, "ms": ms}) -r.status("built in %d ms", ms) ``` -The compiler stops after ten errors unless you tell it not to, and in a tight -loop it is better to see all of them. +A quantifier's domain is folded into its body, as the hypothesis of a `Forall` +and as a conjunct of an `Exists`, so that past this point a quantifier ranges +over all integers and there is one kind of each. `Unique x : P(x)` says that +some `x` makes `P` true and that anything that makes `P` true is that `x`. - ```go -"build", "-gcflags=-e", "-o", bin, filepath.Base(main.Path) +// quantifier parses Forall x in D : body. The body reaches as far as it can. +func (p *reader) quantifier() *expr { + q := p.next() + x := p.next() + if x.kind != "id" || strings.HasSuffix(x.text, "'") { + p.fail(x, "%s needs a variable", q.text) + } + if !p.isWord("in") { + p.fail(p.peek(), "expected \"in\" and a domain, as in %s %s in [0, n) : …", q.text, x.text) + } + p.next() + v := &expr{op: opVar, name: x.text} + within := p.domain(v) + p.expect(":") + body := p.iff() + switch q.text { + case "Forall": + if within != nil { + body = mk(opImp, within, body) + } + return &expr{op: opForall, name: x.text, args: []*expr{body}, pos: q.pos} + case "Exists": + if within != nil { + body = mk(opAnd, within, body) + } + return &expr{op: opExists, name: x.text, args: []*expr{body}, pos: q.pos} + } + // Unique x: some x makes the body true, and whatever makes it true is x. + if within != nil { + body = mk(opAnd, within, body) + } + *p.fresh++ + other := fmt.Sprintf("_u%d", *p.fresh) + same := &expr{op: opForall, name: other, args: []*expr{ + mk(opImp, subst(body, x.text, &expr{op: opVar, name: other}), mk(opEq, &expr{op: opVar, name: other}, v))}} + return &expr{op: opExists, name: x.text, args: []*expr{mk(opAnd, body, same)}, pos: q.pos} +} + +// domain parses Z or an interval, and returns what it says about v. +func (p *reader) domain(v *expr) *expr { + t := p.next() + if t.kind == "id" && t.text == "Z" { + return nil + } + if t.kind == "id" && t.text == "B" { + p.fail(t, "litgo prove quantifies over integers only") + } + if t.kind != "op" || t.text != "[" && t.text != "(" { + p.fail(t, "expected a domain: Z, or an interval like [0, n)") + } + var parts []*expr + if p.is("...") { + p.next() + } else { + parts = append(parts, mk(bound(t.text == "("), p.sum(), v)) + } + p.expect(",") + if p.is("...") { + p.next() + if c := p.next(); c.kind != "op" || c.text != ")" && c.text != "]" { + p.fail(c, "an interval ends in ] or )") + } + } else { + hi := p.sum() + c := p.next() + if c.kind != "op" || c.text != ")" && c.text != "]" { + p.fail(c, "an interval ends in ] or )") + } + parts = append(parts, mk(bound(c.text == ")"), v, hi)) + } + return conj(parts) +} ``` - ```go -if o.vet { - vet := exec.Command("go", "vet", filepath.Base(main.Path)) - vet.Dir = dir - out, _ := vet.CombinedOutput() - m.toolOutput(out, lit.SevWarning, "go vet", r) -} -if def.Package != "main" && def.Package != "" { - r.status("package %s compiles; nothing to run", def.Package) - r.emit("exit", map[string]any{"code": 0, "ms": 0, "ran": false}) - return 0 +// names lists the free variables of e, sorted. +func names(e *expr) []string { + seen := map[string]bool{} + rewrite(e, func(v *expr) *expr { + seen[v.name] = true + return nil + }) + out := make([]string, 0, len(seen)) + for n := range seen { + out = append(out, n) + } + sort.Strings(out) + return out } ``` -## Running +## Walking a function -Output gets streamed line by line, and positions are translated on the way -through. A panic becomes a diagnostic on the innermost stack frame that is in -the document. A failing test does too, when the document's own command runs -`go test`. + + + -The program's stdout and stderr both have to be read while it runs, so each -pipe gets a goroutine. Those goroutines don't report anything themselves. They -send every line down one channel, and the loop in `execute` receives the lines -and does the reporting. The same loop waits for the timeout, so the reporter, -the panic bookkeeping and the `timedOut` flag all belong to one goroutine and -none of them needs a lock. +The verifier executes a function symbolically. A parameter starts out as a +symbol that stands for any value at all. An assignment does not change a +value, it gives the variable a new one and keeps the old, which is what SSA +form does with its subscripts, and it is also exactly what the primes of an +annotation need: in a formula that mentions `x'`, the `x'` is the value now +and the `x` is the one before. -```mermaid -flowchart LR - out[read stdout] --> loop[the loop in execute] - err[read stderr] --> loop - timer[timeout] --> loop - loop --> reporter -``` +A path through the function is a `state`: what every variable holds, and the +formulas known to be true on the way here, which are the hypotheses of +whatever has to be proved next. ```go -func fail(r *reporter, err error) int { - r.diag(lit.Diag{Severity: lit.SevError, Message: err.Error(), Source: "litgo"}) - return 1 +// binding is a program variable on one path: its kind, and every value it +// has had since it was declared or since the loop around it was cut. The +// last one is its value now; the ones before are what primes reach for. +type binding struct { + kind vkind + typ string + hist []*expr } -var ( - frameRe = regexp.MustCompile(`^\s+(\S+\.go):(\d+)`) - // reportRe is looser than compilerLineRe: `go test` indents its failures. - reportRe = regexp.MustCompile(`^\s*(?:\./)?[^\s:]+\.go:\d+(?::\d+)?:\s+(.*)$`) -) +// state is one path through a function: what the variables hold, and what is +// known to be true. +type state struct { + vars map[string]*binding + scopes []map[string]*binding // what each open block shadowed + hyps []*expr + preserved []kept + base []*expr // the BaseCase conditions passed so far +} -// outputLine is one line of a program's output, or the end of a stream. -type outputLine struct { - stream, text string - eof bool +type kept struct { + a *annot + depth int } -// pump sends the lines of one of a program's streams, and then its end. -func pump(stream string, rd io.Reader, lines chan<- outputLine) { - sc := bufio.NewScanner(rd) - sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) - for sc.Scan() { - lines <- outputLine{stream: stream, text: sc.Text()} +func (s *state) clone() *state { + c := &state{vars: make(map[string]*binding, len(s.vars)), hyps: s.hyps[:len(s.hyps):len(s.hyps)], + preserved: s.preserved[:len(s.preserved):len(s.preserved)], base: s.base[:len(s.base):len(s.base)]} + for n, b := range s.vars { + c.vars[n] = &binding{b.kind, b.typ, append([]*expr{}, b.hist...)} } - lines <- outputLine{stream: stream, eof: true} + for _, sc := range s.scopes { + m := make(map[string]*binding, len(sc)) + for n, b := range sc { + m[n] = b + } + c.scopes = append(c.scopes, m) + } + return c } +``` -// execute streams a program's output, translating positions as it goes. A -// panic becomes a diagnostic on the innermost frame that lives in the -// literate source. -func execute(r *reporter, m *remapper, cmd *exec.Cmd, dir string, timeout time.Duration, parseTools bool) int { - cmd.Dir = dir - stdout, _ := cmd.StdoutPipe() - stderr, _ := cmd.StderrPipe() - ownGroup(cmd) - start := time.Now() - if err := cmd.Start(); err != nil { - return fail(r, err) - } - defer killOnSignal(cmd)() - var timeUp <-chan time.Time // stays nil, which never delivers, without a timeout - if timeout > 0 { - timer := time.NewTimer(timeout) - defer timer.Stop() - timeUp = timer.C - } +Blocks have scopes, and a variable declared in an inner block can hide one +outside. Each open block remembers what it hid, and puts it back when it +closes. - lines := make(chan outputLine) - go pump("stdout", stdout, lines) - go pump("stderr", stderr, lines) - timedOut := false - var panicMsg string - panicPlaced := false - for open := 2; open > 0; { - select { - case <-timeUp: - timedOut = true - killGroup(cmd) - case l := <-lines: - if l.eof { - open-- - continue - } - raw := l.text - text, pos := m.line(raw) - if l.stream == "stderr" { - switch { - case strings.HasPrefix(raw, "panic: ") || strings.HasPrefix(raw, "fatal error: "): - panicMsg, panicPlaced = raw, false - case panicMsg != "" && !panicPlaced && pos != nil && frameRe.MatchString(raw): - pos.Severity, pos.Message, pos.Source = lit.SevError, panicMsg, "runtime" - r.diag(*pos) - panicPlaced = true - case parseTools && pos != nil && reportRe.MatchString(raw): - msg, _ := m.line(reportRe.FindStringSubmatch(raw)[1]) - pos.Severity, pos.Message, pos.Source = lit.SevError, msg, "run" - r.diag(*pos) - } - } else if parseTools && pos != nil && reportRe.MatchString(raw) { - msg, _ := m.line(reportRe.FindStringSubmatch(raw)[1]) - pos.Severity, pos.Message, pos.Source = lit.SevError, msg, "run" - r.diag(*pos) +```go +func (s *state) push() { s.scopes = append(s.scopes, map[string]*binding{}) } + +func (s *state) pop() { + top := s.scopes[len(s.scopes)-1] + s.scopes = s.scopes[:len(s.scopes)-1] + for n, old := range top { + for f := range s.vars { + if strings.HasPrefix(f, n+".") { + delete(s.vars, f) } - r.output(l.stream, text) + } + if old == nil { + delete(s.vars, n) + } else { + s.vars[n] = old } } - err := cmd.Wait() - code := 0 - if err != nil { - code = 1 - if ee, ok := err.(*exec.ExitError); ok && ee.ExitCode() >= 0 { - code = ee.ExitCode() + for len(s.preserved) > 0 && s.preserved[len(s.preserved)-1].depth > len(s.scopes) { + s.preserved = s.preserved[:len(s.preserved)-1] + } +} + +func (s *state) popTo(depth int) { + for len(s.scopes) > depth { + s.pop() + } +} + +func (s *state) declare(name string, b *binding) { + if name == "_" { + return + } + if top := s.scopes[len(s.scopes)-1]; top != nil { + if _, ok := top[name]; !ok { + top[name] = s.vars[name] } } - ms := time.Since(start).Milliseconds() - if timedOut { - r.output("stderr", fmt.Sprintf("litgo: killed after %s", timeout)) + for f := range s.vars { + if strings.HasPrefix(f, name+".") { + delete(s.vars, f) + } } - r.emit("exit", map[string]any{"code": code, "ms": ms, "ran": true, "timed_out": timedOut}) - r.status("exit %d (%d ms)", code, ms) - return code + s.vars[name] = b } ``` -The program runs in its own process group. Stopping litgo stops the program and -anything it started, and the timeout does the same. - - +An `if` splits a path in two, and both go on. So a statement does not leave one +state behind but several, sorted by how they left: by falling out of the +bottom, by `break`, or by `continue`. A `return` leaves none, because +everything that has to hold at a return is proved right there. ```go -//go:build unix +// outcomes are the states a statement leaves behind, by how it was left. +// A return leaves none: its obligations are settled on the spot. +type outcomes struct { + normal, brk, cont []*state +} -package main +func (o *outcomes) add(p outcomes) { + o.normal = append(o.normal, p.normal...) + o.brk = append(o.brk, p.brk...) + o.cont = append(o.cont, p.cont...) +} +``` -import ( - "os" - "os/exec" - "os/signal" - "syscall" -) +```go +// run verifies one function. +type run struct { + v *verifier + f *fn + entry map[string]*expr // the parameters' values on entry + blocks map[*ast.BlockStmt][]*annot + measure0 []*expr + paths int + depth int // of one-line functions being unfolded +} -// ownGroup puts the child in its own process group so that it, and anything -// it spawns, can be killed as one. -func ownGroup(cmd *exec.Cmd) { - cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} +// unsupported is how the verifier backs out of a function it cannot follow. +type unsupported struct { + pos token.Pos + msg string } -func killGroup(cmd *exec.Cmd) { - if cmd.Process != nil { - syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) - } +func (r *run) no(n ast.Node, format string, args ...any) { + panic(unsupported{n.Pos(), fmt.Sprintf(format, args...)}) } -// killOnSignal makes stopping litgo (the editor's :LitStop) stop the program -// it is running. The returned function ends the watch. -func killOnSignal(cmd *exec.Cmd) func() { - sigs := make(chan os.Signal, 1) - done := make(chan struct{}) - signal.Notify(sigs, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP) - go func() { - select { - case <-sigs: - killGroup(cmd) - case <-done: +const maxPaths = 400 +``` + +Verifying a function starts with its parameters as unknowns and its +precondition as the first hypothesis. Go gives named results their zero values, +so the proof knows them too. + +```go +func (v *verifier) verify(f *fn) { + r := &run{v: v, f: f, entry: map[string]*expr{}, blocks: map[*ast.BlockStmt][]*annot{}} + defer func() { + if p := recover(); p != nil { + u, ok := p.(unsupported) + if !ok { + // A bug in the verifier. The editor is on the other end of this, + // so say so at the function instead of taking the server down. + u = unsupported{f.decl.Name.Pos(), fmt.Sprintf("the verifier failed (%v); please report this", p)} + } + at := v.position(u.pos) + v.report(at, 0, false, fmt.Sprintf("%s is not verified: %s", f.key, u.msg)) } }() - return func() { - signal.Stop(sigs) - close(done) + r.place() + st := &state{vars: map[string]*binding{}} + st.push() + all := append([]param{}, f.params...) + if f.recv != nil { + all = append(all, *f.recv) + } + for _, p := range all { + if p.kind == kNone && p.name != "_" { + // Whatever it is, the proof cannot mention it. + continue + } + b := &binding{kind: p.kind, typ: p.typ} + if p.kind != kStruct { + b.hist = []*expr{r.fresh(p.name, p.kind)} + r.entry[p.name] = b.hist[0] + } + st.declare(p.name, b) + } + for _, p := range f.results { + if p.kind != kInt && p.kind != kBool { + r.no(f.decl, "its result %s is neither an integer nor a boolean", p.name) + } + zero := num(0) + if p.kind == kBool { + zero = eFalse + } + st.declare(p.name, &binding{kind: p.kind, hist: []*expr{zero}}) + } + for _, a := range f.requires { + if a.e != nil { + st.hyps = append(st.hyps, r.formula(st, a, false)) + } + } + for _, a := range f.measure { + if a.e != nil { + r.measure0 = append(r.measure0, r.formula(st, a, false)) + } + } + r.lint() + if r.recursive() && len(f.measure) == 0 { + v.report(v.position(f.decl.Name.Pos()), 0, true, f.key+" calls itself and has no Measure, so nothing shows that it terminates") + } + out := r.block(st, f.decl.Body) + for _, s := range out.normal { + r.leave(s, f.decl.Body.Rbrace) } } ``` - - ```go -//go:build !unix - -package main +// place sorts the annotations of the body into the blocks they are written in. +func (r *run) place() { + for _, a := range r.f.inside { + var inner *ast.BlockStmt + ast.Inspect(r.f.decl.Body, func(n ast.Node) bool { + if b, ok := n.(*ast.BlockStmt); ok && a.pos > b.Lbrace && a.pos < b.Rbrace { + inner = b + } + return true + }) + if inner != nil { + r.blocks[inner] = append(r.blocks[inner], a) + } + } +} -import "os/exec" +// lint points at contracts that say nothing. +func (r *run) lint() { + for _, list := range [][]*annot{r.f.requires, r.f.ensures} { + for _, a := range list { + if a.e != nil && a.e.op == opTrue { + r.v.report(a.where, a.endCol, true, a.kind+" true says nothing") + } + } + } +} -func ownGroup(cmd *exec.Cmd) {} +func (r *run) recursive() bool { + found := false + ast.Inspect(r.f.decl.Body, func(n ast.Node) bool { + if c, ok := n.(*ast.CallExpr); ok { + if callee, _ := r.callee(nil, c); callee == r.f { + found = true + } + } + return true + }) + return found +} -func killGroup(cmd *exec.Cmd) { - if cmd.Process != nil { - cmd.Process.Kill() +func (r *run) fresh(name string, k vkind) *expr { + r.v.fresh++ + s := sInt + if k == kBool { + s = sBool + } else if k == kSlice { + s = sArr } + return sym(fmt.Sprintf("%s#%d", name, r.v.fresh), s) } - -func killOnSignal(cmd *exec.Cmd) func() { return func() {} } ``` -# Errors as you type +### Obligations -`litgo run` finds every error, but only when you ask, and only after it writes -files and starts a compiler. Most errors can be found sooner than that. The Go -type checker is in the standard library. If you give it the tangled files *in -memory*, it reports what the compiler would report (an undefined name, a wrong -argument, an unused variable, an unused import) in a few milliseconds, and the -source map puts each report on the right line of the Markdown. A language -server runs that check after every change, so the errors show up under the code -that caused them before you have even thought about running anything. +Everything the verifier proves goes through `check`: the hypotheses of the path +on one side, a goal on the other. A failure is reported at the annotation that +was not proved, and mentions the place in the code it was not proved for. An +obligation that comes from the code itself, like an index, is reported at the +code. -```mermaid -sequenceDiagram - participant E as editor - participant S as litgo lsp - participant T as go/types - E->>S: didChange: the whole buffer - S-->>E: chunk and syntax errors, at once - S->>T: the tangled files, never written - T-->>S: pool.go:31:14: undefined: jobb - S-->>E: pool.lit.md:67:15: undefined: jobb -``` +```go +// check is one proof obligation. It reports at the annotation, or at a +// place in the code, and mentions a second place if there is one. +func (r *run) check(st *state, goal *expr, a *annot, at ast.Node, ref ast.Node, msg string) bool { + r.f.obligations++ + if proves(st.hyps, goal) { + return true + } + where, end := Pos{}, 0 + if a != nil { + where, end = a.where, a.endCol + } else { + where = r.v.position(at.Pos()) + end = where.Col + int(at.End()-at.Pos()) + } + var rp Pos + if ref != nil { + rp = r.v.position(ref.Pos()) + } + r.v.reportRef(where, end, false, msg, rp) + return false +} -## Type-checking what is not on disk +func (r *run) assume(st *state, f *expr) { st.hyps = append(st.hyps, f) } - - - +// feasible reports whether a path can be taken at all. Pruning the ones that +// cannot keeps their obligations from being reported as failures of logic. +func (r *run) feasible(st *state) bool { return !proves(st.hyps, eFalse) } +``` -A check works on packages. A package's files are the document's outputs in one -directory. When the document writes a module or has its own `run:` command, the -other Go files already in that directory are included too. A document that is a -single program gets checked by itself, the same way `litgo run` compiles it by -itself. This matters because a directory of examples is full of `main` -functions that can't be in the same package. +### What a formula means at a point in the program -```go -// pkg is one package about to be checked. -type pkg struct { - dir, name, path string - files []*ast.File // what an importer of the package sees - tests []*ast.File // _test.go files of the same package - types *types.Package - checking bool -} +A formula is written in the program's names, and what it says depends on where +it stands. Here the names get their values. With primes, the most-primed +mention of a variable is its value now, and each prime fewer is one assignment +earlier, so after `x = x + 1` the formula `x' = x + 1` says what the assignment +did. A postcondition is different: there a parameter without a prime is the +value the caller passed, whatever the body did to its copy. -type checker struct { - res *lit.Result - fset *token.FileSet - outputs map[string]*lit.Output // by path - pkgs []*pkg - byPath map[string]*pkg // packages this document writes, by import path - exports map[string]string // import path to export data file, for all others - gc types.Importer - goVersion string - seen map[string]bool - diags []lit.Diag +```go +// formula gives an annotation's formula its meaning in a state: predicates +// and one-line functions unfolded, names replaced by values. In a +// postcondition a parameter means its value on entry, and primed, its value +// at the end. +func (r *run) formula(st *state, a *annot, post bool) *expr { + e := r.unfold(st, a, a.e, 0) + most := map[string]int{} + var count func(e *expr) + count = func(e *expr) { + if e == nil { + return + } + if e.op == opVar { + most[e.name] = max(most[e.name], e.primes) + } + for _, x := range e.args { + count(x) + } + } + count(e) + return rewrite(e, func(v *expr) *expr { + if strings.ContainsAny(v.name, "#?") { + return nil + } + if old, ok := r.entry[v.name]; ok && post { + if v.primes == 0 { + return old + } + return r.lookup(st, a, v.name, 0) + } + return r.lookup(st, a, v.name, most[v.name]-v.primes) + }) } -// Cache carries what is worth keeping between checks: where the compiled -// standard library lives. It has no lock, so whoever owns a Cache runs one -// check at a time with it. -type Cache struct { - std map[string]string +// lookup finds the value a name had `back` assignments ago. +func (r *run) lookup(st *state, a *annot, name string, back int) *expr { + b := r.binding(st, name) + if b == nil { + if c, ok := r.v.consts[name]; ok { + return num(c) + } + r.bad(a, "%s is not a variable here", name) + } + if b.kind == kStruct || len(b.hist) == 0 { + r.bad(a, "%s is not an integer, a boolean or a slice of integers", name) + } + if back >= len(b.hist) { + r.bad(a, "%s has not been assigned often enough on this path for that many primes", name) + } + return b.hist[len(b.hist)-1-back] } +``` -// Types type-checks the Go files a document tangles to, without writing -// them, and reports errors at their positions in the document. It reports -// nothing if the files do not parse (the syntax check covers that) or if -// there is no go command to ask about imports. -func Types(res *lit.Result, cache *Cache) []lit.Diag { - if res.HasErrors() { +```go +// binding finds a variable, or makes one for the field of a struct the +// first time it is mentioned. Nothing in the fragment assigns to a field, so +// a field holds one value throughout. +func (r *run) binding(st *state, name string) *binding { + if b := st.vars[name]; b != nil { + return b + } + base, field, ok := strings.Cut(name, ".") + if !ok { return nil } - c := &checker{res: res, fset: token.NewFileSet(), outputs: map[string]*lit.Output{}, - byPath: map[string]*pkg{}, exports: map[string]string{}, seen: map[string]bool{}} - if !c.parse() { + s := st.vars[base] + if s == nil || s.kind != kStruct { return nil } - c.locate() - if !c.resolve(cache) { + var k vkind + switch r.v.structs[s.typ][field] { + case "int", "int8", "int16", "int32", "int64": + k = kInt + case "bool": + k = kBool + case "[]int": + k = kSlice + default: return nil } - c.gc = importer.ForCompiler(c.fset, "gc", c.lookup) - for _, p := range c.pkgs { - c.check(p, true) + b := &binding{kind: k, hist: []*expr{r.fresh(name, k)}} + st.vars[name] = b + return b +} + +// bad gives up on an annotation that does not make sense. +func (r *run) bad(a *annot, format string, args ...any) { + msg := fmt.Sprintf(format, args...) + if a == nil { + panic(unsupported{r.f.decl.Pos(), msg}) } - return c.diags + r.v.report(a.where, a.endCol, false, "in this "+a.kind+": "+msg) + panic(unsupported{r.f.decl.Name.Pos(), "one of its annotations does not make sense"}) } ``` -Build constraints matter here. litgo itself has two files that define the same -functions for different operating systems. `go/build` decides which files -belong, and it gets the in-memory content through its `OpenFile` hook. +A predicate is unfolded into its body. So is a call of a Go function whose +whole body is `return` and one expression, which makes small Go functions +usable as vocabulary: a contract can say `s.src(c) = col` and mean it about +the real `src`. A function with more to it than that cannot be used this way, +because a formula needs a value and such a function might not even return. ```go -// parse sorts the files into packages. It fails if an output does not parse. -func (c *checker) parse() bool { - doc := c.res.Doc - for _, o := range c.res.Files { - if o.File.Lang == "go" { - c.outputs[o.Path] = o - } +// unfold replaces P(x, y) by what it stands for: the body of a predicate, or +// the expression a one-line Go function returns. +func (r *run) unfold(st *state, a *annot, e *expr, depth int) *expr { + if e == nil { + return nil } - ctx := build.Default - ctx.OpenFile = func(path string) (io.ReadCloser, error) { - if o := c.outputs[path]; o != nil { - return io.NopCloser(bytes.NewReader(o.Bytes())), nil - } - return os.Open(path) + if depth > 12 { + r.bad(a, "%s is defined in terms of itself", e.name) } - groups := map[string]*pkg{} - add := func(path string, src any) error { - dir, base := filepath.Dir(path), filepath.Base(path) - if ok, _ := ctx.MatchFile(dir, base); !ok { - return nil - } - f, err := parser.ParseFile(c.fset, path, src, parser.SkipObjectResolution) - if err != nil { - return err + c := *e + c.args = make([]*expr, len(e.args)) + for i, x := range e.args { + c.args[i] = r.unfold(st, a, x, depth) + } + if e.op != opCall { + return &c + } + if p := r.v.preds[e.name]; p != nil { + if len(p.params) != len(c.args) { + r.bad(a, "%s takes %d arguments", p.name, len(p.params)) } - test := strings.HasSuffix(base, "_test.go") - name := f.Name.Name - p := groups[dir+"\x00"+name] - if p == nil { - p = &pkg{dir: dir, name: name} - groups[dir+"\x00"+name] = p - c.pkgs = append(c.pkgs, p) + return r.unfold(st, a, replace(p.body, p.params, c.args, "", ""), depth+1) + } + callee, recv := r.named(st, e.name) + if callee == nil { + r.bad(a, "%s is neither a predicate nor a function of this package", e.name) + } + body := r.oneLine(callee) + if body == nil { + r.bad(a, "%s cannot be used in a formula: only a function whose body is one return of one expression can", e.name) + } + if len(callee.params) != len(c.args) { + r.bad(a, "%s takes %d arguments", e.name, len(callee.params)) + } + var formals []string + for _, p := range callee.params { + formals = append(formals, p.name) + } + from := "" + if callee.recv != nil { + from = callee.recv.name + } + return r.unfold(st, a, replace(body, formals, c.args, from, recv), depth+1) +} + +// replace substitutes all the parameters at once, and renames the fields of +// the callee's receiver to fields of the caller's. +func replace(body *expr, formals []string, actuals []*expr, from, to string) *expr { + return rewrite(body, func(v *expr) *expr { + for i, f := range formals { + if v.name == f && v.primes == 0 { + return actuals[i] + } } - if test && !strings.HasSuffix(name, "_test") { - p.tests = append(p.tests, f) - } else { - p.files = append(p.files, f) + if from != "" && strings.HasPrefix(v.name, from+".") { + c := *v + c.name = to + strings.TrimPrefix(v.name, from) + return &c } return nil - } + }) +} - dirs := map[string]bool{} - for path, o := range c.outputs { - if add(path, o.Bytes()) != nil { - return false +// named finds the function a formula or a call mentions: f, or x.m where x +// is a struct. +func (r *run) named(st *state, name string) (*fn, string) { + if f := r.v.funcs[name]; f != nil && f.recv == nil { + return f, "" + } + if i := strings.LastIndex(name, "."); i > 0 && st != nil { + if b := st.vars[name[:i]]; b != nil && b.kind == kStruct { + return r.v.funcs[b.typ+name[i:]], name[:i] } - dirs[filepath.Dir(path)] = true } - if doc.Run != "" || len(doc.Files) > 1 { - // <> + return nil, "" +} + +// oneLine is the expression a function returns, if that is all it does. +func (r *run) oneLine(f *fn) *expr { + if len(f.decl.Body.List) != 1 || len(f.results) != 1 || len(f.inside) > 0 { + return nil } - sort.Slice(c.pkgs, func(i, j int) bool { - return c.pkgs[i].dir+c.pkgs[i].name < c.pkgs[j].dir+c.pkgs[j].name - }) - return true + ret, ok := f.decl.Body.List[0].(*ast.ReturnStmt) + if !ok || len(ret.Results) != 1 { + return nil + } + kinds := map[string]vkind{} + for _, p := range f.params { + kinds[p.name] = p.kind + } + var e *expr + func() { + defer func() { + if p := recover(); p != nil { + if _, ok := p.(unsupported); !ok { + panic(p) + } + e = nil + } + }() + e = r.pure(ret.Results[0], func(name string) vkind { + if k, ok := kinds[name]; ok { + return k + } + if f.recv != nil && strings.HasPrefix(name, f.recv.name+".") { + if r.v.structs[f.recv.typ][strings.TrimPrefix(name, f.recv.name+".")] == "bool" { + return kBool + } + } + return kInt + }) + }() + return e } ``` -If a hand-written file doesn't parse, that is its author's problem and not this -document's, so the file gets left out. +### Go expressions + +A Go expression is read in two steps. `pure` translates the syntax into a term +over the program's names, and remembers for every part where it came from. +`eval` then gives the names their values and, on the way, states what has to be +true for the expression to be evaluated at all. - ```go -for dir := range dirs { - entries, _ := os.ReadDir(dir) - for _, e := range entries { - path := filepath.Join(dir, e.Name()) - if e.IsDir() || !strings.HasSuffix(path, ".go") || c.outputs[path] != nil { - continue +// pure reads a Go expression as a term over the names it mentions. It +// checks nothing yet; pos remembers where each part came from. +func (r *run) pure(e ast.Expr, kind func(string) vkind) *expr { + at := func(x *expr) *expr { x.pos = int(e.Pos()); return x } + switch e := e.(type) { + case *ast.ParenExpr: + return r.pure(e.X, kind) + case *ast.BasicLit: + if c, ok := constInt(e); ok { + return at(num(c)) + } + case *ast.Ident: + switch e.Name { + case "true": + return eTrue + case "false": + return eFalse + } + s := sInt + if kind(e.Name) == kBool { + s = sBool + } else if kind(e.Name) == kSlice { + s = sArr + } + return at(sym(e.Name, s)) + case *ast.SelectorExpr: + if x, ok := e.X.(*ast.Ident); ok { + name := x.Name + "." + e.Sel.Name + s := sInt + if kind(name) == kBool { + s = sBool + } else if kind(name) == kSlice { + s = sArr + } + return at(sym(name, s)) + } + case *ast.UnaryExpr: + x := r.pure(e.X, kind) + switch e.Op { + case token.SUB: + return at(mk(opNeg, x)) + case token.ADD: + return x + case token.NOT: + return at(mk(opNot, x)) + } + case *ast.BinaryExpr: + l, rt := r.pure(e.X, kind), r.pure(e.Y, kind) + switch e.Op { + case token.ADD: + return at(mk(opAdd, l, rt)) + case token.SUB: + return at(mk(opSub, l, rt)) + case token.MUL: + return at(mk(opMul, l, rt)) + case token.QUO: + return at(mk(opDiv, l, rt)) + case token.REM: + return at(mk(opMod, l, rt)) + case token.LAND: + return at(mk(opAnd, l, rt)) + case token.LOR: + return at(mk(opOr, l, rt)) + case token.LSS: + return at(mk(opLt, l, rt)) + case token.LEQ: + return at(mk(opLe, l, rt)) + case token.GTR: + return at(mk(opLt, rt, l)) + case token.GEQ: + return at(mk(opLe, rt, l)) + case token.EQL, token.NEQ: + var c *expr + if isBool(l) || isBool(rt) { + c = mk(opIff, l, rt) + } else { + c = mk(opEq, l, rt) + } + if e.Op == token.NEQ { + c = mk(opNot, c) + } + return at(c) + } + case *ast.IndexExpr: + a := r.pure(e.X, kind) + if a.op == opVar { + a.sort = sArr + return at(mk(opSel, a, r.pure(e.Index, kind))) + } + case *ast.CallExpr: + name := "" + switch f := e.Fun.(type) { + case *ast.Ident: + name = f.Name + case *ast.SelectorExpr: + if x, ok := f.X.(*ast.Ident); ok { + name = x.Name + "." + f.Sel.Name + } } - add(path, nil) + if name == "" { + break + } + var args []*expr + for _, x := range e.Args { + args = append(args, r.pure(x, kind)) + } + if name == "len" && len(args) == 1 && args[0].op == opVar { + args[0].sort = sArr + return at(mk(opLen, args[0])) + } + if name == "int" && len(args) == 1 { + return args[0] + } + c := at(&expr{op: opCall, name: name, args: args}) + return c } + r.no(e, "this expression is outside what litgo prove understands (integers, booleans, slices of integers, fields of structs, calls of verified functions)") + return nil } ``` -The packages a document writes can import each other. For example, litgo's -`main` imports its `internal/lit`. Those imports have to resolve to the version -in the buffer and not to whatever was last tangled, so each package needs its -import path, which is the module path plus the directory. The `go.mod` can be -one of the outputs too. +These are the safety obligations: an index is within its slice, a divisor is +not zero, a callee's precondition holds. Go evaluates `&&` and `||` from the +left and stops early, so the right side of `i < len(A) && A[i] > 0` is only +obliged to be safe when the left side is true. The guards carry that. ```go -// locate works out the import path of every package. -func (c *checker) locate() { - modDir, mod := "", "" - for _, o := range c.res.Files { - if filepath.Base(o.Path) == "go.mod" { - modDir, mod = filepath.Dir(o.Path), string(o.Bytes()) - } +// value evaluates a Go expression in a state, and states what has to be +// true for the evaluation to be safe: indices in range, divisors not zero, +// the preconditions of what is called. +func (r *run) value(st *state, e ast.Expr) *expr { + p := r.pure(e, func(name string) vkind { + if b := r.binding(st, name); b != nil { + return b.kind + } + return kNone + }) + vals := r.eval(st, p, nil, e) + if len(vals) != 1 { + r.no(e, "a call with %d results cannot be used as a value", len(vals)) } - for dir := filepath.Dir(c.res.Doc.Path); mod == ""; dir = filepath.Dir(dir) { - if b, err := os.ReadFile(filepath.Join(dir, "go.mod")); err == nil { - modDir, mod = dir, string(b) - } - if dir == filepath.Dir(dir) { - break + return vals[0] +} + +// eval does the work of value. The guards are what is known at this point +// of the expression and not before: the left side of an && while its right +// side is being evaluated. +func (r *run) eval(st *state, e *expr, guards []*expr, at ast.Node) []*expr { + one := func(x *expr) *expr { + vals := r.eval(st, x, guards, at) + if len(vals) != 1 { + r.no(at, "a call with %d results cannot be used as a value", len(vals)) } + return vals[0] } - module := "" - for _, line := range strings.Split(mod, "\n") { - if f := strings.Fields(line); len(f) == 2 && f[0] == "module" { - module = strings.Trim(f[1], `"`) - } else if len(f) == 2 && f[0] == "go" { - c.goVersion = "go" + f[1] + guarded := func(goal *expr) *expr { + if len(guards) == 0 { + return goal } + return mk(opImp, conj(guards), goal) } - for _, p := range c.pkgs { - p.path = p.name - if rel, err := filepath.Rel(modDir, p.dir); module != "" && err == nil && !strings.HasPrefix(rel, "..") { - p.path = strings.TrimSuffix(module+"/"+filepath.ToSlash(rel), "/.") - if strings.HasSuffix(p.name, "_test") { - p.path += "_test" + here := ast.Node(at) + if e.pos != 0 { + here = point(e.pos) + } + switch e.op { + case opNum, opTrue, opFalse: + return []*expr{e} + case opVar: + if strings.ContainsAny(e.name, "#?") { + return []*expr{e} + } + b := r.binding(st, e.name) + if b == nil { + if c, ok := r.v.consts[e.name]; ok { + return []*expr{num(c)} } + r.no(here, "%s is not a variable the proof can talk about", e.name) + } + if len(b.hist) == 0 { + r.no(here, "%s is a struct, and only its fields have values", e.name) + } + return []*expr{b.hist[len(b.hist)-1]} + case opAnd, opOr: + l := one(e.args[0]) + g := l + if e.op == opOr { + g = not(l) + } + rt := r.eval(st, e.args[1], append(guards[:len(guards):len(guards)], g), at) + return []*expr{mk(e.op, l, rt[0])} + case opSel: + arr, idx := one(e.args[0]), one(e.args[1]) + in := mk(opAnd, mk(opLe, num(0), idx), mk(opLt, idx, mk(opLen, arr))) + r.check(st, guarded(in), nil, here, nil, "the index may be out of range: 0 <= "+e.args[1].String()+" < len("+e.args[0].String()+") is not proved") + r.assume(st, guarded(in)) + return []*expr{mk(opSel, arr, idx)} + case opDiv, opMod: + l, d := one(e.args[0]), one(e.args[1]) + r.check(st, guarded(mk(opNe, d, num(0))), nil, here, nil, "the divisor may be zero: "+e.args[1].String()+" <> 0 is not proved") + r.assume(st, guarded(mk(opNe, d, num(0)))) + return []*expr{mk(e.op, l, d)} + case opCall: + var args []*expr + for _, x := range e.args { + args = append(args, one(x)) + } + return r.call(st, e, args, guards, here) + } + c := *e + c.args = make([]*expr, len(e.args)) + for i, x := range e.args { + c.args[i] = one(x) + } + return []*expr{&c} +} +``` + +```go +// point is a position that stands in for a node. +type point token.Pos + +func (p point) Pos() token.Pos { return token.Pos(p) } +func (p point) End() token.Pos { return token.Pos(p) + 1 } + +// callee finds the declared function a call expression calls. +func (r *run) callee(st *state, c *ast.CallExpr) (*fn, string) { + switch f := c.Fun.(type) { + case *ast.Ident: + if fn := r.v.funcs[f.Name]; fn != nil && fn.recv == nil { + return fn, "" } - if !strings.HasSuffix(p.name, "_test") && p.name != "main" { - c.byPath[p.path] = p + case *ast.SelectorExpr: + x, ok := f.X.(*ast.Ident) + if !ok { + return nil, "" + } + if st == nil { // asked only whether the function calls itself + if r.f.recv != nil && x.Name == r.f.recv.name && r.f.key == r.f.recv.typ+"."+f.Sel.Name { + return r.f, x.Name + } + return nil, "" } + return r.named(st, x.Name+"."+f.Sel.Name) } + return nil, "" } ``` -Everything else gets imported the way `go vet` does it, from the export data -the compiler left in the build cache. `go list -export` finds that data, and -compiles first if it has to. This is exact, and it is fast. Reading the export -data of `fmt` takes a fraction of the time it would take to type-check its -source. One `go list` call covers all the imports of a check. The standard -library gets remembered for the life of the process, because it doesn't change -while an editor is running. +A call is where verification is modular. The caller does not look inside the +callee. It proves the callee's `Requires`, gets fresh unknowns for the results, +and learns the callee's `Ensures` about them and nothing else. That is also +what makes recursion work: inside `Triangle`, the call `Triangle(n-1)` is +known by the very contract being proved, which is the induction hypothesis, +and the `Measure` is what makes the induction well-founded. ```go -// resolve finds the export data of every package that is imported but not -// written by this document. It fails if the go command cannot be run. -func (c *checker) resolve(cache *Cache) bool { - if cache.std == nil { - cache.std = map[string]string{} +// call uses a function by its contract: the precondition is an obligation, +// the postcondition is what is known about the results. A function that is +// one expression is used as that expression, which says more. +func (r *run) call(st *state, e *expr, args []*expr, guards []*expr, at ast.Node) []*expr { + switch e.name { + case "min", "max": + if len(args) != 2 || r.v.funcs[e.name] != nil { + break + } + m := r.fresh(e.name, kInt) + lo, hi := m, args[0] + lo2, hi2 := m, args[1] + if e.name == "max" { + lo, hi, lo2, hi2 = args[0], m, args[1], m + } + r.assume(st, conj([]*expr{mk(opLe, lo, hi), mk(opLe, lo2, hi2), mk(opOr, mk(opEq, m, args[0]), mk(opEq, m, args[1]))})) + return []*expr{m} } - var missing []string - for _, p := range c.pkgs { - for _, f := range append(append([]*ast.File{}, p.files...), p.tests...) { - for _, imp := range f.Imports { - path := strings.Trim(imp.Path.Value, "\"`") - if c.byPath[path] != nil || path == "C" || path == "unsafe" || c.seen[path] { - continue - } - c.seen[path] = true - if file, ok := cache.std[path]; ok && exists(file) { - c.exports[path] = file - } else { - missing = append(missing, path) + callee, recv := r.named(st, e.name) + if callee == nil { + r.no(at, "%s has no contract the proof could use; only functions of this package can be called", e.name) + } + if len(args) != len(callee.params) { + r.no(at, "%s takes %d arguments", e.name, len(callee.params)) + } + if !callee.annotated && r.oneLine(callee) == nil { + // Nobody has checked what it does to the slices and structs it can reach. + r.no(at, "%s has no annotations, so nothing is known about what it does; give it a contract, even an empty one like Requires true", e.name) + } + guarded := func(goal *expr) *expr { + if len(guards) == 0 { + return goal + } + return mk(opImp, conj(guards), goal) + } + // The callee's formulas, in the caller's terms. + inst := func(a *annot, results []*expr) *expr { + if a.e == nil { + return eTrue + } + sub := &state{vars: map[string]*binding{}} + sub.push() + for i, p := range callee.params { + if p.kind == kNone { + continue + } + sub.vars[p.name] = &binding{kind: p.kind, typ: p.typ, hist: []*expr{args[i]}} + if results != nil && p.kind != kSlice { + // Primed, a parameter is the callee's own copy when it returned, + // and the caller knows nothing about that. + sub.vars[p.name].hist = []*expr{r.fresh(p.name, p.kind)} + } + } + if callee.recv != nil && callee.recv.kind == kStruct { + sub.vars[callee.recv.name] = &binding{kind: kStruct, typ: callee.recv.typ} + for field := range r.v.structs[callee.recv.typ] { + if b := r.binding(st, recv+"."+field); b != nil { + sub.vars[callee.recv.name+"."+field] = b } } } + for i, p := range callee.results { + if results != nil { + sub.vars[p.name] = &binding{kind: p.kind, hist: []*expr{results[i]}} + } + } + inner := &run{v: r.v, f: callee, entry: map[string]*expr{}} + for i, p := range callee.params { + if p.kind != kNone && p.kind != kStruct { + inner.entry[p.name] = args[i] + } + } + defer func() { + // What goes wrong in the callee's annotation is the callee's problem. + if p := recover(); p != nil { + if _, ok := p.(unsupported); !ok { + panic(p) + } + r.no(at, "the contract of %s cannot be used here", e.name) + } + }() + return inner.formula(sub, a, results != nil) } - c.seen = map[string]bool{} - if len(missing) == 0 { - return true + for _, a := range callee.requires { + r.check(st, guarded(inst(a, nil)), nil, at, nil, fmt.Sprintf("this call does not establish what %s requires: %s", callee.key, a.text)) } - args := append([]string{"list", "-e", "-export", "-deps", "-f", "{{if .Export}}{{.ImportPath}}\t{{.Export}}{{end}}", "--"}, missing...) - cmd := exec.Command("go", args...) - cmd.Dir = filepath.Dir(c.res.Doc.Path) - out, err := cmd.Output() - if _, broken := err.(*exec.Error); broken { - return false + if callee == r.f { + r.descends(st, e, args, guards, at) } - for _, line := range strings.Split(string(out), "\n") { - if path, file, ok := strings.Cut(line, "\t"); ok { - c.exports[path] = file - if !strings.Contains(strings.Split(path, "/")[0], ".") { - cache.std[path] = file - } + if body := r.oneLine(callee); body != nil { + if r.depth++; r.depth > 40 { + r.no(at, "%s never stops calling itself", callee.key) + } + defer func() { r.depth-- }() + var formals []string + for _, p := range callee.params { + formals = append(formals, p.name) } + from := "" + if callee.recv != nil { + from = callee.recv.name + } + return r.eval(st, replace(body, formals, args, from, recv), guards, at) } - return true -} - -func exists(file string) bool { - _, err := os.Stat(file) - return err == nil + var results []*expr + for _, p := range callee.results { + results = append(results, r.fresh(callee.decl.Name.Name+"."+p.name, p.kind)) + } + for _, a := range callee.ensures { + r.assume(st, guarded(inst(a, results))) + } + for _, a := range callee.exsures { + r.assume(st, guarded(not(inst(a, results)))) + } + return results } +``` -func (c *checker) lookup(path string) (io.ReadCloser, error) { - if file := c.exports[path]; file != "" { - return os.Open(file) +```go +// descends checks a recursive call: the measure is smaller than it was on +// entry and was not negative, and no base case has been passed. +func (r *run) descends(st *state, e *expr, args []*expr, guards []*expr, at ast.Node) { + guarded := func(goal *expr) *expr { + if len(guards) == 0 { + return goal + } + return mk(opImp, conj(guards), goal) + } + for i, a := range r.f.measure { + if a.e == nil { + continue + } + sub := &state{vars: map[string]*binding{}} + for j, p := range r.f.params { + if p.kind != kNone && p.kind != kStruct { + sub.vars[p.name] = &binding{kind: p.kind, hist: []*expr{args[j]}} + } + } + m := (&run{v: r.v, f: r.f, entry: map[string]*expr{}}).formula(sub, a, false) + r.check(st, guarded(mk(opLt, m, r.measure0[i])), a, nil, at, "Measure is not shown to get smaller in the recursive call at {ref}") + r.check(st, guarded(mk(opLe, num(0), r.measure0[i])), a, nil, at, "Measure can be negative when the recursive call at {ref} is made") + } + for _, b := range st.base { + r.check(st, guarded(not(b)), nil, at, nil, "this recursive call can be reached in the base case") } - return nil, os.ErrNotExist } ``` -The checker acts as its own importer. A package from this document gets checked -on demand, once, without its tests. Anything else comes from the export data. +### Statements + +A block is its statements, with the annotations that stand between them taken +in order. `Invariant` and `Variant` are the exception: they are collected by +the loop they belong to. ```go -// Import implements types.Importer. -func (c *checker) Import(path string) (*types.Package, error) { - if p := c.byPath[path]; p != nil { - return c.check(p, false) +// block runs the statements of a block, with the annotations between them. +func (r *run) block(st *state, b *ast.BlockStmt) outcomes { + st.push() + depth := len(st.scopes) + var out outcomes + live := []*state{st} + notes := r.blocks[b] + n := 0 + for i := 0; i <= len(b.List); i++ { + var next ast.Stmt + limit := b.Rbrace + if i < len(b.List) { + next = b.List[i] + limit = next.Pos() + } + var pending []*annot + for n < len(notes) && notes[n].pos < limit { + a := notes[n] + n++ + if a.kind == "Invariant" || a.kind == "Variant" { + if isLoop(next) { + pending = append(pending, a) + } + continue // at the end of a loop's body it belongs to that loop + } + for _, s := range live { + r.note(s, a) + } + } + if next == nil { + break + } + var after []*state + for _, s := range live { + o := r.stmt(s, next, pending) + after = append(after, o.normal...) + out.brk = append(out.brk, o.brk...) + out.cont = append(out.cont, o.cont...) + } + live = after + if r.paths += len(live); r.paths > maxPaths*50 || len(live) > maxPaths { + r.no(next, "there are too many paths through it") + } } - return c.gc.Import(path) + for _, s := range live { + s.popTo(depth - 1) + } + out.normal = live + return out } -// check type-checks a package: as a root, with its tests and with its -// errors reported, or as the import of another package, silently. -func (c *checker) check(p *pkg, root bool) (*types.Package, error) { - files := p.files - if root { - files = append(append([]*ast.File{}, p.files...), p.tests...) - } else if p.types != nil { - return p.types, nil - } else if p.checking { - return nil, os.ErrInvalid // an import cycle; the root check of it says so +func isLoop(s ast.Stmt) bool { + switch s.(type) { + case *ast.ForStmt, *ast.RangeStmt: + return true } - conf := types.Config{Importer: c, GoVersion: c.goVersion, FakeImportC: true, Error: func(err error) { - if root { - c.report(err) - } - }} - p.checking = true - tp, _ := conf.Check(p.path, c.fset, files, nil) - p.checking = false - if !root { - p.types = tp + return false +} +``` + +```go +// note handles an annotation that stands where a statement could. +func (r *run) note(st *state, a *annot) { + a.used = true + if a.e == nil { + return + } + switch a.kind { + case "Assert", "InductionHypothesis": + f := r.formula(st, a, false) + r.check(st, f, a, nil, nil, a.kind+" is not proved: "+a.text) + r.assume(st, f) + case "Axiom": + r.v.report(a.where, a.endCol, true, "Axiom is taken on trust, not proved: "+a.text) + r.assume(st, r.formula(st, a, false)) + case "Preserves": + f := r.formula(st, a, false) + r.check(st, f, a, nil, nil, "Preserves does not hold to begin with: "+a.text) + r.assume(st, f) + st.preserved = append(st.preserved, kept{a, len(st.scopes)}) + case "BaseCase": + st.base = append(st.base, r.formula(st, a, false)) + case "Immutable": + default: + r.v.report(a.where, a.endCol, false, a.kind+" does not belong inside a function body") } - return tp, nil } ``` -If a chunk is used twice, an error in it gets reported once, not twice. +```go +func (r *run) stmt(st *state, s ast.Stmt, pending []*annot) outcomes { + switch s := s.(type) { + case *ast.EmptyStmt: + return outcomes{normal: []*state{st}} + case *ast.BlockStmt: + return r.block(st, s) + case *ast.DeclStmt: + g, ok := s.Decl.(*ast.GenDecl) + if !ok || g.Tok != token.VAR { + r.no(s, "only variables can be declared inside a verified function") + } + for _, spec := range g.Specs { + vs := spec.(*ast.ValueSpec) + for i, n := range vs.Names { + var val *expr + k := kNone + if vs.Type != nil { + k, _ = r.v.kindOf(vs.Type) + } + if i < len(vs.Values) { + val = r.value(st, vs.Values[i]) + if k == kNone { + k = kInt + if isBool(val) { + k = kBool + } + } + } else if k == kInt { + val = num(0) + } else if k == kBool { + val = eFalse + } + if val == nil || k != kInt && k != kBool { + r.no(s, "%s is neither an integer nor a boolean", n.Name) + } + st.declare(n.Name, &binding{kind: k, hist: []*expr{val}}) + } + } + return outcomes{normal: []*state{st}} + case *ast.AssignStmt: + r.assign(st, s) + return outcomes{normal: []*state{st}} + case *ast.IncDecStmt: + one := mk(opAdd, r.value(st, s.X), num(1)) + if s.Tok == token.DEC { + one.op = opSub + } + r.set(st, s.X, one, s) + return outcomes{normal: []*state{st}} + case *ast.ExprStmt: + r.effect(st, s) + return outcomes{normal: []*state{st}} + case *ast.ReturnStmt: + r.ret(st, s) + return outcomes{} + case *ast.BranchStmt: + if s.Label != nil { + r.no(s, "labels are outside what litgo prove understands") + } + switch s.Tok { + case token.BREAK: + return outcomes{brk: []*state{st}} + case token.CONTINUE: + return outcomes{cont: []*state{st}} + } + case *ast.IfStmt: + return r.branch(st, s) + case *ast.ForStmt: + return r.loop(st, &loop{stmt: s, init: s.Init, cond: s.Cond, post: s.Post, body: s.Body}, pending) + case *ast.RangeStmt: + return r.loop(st, &loop{stmt: s, over: s, body: s.Body}, pending) + } + r.no(s, "this statement is outside what litgo prove understands") + return outcomes{} +} +``` ```go -func (c *checker) report(err error) { - te, ok := err.(types.Error) - if !ok || len(c.diags) >= 100 { - return +// effect is a call made for what it does. A call of a verified function is +// checked like any other. A call of anything else is let through if all it +// is handed are numbers, booleans and literals, because then it cannot +// change anything the proof is about. panic ends the path. +func (r *run) effect(st *state, s *ast.ExprStmt) { + c, ok := s.X.(*ast.CallExpr) + if !ok { + r.no(s, "this statement is outside what litgo prove understands") } - pos := c.fset.Position(te.Pos) - o := c.outputs[pos.Filename] - if o == nil { + if id, ok := c.Fun.(*ast.Ident); ok && id.Name == "panic" { + r.assume(st, eFalse) return } - line, col, _, ok := o.Map(pos.Line-1, pos.Column-1) - key := fmt.Sprintf("%d:%d:%s", line, col, te.Msg) - if !ok || c.seen[key] { + if callee, _ := r.callee(st, c); callee != nil { + r.eval(st, r.pure(c, func(name string) vkind { + if b := r.binding(st, name); b != nil { + return b.kind + } + return kNone + }), nil, c) return } - c.seen[key] = true - c.diags = append(c.diags, lit.Diag{File: c.res.Doc.Path, Line: line, Col: col, Severity: lit.SevError, Message: te.Msg, Source: "types"}) + if sel, ok := c.Fun.(*ast.SelectorExpr); ok { + if x, ok := sel.X.(*ast.Ident); ok && st.vars[x.Name] != nil { + r.no(c, "%s.%s has no contract, and a method may change what it is called on", x.Name, sel.Sel.Name) + } + } + for _, a := range c.Args { + if lit, ok := a.(*ast.BasicLit); ok && lit.Kind != token.INT { + continue + } + if v := r.value(st, a); v.sort == sArr { + r.no(a, "a slice handed to a function without a contract may come back changed") + } + } } ``` - - - - -The test document writes a small module with a library, a program that uses it, -and a test. None of it exists on disk, and each error is in a different kind of -place. +Assignments go to variables only. `A[i] = x` and `p.f = x` are refused, and the +reason is aliasing: another slice may share the array, another pointer may +reach the struct, and a proof that did not see them change would be a proof of +something false. - ```go -const module = "\n```gomod\nmodule example.com/m\n\ngo 1.22\n```\n\n" + - "\n\n\n" + - "```go\nfunc Shout(s string) string {\n\t// <>\n}\n```\n\n" + - "\n```go\nreturn strings.ToUpper(s) + suffix\n```\n\n" + - "\n\n\n" + - "```go\nfunc TestShout(t *testing.T) {\n\tif Shout(1) != \"A\" {\n\t\tt.Fail()\n\t}\n}\n```\n\n" + - "\n\n\n" + - "```go\nfunc main() {\n\tn := 1\n\tfmt.Println(lib.Shout(\"a\"), lib.Whisper)\n}\n```\n" - -func TestTypes(t *testing.T) { - path := filepath.Join(t.TempDir(), "m.lit.md") - os.WriteFile(path, []byte(module), 0o644) - doc := lit.Parse(path, []byte(module)) - diags := Types(doc.Tangle(), &Cache{}) - - var got []string - for _, d := range diags { - got = append(got, strings.TrimSpace(doc.Lines[d.Line][d.Col:])+" ← "+d.Message) +func (r *run) assign(st *state, s *ast.AssignStmt) { + ops := map[token.Token]op{token.ADD_ASSIGN: opAdd, token.SUB_ASSIGN: opSub, token.MUL_ASSIGN: opMul, + token.QUO_ASSIGN: opDiv, token.REM_ASSIGN: opMod} + if o, ok := ops[s.Tok]; ok { + bin := &ast.BinaryExpr{X: s.Lhs[0], OpPos: s.TokPos, Y: s.Rhs[0], + Op: map[op]token.Token{opAdd: token.ADD, opSub: token.SUB, opMul: token.MUL, opDiv: token.QUO, opMod: token.REM}[o]} + r.set(st, s.Lhs[0], r.value(st, bin), s) + return } - for _, want := range []string{ - `os, strings --> ← "os" imported and not used`, // on its entry in the directive - "suffix ← undefined: suffix", // inside a chunk - "1) != \"A\" { ← cannot use 1", // in a test file of the package - "n := 1 ← declared and not used: n", - "Whisper) ← undefined: lib.Whisper", // through an import of a package that is not on disk - } { - text, msg, _ := strings.Cut(want, " ← ") - found := false - for _, g := range got { - found = found || strings.HasPrefix(g, text) && strings.Contains(g, msg) + if s.Tok != token.ASSIGN && s.Tok != token.DEFINE { + r.no(s, "this assignment is outside what litgo prove understands") + } + var vals []*expr + if len(s.Rhs) == 1 && len(s.Lhs) > 1 { + c, ok := s.Rhs[0].(*ast.CallExpr) + if !ok { + r.no(s, "only a call can give several values at once") } - if !found { - t.Errorf("missing %q in:\n%s", want, strings.Join(got, "\n")) + vals = r.eval(st, r.pure(c, func(name string) vkind { + if b := r.binding(st, name); b != nil { + return b.kind + } + return kNone + }), nil, c) + } else { + for _, x := range s.Rhs { + vals = append(vals, r.value(st, x)) } } - if len(diags) != 5 { - t.Errorf("%d diagnostics, want 5:\n%s", len(diags), strings.Join(got, "\n")) + if len(vals) != len(s.Lhs) { + r.no(s, "the two sides of this assignment do not match") } -} - -func TestSingleProgramStandsAlone(t *testing.T) { - dir := t.TempDir() - os.WriteFile(filepath.Join(dir, "other.go"), []byte("package main\n\nfunc main() {}\n"), 0o644) - src := "\n```go\nfunc main() {}\n```\n" - doc := lit.Parse(filepath.Join(dir, "one.lit.md"), []byte(src)) - if diags := Types(doc.Tangle(), &Cache{}); len(diags) != 0 { - t.Errorf("a second main in the directory is not this program's business: %+v", diags) + for i, l := range s.Lhs { + id, ok := l.(*ast.Ident) + if s.Tok == token.DEFINE && ok && (st.vars[id.Name] == nil || !r.inScope(st, id.Name)) { + k := kInt + if isBool(vals[i]) { + k = kBool + } else if vals[i].sort == sArr { + r.no(s, "slices cannot be copied inside a verified function") + } + st.declare(id.Name, &binding{kind: k, hist: []*expr{vals[i]}}) + continue + } + r.store(st, l, vals[i]) } + // A parallel assignment is one step: what is preserved is looked at + // once all of it has happened. + r.keep(st, s) } -``` - -## The language server - - - - -The Language Server Protocol is JSON-RPC over standard input and output, with a -`Content-Length` header in front of each message. The part litgo needs for -publishing diagnostics is small enough to write out here. - -```go -type message struct { - JSONRPC string `json:"jsonrpc"` - ID *json.RawMessage `json:"id,omitempty"` - Method string `json:"method,omitempty"` - Params json.RawMessage `json:"params,omitempty"` - Result any `json:"result,omitempty"` - Error *rpcError `json:"error,omitempty"` +// inScope reports whether a name was declared in the innermost block, which +// is what decides whether := declares it again. +func (r *run) inScope(st *state, name string) bool { + _, ok := st.scopes[len(st.scopes)-1][name] + return ok } -type rpcError struct { - Code int `json:"code"` - Message string `json:"message"` +// set assigns to a variable, and then looks at what was promised to be +// preserved. +func (r *run) set(st *state, target ast.Expr, val *expr, at ast.Node) { + r.store(st, target, val) + r.keep(st, at) } -func read(r *bufio.Reader) (*message, error) { - length := 0 - for { - line, err := r.ReadString('\n') - if err != nil { - return nil, err - } - line = strings.TrimSpace(line) - if line == "" { - break - } - if name, value, ok := strings.Cut(line, ":"); ok && strings.EqualFold(name, "Content-Length") { - length, _ = strconv.Atoi(strings.TrimSpace(value)) - } +func (r *run) store(st *state, target ast.Expr, val *expr) { + id, ok := target.(*ast.Ident) + if !ok { + r.no(target, "only variables can be assigned to: an element of a slice or a field may have other names, and the proof would not see them change") } - body := make([]byte, length) - if _, err := io.ReadFull(r, body); err != nil { - return nil, err + if id.Name == "_" { + return } - m := &message{} - return m, json.Unmarshal(body, m) + b := st.vars[id.Name] + if b == nil || b.kind != kInt && b.kind != kBool { + r.no(target, "%s is not an integer or boolean variable", id.Name) + } + b.hist = append(b.hist, val) } -// frame spells a message the way it goes over the wire. -func frame(m *message) []byte { - m.JSONRPC = "2.0" - body, _ := json.Marshal(m) - return []byte(fmt.Sprintf("Content-Length: %d\r\n\r\n%s", len(body), body)) +func (r *run) keep(st *state, at ast.Node) { + for _, k := range st.preserved { + f := r.formula(st, k.a, false) + r.check(st, f, k.a, nil, at, "Preserves is broken by the assignment at {ref}: "+k.a.text) + r.assume(st, f) + } } ``` -The server has no locks. One goroutine, the loop in `Serve`, owns all of the -server's state: the open documents, the overlays, and the bookkeeping for -gopls. Every other goroutine does one job that is slow or that blocks, and -talks to the loop over a channel. One goroutine reads messages from the editor -and one writes messages to the editor. Each document that changed has a timer -that tells the loop when the typing has paused. At most one goroutine runs the -type checker. Since only the loop touches the state, nothing needs a mutex. +A branch that cannot be taken is dropped, by asking the prover whether the +hypotheses have become contradictory. Without that, the obligations of +unreachable code would be "proved" from the contradiction, which is harmless, +but paths multiply and these are the cheap ones to lose. -```mermaid -flowchart LR - reader[read the editor] --> loop - timers --> loop - checker[type checker] --> loop - loop --> writer[write to the editor] +```go +func (r *run) branch(st *state, s *ast.IfStmt) outcomes { + st.push() + depth := len(st.scopes) + var out outcomes + starts := []*state{st} + if s.Init != nil { + starts = r.stmt(st, s.Init, nil).normal + } + for _, st := range starts { + cond := r.value(st, s.Cond) + yes, no := st, st.clone() + r.assume(yes, cond) + r.assume(no, not(cond)) + if r.feasible(yes) { + out.add(r.block(yes, s.Body)) + } + if r.feasible(no) { + switch e := s.Else.(type) { + case nil: + out.normal = append(out.normal, no) + case *ast.BlockStmt: + out.add(r.block(no, e)) + case *ast.IfStmt: + out.add(r.branch(no, e)) + } + } + } + for _, s := range out.normal { + s.popTo(depth - 1) + } + return out +} ``` -A pipe gets written by one goroutine that owns it. Messages are framed by -whoever sends them, so the writer never sees a value that someone else might -still change. The channel has a buffer, so the loop can keep going while a -reader on the other end of the pipe is slow. Closing the channel says there is -nothing more to write, and `done` closes after the last message has gone out. - ```go -// writer starts the goroutine that writes to a pipe. -func writer(w io.Writer) (messages chan []byte, done chan struct{}) { - messages, done = make(chan []byte, 64), make(chan struct{}) - go func() { - defer close(done) - for b := range messages { - w.Write(b) +// ret settles a return: the results get their values, and every +// postcondition is an obligation. +func (r *run) ret(st *state, s *ast.ReturnStmt) { + if len(s.Results) > 0 { + var vals []*expr + if len(s.Results) == 1 && len(r.f.results) > 1 { + c, ok := s.Results[0].(*ast.CallExpr) + if !ok { + r.no(s, "only a call can give several values at once") + } + vals = r.eval(st, r.pure(c, func(name string) vkind { + if b := r.binding(st, name); b != nil { + return b.kind + } + return kNone + }), nil, c) + } else { + for _, x := range s.Results { + vals = append(vals, r.value(st, x)) + } } - }() - return messages, done + if len(vals) != len(r.f.results) { + r.no(s, "this return does not match the results") + } + // The results may be shadowed here; a return assigns the real ones. + for i, p := range r.f.results { + st.vars[p.name] = &binding{kind: p.kind, hist: []*expr{vals[i]}} + } + } + r.leave(st, s.Pos()) } -func (s *server) write(m *message) { s.out <- frame(m) } +func (r *run) leave(st *state, at token.Pos) { + for _, a := range r.f.ensures { + if a.e != nil { + r.check(st, r.formula(st, a, true), a, nil, point(at), "Ensures is not proved for the return at {ref}: "+a.text) + } + } + for _, a := range r.f.exsures { + if a.e != nil { + r.check(st, not(r.formula(st, a, true)), a, nil, point(at), "Exsures is not ruled out for the return at {ref}: "+a.text) + } + } +} ``` -The server keeps the text of every open document, and the whole text arrives -with each change. A change starts a short timer instead of an analysis, so a -burst of typing costs one analysis instead of one per keystroke. +### Loops -```go -type document struct { - text string - version int - timer *time.Timer - res *lit.Result // what the text tangles to, - tangled int // and the version of the text that was tangled -} +A loop cannot be walked, because nobody knows how often it goes round. The +invariant is what cuts it: -// server is the state of the loop in Serve. Only that goroutine touches it. -type server struct { - out chan<- []byte // to the goroutine that writes to the editor - docs map[string]*document - utf8 bool // the client counts columns in bytes, as litgo does - delay time.Duration +1. The invariant is proved for the state that reaches the loop. +2. Every variable the loop assigns to is forgotten: it gets a fresh symbol, and + all that is assumed about the new symbols is the invariant (and whatever the + enclosing blocks promised to preserve). This state stands for the start of + *any* iteration. +3. From there, with the condition true, the body is walked once. At its end the + invariant is proved again, and the variant is proved smaller than it was and + to have been non-negative. +4. From the same state, with the condition false, the walk goes on after the + loop. - due chan due // a document's timer went off - checked chan *job // the type checker is done with a job - quit chan struct{} // closed when the server stops - waiting map[string]*job - checking bool // the type checker is busy, and has the cache - cache check.Cache +A `break` leaves with whatever is known at that point, which need not include +the invariant. A `range` loop is a counted loop whose counter cannot be +tampered with, so it needs no variant to be known to end. - // The rest is for the chapter after this one. - goplsBin string - gopls *gopls - overlays map[string]*overlay // by path of the tangled file -} - -// due says that a document has not changed for a while since this version. -type due struct { - uri string - version int -} - -// Serve speaks the protocol until the client says exit or hangs up. It puts -// the gopls named (or found, if none is named; "off" means none) behind it. -func Serve(in io.Reader, out io.Writer, gopls string) error { - toEditor, written := writer(out) - s := &server{out: toEditor, docs: map[string]*document{}, delay: 120 * time.Millisecond, - due: make(chan due), checked: make(chan *job), quit: make(chan struct{}), waiting: map[string]*job{}, - goplsBin: gopls, overlays: map[string]*overlay{}} - defer func() { - close(s.quit) - s.stop() - close(toEditor) - <-written - }() - editor, failed := make(chan *message), make(chan error, 1) - go func() { - r := bufio.NewReader(in) - for { - m, err := read(r) - if err != nil { - failed <- err - return - } - select { - case editor <- m: - case <-s.quit: - return - } +```go +// loop is a for statement, or a range statement read as one. +type loop struct { + stmt ast.Stmt + init ast.Stmt + cond ast.Expr + post ast.Stmt + body *ast.BlockStmt + over *ast.RangeStmt +} + +// loop cuts a loop at its invariant. The invariant is proved on entry; +// then everything the loop assigns is forgotten, the invariant is assumed, +// and one iteration has to give it back, with a smaller variant. What +// follows the loop knows the invariant and that the condition is false. +func (r *run) loop(st *state, l *loop, pending []*annot) outcomes { + var invs, vars []*annot + notes := append(append([]*annot{}, pending...), r.trailing(l.body)...) + for _, a := range notes { + a.used = true + if a.e == nil { + continue } - }() - for { - select { - case m := <-editor: - if m.Method == "exit" { - return nil + if a.kind == "Invariant" { + invs = append(invs, a) + } else { + vars = append(vars, a) + } + } + if len(vars) == 0 && l.over == nil { + r.v.report(r.v.position(l.stmt.Pos()), r.v.position(l.stmt.Pos()).Col+3, true, "this loop has no Variant, so nothing shows that it terminates") + } + st.push() + depth := len(st.scopes) + var out outcomes + starts := []*state{st} + var index string // the counter of a range loop + var bound *expr + if l.over != nil { + index, bound = r.rangeStart(st, l.over) + } else if l.init != nil { + starts = r.stmt(st, l.init, nil).normal + } + for _, s0 := range starts { + for _, a := range invs { + r.check(s0, r.formula(s0, a, false), a, nil, l.stmt, "Invariant does not hold when the loop at {ref} is entered: "+a.text) + } + head := s0.clone() + for _, name := range r.assigned(l) { + if b := head.vars[name]; b != nil && (b.kind == kInt || b.kind == kBool) { + b.hist = []*expr{r.fresh(name, b.kind)} } - s.handle(m) - case err := <-failed: - if err == io.EOF { - return nil + } + if index != "" { + b := head.vars[index] + b.hist = []*expr{r.fresh(index, kInt)} + r.assume(head, mk(opAnd, mk(opLe, num(0), b.hist[0]), mk(opLe, b.hist[0], bound))) + } + for _, a := range invs { + r.assume(head, r.formula(head, a, false)) + } + for _, k := range head.preserved { + r.assume(head, r.formula(head, k.a, false)) + } + var cond *expr + switch { + case index != "": + cond = mk(opLt, head.vars[index].hist[0], bound) + case l.cond != nil: + cond = r.value(head, l.cond) + default: + cond = eTrue + } + exit := head.clone() + r.assume(exit, not(cond)) + if r.feasible(exit) { + out.normal = append(out.normal, exit) + } + body := head + r.assume(body, cond) + if !r.feasible(body) { + continue + } + var before []*expr + for _, a := range vars { + m := r.formula(body, a, false) + before = append(before, m) + r.check(body, mk(opLe, num(0), m), a, nil, nil, "Variant can be negative while the loop still runs: "+a.text) + } + if l.over != nil && l.over.Value != nil { + if id, ok := l.over.Value.(*ast.Ident); ok && id.Name != "_" { + body.push() + arr := r.value(body, l.over.X) + body.declare(id.Name, &binding{kind: kInt, hist: []*expr{mk(opSel, arr, body.vars[index].hist[0])}}) } - return err - case d := <-s.due: - if _, now := s.current(d.uri); now == d.version { - s.analyze(d.uri) + } + o := r.block(body, l.body) + out.normal = append(out.normal, o.brk...) + for _, s := range append(o.normal, o.cont...) { + s.popTo(depth) + ends := []*state{s} + if index != "" { + b := s.vars[index] + b.hist = append(b.hist, mk(opAdd, b.hist[len(b.hist)-1], num(1))) + } else if l.post != nil { + ends = r.stmt(s, l.post, nil).normal } - case j := <-s.checked: - s.finish(j) - case m, ok := <-s.heard(): - if ok { - s.gopls.hear(m, s) - } else { - s.gopls.hangUp() + for _, s := range ends { + for _, a := range invs { + r.check(s, r.formula(s, a, false), a, nil, l.stmt, "Invariant is not maintained by the body of the loop at {ref}: "+a.text) + } + for i, a := range vars { + r.check(s, mk(opLt, r.formula(s, a, false), before[i]), a, nil, l.stmt, "Variant is not shown to decrease in the loop at {ref}: "+a.text) + } } } } + for _, s := range out.normal { + s.popTo(depth - 1) + } + return out } ``` -Every goroutine that sends to the loop also watches `quit`, so none of them is -left waiting for a loop that has gone away. - ```go -func (s *server) handle(m *message) { - switch m.Method { - case "initialize": - // <> - case "shutdown": - s.write(&message{ID: m.ID, Result: json.RawMessage("null")}) - case "textDocument/didOpen": - var p struct { - TextDocument struct { - URI string `json:"uri"` - Text string `json:"text"` - Version int `json:"version"` - } `json:"textDocument"` +// trailing finds the Invariant and Variant lines written inside a loop's +// body rather than above the loop: the ones no inner loop follows. +func (r *run) trailing(body *ast.BlockStmt) []*annot { + var out []*annot + for _, a := range r.blocks[body] { + if a.kind != "Invariant" && a.kind != "Variant" { + continue } - json.Unmarshal(m.Params, &p) - s.change(p.TextDocument.URI, p.TextDocument.Text, p.TextDocument.Version) - case "textDocument/didChange": - var p struct { - TextDocument struct { - URI string `json:"uri"` - Version int `json:"version"` - } `json:"textDocument"` - ContentChanges []struct { - Text string `json:"text"` - } `json:"contentChanges"` + var next ast.Stmt + for _, s := range body.List { + if s.Pos() > a.pos { + next = s + break + } } - json.Unmarshal(m.Params, &p) - if n := len(p.ContentChanges); n > 0 { - s.change(p.TextDocument.URI, p.ContentChanges[n-1].Text, p.TextDocument.Version) + if !isLoop(next) { + out = append(out, a) } - case "textDocument/didClose": - var p struct { - TextDocument struct { - URI string `json:"uri"` - } `json:"textDocument"` + } + return out +} + +// rangeStart declares the counter of a range loop, under the name the +// program gives it if it gives it one, and finds where the counting stops. +func (r *run) rangeStart(st *state, s *ast.RangeStmt) (string, *expr) { + if s.Tok == token.ASSIGN { + r.no(s, "a range loop has to declare its variables with :=") + } + index := "range counter" + if id, ok := s.Key.(*ast.Ident); ok && id.Name != "_" { + index = id.Name + } + for _, name := range r.assigned(&loop{body: s.Body}) { + if name == index { + r.no(s, "the body assigns to the loop's own counter") } - json.Unmarshal(m.Params, &p) - if d := s.docs[p.TextDocument.URI]; d != nil && d.timer != nil { - d.timer.Stop() + } + var bound *expr + if id, ok := s.X.(*ast.Ident); ok { + if b := r.binding(st, id.Name); b != nil && b.kind == kSlice { + bound = mk(opLen, b.hist[0]) } - delete(s.docs, p.TextDocument.URI) - delete(s.waiting, p.TextDocument.URI) - s.forget(p.TextDocument.URI) - default: - if !s.proxy(m) && m.ID != nil { // a request for something this server does not do - s.write(&message{ID: m.ID, Error: &rpcError{-32601, "litgo lsp does not implement " + m.Method}}) + } + if sel, ok := s.X.(*ast.SelectorExpr); ok && bound == nil { + if x, ok := sel.X.(*ast.Ident); ok { + if b := r.binding(st, x.Name+"."+sel.Sel.Name); b != nil && b.kind == kSlice { + bound = mk(opLen, b.hist[0]) + } + } + } + if bound == nil { + if s.Value != nil { + r.no(s, "only a slice of integers or an integer can be ranged over") + } + bound = r.value(st, s.X) + if isBool(bound) || bound.sort == sArr { + r.no(s, "only a slice of integers or an integer can be ranged over") + } + } + st.declare(index, &binding{kind: kInt, hist: []*expr{num(0)}}) + return index, bound +} + +// assigned lists the variables a loop can change. +func (r *run) assigned(l *loop) []string { + seen := map[string]bool{} + visit := func(n ast.Node) bool { + switch s := n.(type) { + case *ast.AssignStmt: + for _, x := range s.Lhs { + if id, ok := x.(*ast.Ident); ok { + seen[id.Name] = true + } + } + case *ast.IncDecStmt: + if id, ok := s.X.(*ast.Ident); ok { + seen[id.Name] = true + } + case *ast.RangeStmt: + for _, x := range []ast.Expr{s.Key, s.Value} { + if id, ok := x.(*ast.Ident); ok { + seen[id.Name] = true + } + } } + return true } + ast.Inspect(l.body, visit) + if l.post != nil { + ast.Inspect(l.post, visit) + } + out := make([]string, 0, len(seen)) + for n := range seen { + out = append(out, n) + } + sort.Strings(out) + return out } ``` -The protocol counts columns in UTF-16 code units unless both sides agree on -something else. litgo counts bytes and so does Neovim, which offers `utf-8`. -For a client that doesn't offer it, litgo converts columns on the way out. -Documents are synchronised whole (`change: 1`), because they are small and -litgo parses the whole document anyway. +## The prover + + + + + +An obligation is a list of hypotheses and a goal. The prover looks for a way +the hypotheses could all be true and the goal false; if there is none, the +goal follows. That takes four steps, each sound by itself: + +1. Negations are pushed inward, which turns the negated goal's `Forall` into + an `Exists`, and every `Exists` that is not under a `Forall` gets a fresh + constant as its witness. +2. Every `Forall` is replaced by a handful of its instances: at the terms + that index a slice somewhere in the obligation, and at the witnesses. A + universal fact is only ever weakened by this. If that is not enough, it is + done once more, at the terms the instances brought with them, which is + what a proof about `A[Parent(k)]` needs and what most proofs are better + off without. +3. What is left has no quantifiers. Its atoms are turned into linear + constraints over polynomials. +4. A search splits on the disjunctions, and Fourier-Motzkin elimination closes + a branch when its constraints are contradictory. + +Arithmetic is normalised into polynomials. That is what makes `(q+1)*d + (r-d)` +and `q*d + r` the same thing without any search: they are the same +polynomial. - ```go -var p struct { - Capabilities struct { - General struct { - PositionEncodings []string `json:"positionEncodings"` - } `json:"general"` - } `json:"capabilities"` +// A poly is a sum of monomials with integer coefficients. A monomial is a +// product of atoms, written as their names joined by sep; the constant's +// monomial is the empty string. +type poly map[string]int64 + +const sep = "\x1f" + +func constant(v int64) poly { + if v == 0 { + return poly{} + } + return poly{"": v} } -json.Unmarshal(m.Params, &p) -s.start(m.Params) -encoding := "utf-16" -for _, e := range p.Capabilities.General.PositionEncodings { - if e == "utf-8" && s.gopls == nil { // gopls counts in UTF-16, and that settles it - encoding, s.utf8 = e, true + +func (p poly) add(q poly, k int64) poly { + out := poly{} + for m, c := range p { + out[m] = c + } + for m, c := range q { + out[m] += k * c + if out[m] == 0 { + delete(out, m) + } } + return out } -capabilities := s.capabilities() -capabilities["positionEncoding"] = encoding -capabilities["textDocumentSync"] = map[string]any{"openClose": true, "change": 1} -s.write(&message{ID: m.ID, Result: map[string]any{ - "serverInfo": map[string]any{"name": "litgo"}, - "capabilities": capabilities, -}}) -``` -```go -func (s *server) change(uri, text string, version int) { - if !strings.HasSuffix(uri, ".lit.md") { - return +func (p poly) mul(q poly) poly { + out := poly{} + for m1, c1 := range p { + for m2, c2 := range q { + var atoms []string + if m1 != "" { + atoms = append(atoms, strings.Split(m1, sep)...) + } + if m2 != "" { + atoms = append(atoms, strings.Split(m2, sep)...) + } + sort.Strings(atoms) + m := strings.Join(atoms, sep) + out[m] += c1 * c2 + if out[m] == 0 { + delete(out, m) + } + } } - d := s.docs[uri] - if d == nil { - d = &document{} - s.docs[uri] = d + return out +} + +func (p poly) isConst() (int64, bool) { + if len(p) == 0 { + return 0, true } - if d.timer != nil { - d.timer.Stop() + if c, ok := p[""]; ok && len(p) == 1 { + return c, true } - d.text, d.version = text, version - d.timer = time.AfterFunc(s.delay, func() { - select { - case s.due <- due{uri, version}: - case <-s.quit: - } - }) + return 0, false } -// current returns the text and version of a document, or version -1. -func (s *server) current(uri string) (string, int) { - if d := s.docs[uri]; d != nil { - return d.text, d.version +func (p poly) String() string { + keys := make([]string, 0, len(p)) + for m := range p { + keys = append(keys, m) } - return "", -1 + sort.Strings(keys) + var b strings.Builder + for _, m := range keys { + fmt.Fprintf(&b, "%+d", p[m]) + if m != "" { + b.WriteString("*" + strings.ReplaceAll(m, sep, "*")) + } + } + if b.Len() == 0 { + return "0" + } + return b.String() } ``` -Diagnostics go out in two rounds. Problems with the document's structure and -with Go's syntax take microseconds to find, so the loop finds them itself and -publishes them right away. If there are none, the document needs a type check. -That can take a while, because the first one in a session has to wait for -`go list`, so it runs in a goroutine of its own and the loop stays free to -answer the editor. - -Only one type check runs at a time, because the cache is not locked, and -because a machine has better things to do than check five versions of one -document at once. While the checker is busy, the loop keeps the newest job for -each document in `waiting`, and a newer job replaces an older one. When the -checker sends its job back, the loop publishes the findings unless the document -has changed in the meantime, and then starts the next job. +Over the integers `p < 0` is `p + 1 <= 0`, so there are no strict +inequalities, and `p <> 0` is a choice between `p < 0` and `p > 0`. ```go -// job is a type check: what to check, and then what the check found. The -// loop and the checker pass it back and forth and never hold it together. -type job struct { - uri string - version int - res *lit.Result - diags []lit.Diag // published already - typed []lit.Diag // what the type checker adds +// A node is a formula whose leaves are constraints: p <= 0, p = 0, or a +// boolean atom with a sign. +type node struct { + kind byte // 'a' and, 'o' or, 'l' p <= 0, 'e' p = 0, 'b' boolean atom, 't' true, 'f' false + kids []*node + p poly + atom string + truth bool } -func (s *server) analyze(uri string) { - res, version := s.sync(uri) - if res == nil { - return +var ( + nTrue = &node{kind: 't'} + nFalse = &node{kind: 'f'} +) + +func le0(p poly) *node { + if c, ok := p.isConst(); ok { + if c <= 0 { + return nTrue + } + return nFalse } - diags := append(append([]lit.Diag{}, res.Diags...), res.SyntaxCheck()...) - s.publish(uri, version, res.Doc, diags) - if res.HasErrors() || len(diags) > len(res.Diags) { - return + return &node{kind: 'l', p: p} +} + +func eq0(p poly) *node { + if c, ok := p.isConst(); ok { + if c == 0 { + return nTrue + } + return nFalse } - s.waiting[uri] = &job{uri: uri, version: version, res: res, diags: diags} - s.check() + return &node{kind: 'e', p: p} } -// check gives the type checker its next job, if it is free and there is one -// that is still worth doing. The cache goes to the checker with the job and -// comes back with it. -func (s *server) check() { - if s.checking { - return - } - for uri, j := range s.waiting { - delete(s.waiting, uri) - if _, now := s.current(uri); now != j.version { - continue +// lt0 is p < 0, which over the integers is p + 1 <= 0. +func lt0(p poly) *node { return le0(p.add(constant(1), 1)) } + +func ne0(p poly) *node { return nOr(lt0(p), lt0(poly{}.add(p, -1))) } + +func nAnd(kids ...*node) *node { + var out []*node + for _, k := range kids { + switch k.kind { + case 't': + case 'f': + return nFalse + case 'a': + out = append(out, k.kids...) + default: + out = append(out, k) } - s.checking = true - go func() { - j.typed = check.Types(j.res, &s.cache) - select { - case s.checked <- j: - case <-s.quit: - } - }() - return } + if len(out) == 0 { + return nTrue + } + if len(out) == 1 { + return out[0] + } + return &node{kind: 'a', kids: out} } -// finish publishes what a type check found, if it is still true. -func (s *server) finish(j *job) { - s.checking = false - if _, now := s.current(j.uri); now == j.version && len(j.typed) > 0 { - s.publish(j.uri, j.version, j.res.Doc, append(j.diags, j.typed...)) +func nOr(kids ...*node) *node { + var out []*node + for _, k := range kids { + switch k.kind { + case 'f': + case 't': + return nTrue + case 'o': + out = append(out, k.kids...) + default: + out = append(out, k) + } } - s.check() + if len(out) == 0 { + return nFalse + } + if len(out) == 1 { + return out[0] + } + return &node{kind: 'o', kids: out} } +``` -func path(uri string) string { - if u, err := url.Parse(uri); err == nil && u.Scheme == "file" { - return u.Path +```go +// The budgets keep a hopeless proof from holding up the editor. Running out +// of budget means "not proved", never "proved". +const ( + maxSteps = 2000 + maxConstraints = 1500 + maxInstances = 4000 + maxTerms = 14 +) + +type prover struct { + fresh int + steps int + instances int + gaveUp bool + atoms map[string]bool // atoms whose axioms have been stated + axioms []*node + sels map[string][]selAtom // by array +} + +type selAtom struct { + name string + idx poly +} + +// proves reports whether the hypotheses entail the goal. It answers by +// refutation: the hypotheses and the goal's negation must have no model. +func proves(hyps []*expr, goal *expr) bool { + // Most proofs need every Forall only at the terms that are already there. + // Some need it again at the terms those instances mention, and that costs + // much more, so it is the second thing to try and not the first. + return attempt(hyps, goal, 1) || attempt(hyps, goal, 2) +} + +func attempt(hyps []*expr, goal *expr, rounds int) bool { + pr := &prover{atoms: map[string]bool{}, sels: map[string][]selAtom{}} + f := nnf(and(conj(hyps), not(goal)), true) + f = pr.skolem(f) + for round := 0; round < rounds; round++ { + f = pr.instantiate(f, pr.terms(f), round < rounds-1) } - return uri + f = pr.instantiate(f, nil, false) + g := pr.ground(f) + pr.congruence() + all := nAnd(append([]*node{g}, pr.axioms...)...) + return pr.refute(nil, map[string]bool{}, []*node{all}) && !pr.gaveUp } ``` -A diagnostic from litgo is a point, but an editor underlines a range. So litgo -stretches the range over the identifier at that point, when there is one. - ```go -func (s *server) publish(uri string, version int, doc *lit.Doc, diags []lit.Diag) { - type position struct { - Line int `json:"line"` - Character int `json:"character"` +// nnf pushes negations down to the atoms. pos is false under a negation. +func nnf(e *expr, pos bool) *expr { + switch e.op { + case opTrue, opFalse: + if pos == (e.op == opTrue) { + return eTrue + } + return eFalse + case opNot: + return nnf(e.args[0], !pos) + case opAnd, opOr: + o := e.op + if !pos { + o = opAnd + opOr - o + } + return mk(o, nnf(e.args[0], pos), nnf(e.args[1], pos)) + case opImp: + if pos { + return mk(opOr, nnf(e.args[0], false), nnf(e.args[1], true)) + } + return mk(opAnd, nnf(e.args[0], true), nnf(e.args[1], false)) + case opIff: + a, b := e.args[0], e.args[1] + if pos { + return mk(opAnd, mk(opOr, nnf(a, false), nnf(b, true)), mk(opOr, nnf(b, false), nnf(a, true))) + } + return mk(opOr, mk(opAnd, nnf(a, true), nnf(b, false)), mk(opAnd, nnf(b, true), nnf(a, false))) + case opForall, opExists: + o := e.op + if !pos { + o = opForall + opExists - o + } + return &expr{op: o, name: e.name, args: []*expr{nnf(e.args[0], pos)}} + } + if pos { + return e } - type diagnostic struct { - Range struct { - Start position `json:"start"` - End position `json:"end"` - } `json:"range"` - Severity int `json:"severity"` - Source string `json:"source"` - Message string `json:"message"` + switch e.op { + case opLt: + return mk(opLe, e.args[1], e.args[0]) + case opLe: + return mk(opLt, e.args[1], e.args[0]) + case opEq: + return mk(opNe, e.args[0], e.args[1]) + case opNe: + return mk(opEq, e.args[0], e.args[1]) } - out := []diagnostic{} - for _, d := range diags { - line := "" - if d.Line >= 0 && d.Line < len(doc.Lines) { - line = doc.Lines[d.Line] + return not(e) // a boolean atom +} +``` + +A witness may only be named once nothing universal is above it. For +`Forall i : Exists j : P(i, j)` the `j` depends on the `i`, so the `Exists` +waits until the `Forall` has been instantiated and gets a new witness per +instance. + +```go +// skolem names a witness for every Exists that is not under a Forall. One +// under a Forall waits until the Forall has been instantiated. +func (pr *prover) skolem(e *expr) *expr { + switch e.op { + case opAnd, opOr: + return mk(e.op, pr.skolem(e.args[0]), pr.skolem(e.args[1])) + case opExists: + pr.fresh++ + w := sym(fmt.Sprintf("%s?%d", e.name, pr.fresh), sInt) + return pr.skolem(subst(e.args[0], e.name, w)) + } + return e +} + +// terms collects what a Forall might be worth instantiating with: the ground +// terms that index a slice, and the witnesses. +func (pr *prover) terms(e *expr) []*expr { + seen := map[string]bool{} + var out []*expr + var walk func(e *expr, bound []string) + walk = func(e *expr, bound []string) { + if e == nil { + return } - start := min(max(d.Col, 0), len(line)) - end := d.EndCol - if end <= start { - end = start - for end < len(line) && (line[end] == '_' || line[end] >= '0' && line[end] <= '9' || line[end] >= 'A' && line[end] <= 'Z' || line[end] >= 'a' && line[end] <= 'z' || line[end] >= 0x80) { - end++ + if e.op == opForall || e.op == opExists { + bound = append(bound[:len(bound):len(bound)], e.name) + } + candidate := e.op == opSel && len(e.args) == 2 + var t *expr + if candidate { + t = e.args[1] + } else if e.op == opVar && strings.Contains(e.name, "?") { + t, candidate = e, true + } + if candidate { + free := true + for _, n := range names(t) { + for _, b := range bound { + free = free && n != b + } } - if end == start { - end = min(start+1, len(line)) + if key := t.String(); free && !seen[key] && len(out) < maxTerms { + seen[key] = true + out = append(out, t) } } - end = min(end, len(line)) - var ld diagnostic - ld.Range.Start = position{d.Line, s.column(line, start)} - ld.Range.End = position{d.Line, s.column(line, end)} - ld.Severity, ld.Source, ld.Message = d.Severity, "litgo", d.Message - if d.Source != "" && d.Source != "litgo" { - ld.Source = "litgo " + d.Source + for _, a := range e.args { + walk(a, bound) } - out = append(out, ld) } - params, _ := json.Marshal(map[string]any{"uri": uri, "version": version, "diagnostics": out}) - s.write(&message{Method: "textDocument/publishDiagnostics", Params: params}) -} - -// column converts a byte offset into the client's unit. -func (s *server) column(line string, offset int) int { - return units(s.utf8, line, offset) + walk(e, nil) + return out } -// units converts a byte offset in a line to UTF-16 code units, unless bytes -// are what is wanted. -func units(utf8 bool, line string, offset int) int { - offset = min(max(offset, 0), len(line)) - if utf8 { - return offset - } - n := 0 - for _, r := range line[:offset] { - n++ - if r >= 0x10000 { - n++ +// instantiate replaces every outermost Forall by its instances at the terms, +// and keeps the Forall itself for a later round if asked to. +func (pr *prover) instantiate(e *expr, terms []*expr, keep bool) *expr { + switch e.op { + case opAnd, opOr: + return mk(e.op, pr.instantiate(e.args[0], terms, keep), pr.instantiate(e.args[1], terms, keep)) + case opForall: + out := eTrue + if keep { + out = e + } + for _, t := range terms { + if pr.instances++; pr.instances > maxInstances { + pr.gaveUp = true + break + } + inst := pr.skolem(subst(e.args[0], e.name, t)) + out = and(out, pr.instantiate(inst, terms, keep)) } + return out } - return n + return e } ``` - - - +```go +// ground turns a quantifier-free formula into constraints. +func (pr *prover) ground(e *expr) *node { + switch e.op { + case opTrue: + return nTrue + case opFalse: + return nFalse + case opAnd: + return nAnd(pr.ground(e.args[0]), pr.ground(e.args[1])) + case opOr: + return nOr(pr.ground(e.args[0]), pr.ground(e.args[1])) + case opForall: + return nTrue // a fact we could not use + case opExists: + pr.gaveUp = true + return nTrue + case opNot: + return &node{kind: 'b', atom: e.args[0].String(), truth: false} + case opVar, opCall: + return &node{kind: 'b', atom: e.String(), truth: true} + } + if len(e.args) != 2 || isBool(e.args[0]) || isBool(e.args[1]) { + pr.gaveUp = true // a number where a formula belongs, or the other way round + return nTrue + } + d := pr.poly(e.args[0]).add(pr.poly(e.args[1]), -1) + switch e.op { + case opLe: + return le0(d) + case opLt: + return lt0(d) + case opEq: + return eq0(d) + case opNe: + return ne0(d) + } + pr.gaveUp = true + return nTrue +} +``` -The test acts as a client. It opens a document that has a type error in a -chunk, fixes the error, and watches the diagnostics show up and then go away. +Whatever is not addition or multiplication by a constant becomes an atom, an +unknown of its own, and the facts that make it more than an unknown are added +as axioms the first time it appears: a length is not negative, a quotient and +its remainder add up, equal indices select equal elements, and a product of +non-negative numbers is not negative. - ```go -const program = "\n\n\n```go\nfunc main() {\n\t// <>\n}\n```\n\n\n```go\nfmt.Println(\"π„ž\", whom)\n```\n" - -// client is the editor's end of a conversation with the server. -type client struct { - t *testing.T - to io.Writer - incoming chan *message +// poly normalises an integer term. Whatever is not arithmetic becomes an +// atom: a variable, an element of a slice, a length, a quotient. +func (pr *prover) poly(e *expr) poly { + switch e.op { + case opNum: + return constant(e.val) + case opVar: + return poly{e.name: 1} + case opAdd: + return pr.poly(e.args[0]).add(pr.poly(e.args[1]), 1) + case opSub: + return pr.poly(e.args[0]).add(pr.poly(e.args[1]), -1) + case opNeg: + return poly{}.add(pr.poly(e.args[0]), -1) + case opMul: + p := pr.poly(e.args[0]).mul(pr.poly(e.args[1])) + pr.products(p) + return p + case opLen: + name := "len(" + e.args[0].name + ")" + if !pr.atoms[name] { + pr.atoms[name] = true + pr.axioms = append(pr.axioms, le0(poly{name: -1})) + } + return poly{name: 1} + case opSel: + idx := pr.poly(e.args[1]) + arr := e.args[0].name + name := arr + "[" + idx.String() + "]" + if !pr.atoms[name] { + pr.atoms[name] = true + pr.sels[arr] = append(pr.sels[arr], selAtom{name, idx}) + } + return poly{name: 1} + case opDiv, opMod: + return pr.divide(e) + } + pr.gaveUp = true + return poly{e.String(): 1} +} + +// divide states what Go's division means. For a = q*b + r with b > 0, the +// remainder has the sign of a and is smaller than b. +func (pr *prover) divide(e *expr) poly { + a, b := pr.poly(e.args[0]), pr.poly(e.args[1]) + key := "(" + a.String() + ")/(" + b.String() + ")" + q, r := "quo"+key, "rem"+key + if !pr.atoms[q] { + pr.atoms[q] = true + qb := poly{q: 1}.mul(b) + pr.products(qb) + // a = q*b + r + pr.axioms = append(pr.axioms, eq0(a.add(qb, -1).add(poly{r: 1}, -1))) + pos := lt0(poly{}.add(b, -1)) // 0 < b + nonneg, nonpos := le0(poly{}.add(a, -1)), le0(a) // 0 <= a, a <= 0 + small := lt0(poly{r: 1}.add(b, -1)) // r < b + smallNeg := lt0(poly{r: -1}.add(b, -1)) // -r < b + pr.axioms = append(pr.axioms, + nOr(negate(pos), negate(nonneg), nAnd(le0(poly{r: -1}), small, le0(poly{q: -1}))), + nOr(negate(pos), negate(nonpos), nAnd(le0(poly{r: 1}), smallNeg, le0(poly{q: 1})))) + } + if e.op == opDiv { + return poly{q: 1} + } + return poly{r: 1} +} + +// negate works on the constraints divide builds: p <= 0 becomes -p + 1 <= 0. +func negate(n *node) *node { + switch n.kind { + case 't': + return nFalse + case 'f': + return nTrue + case 'l': + return lt0(poly{}.add(n.p, -1)) + } + panic("negate: not an inequality") +} + +// products says the little that is known about x*y: it is not negative when +// neither factor is, and a square never is. +func (pr *prover) products(p poly) { + for m := range p { + atoms := strings.Split(m, sep) + if len(atoms) != 2 || pr.atoms[m] { + continue + } + pr.atoms[m] = true + x, y := atoms[0], atoms[1] + if x == y { + pr.axioms = append(pr.axioms, le0(poly{m: -1})) + continue + } + pr.axioms = append(pr.axioms, nOr(lt0(poly{x: 1}), lt0(poly{y: 1}), le0(poly{m: -1})), + nOr(lt0(poly{x: 1}), le0(poly{y: 1}), le0(poly{x: 1, m: -1})), // 0 <= x and 1 <= y: x <= xy + nOr(lt0(poly{y: 1}), le0(poly{x: 1}), le0(poly{y: 1, m: -1}))) + } } -func connect(t *testing.T, gopls string) *client { - toServer, fromClient := io.Pipe() - fromServer, toClient := io.Pipe() - go Serve(toServer, toClient, gopls) - t.Cleanup(func() { fromClient.Close() }) - c := &client{t: t, to: fromClient, incoming: make(chan *message, 64)} - go func() { - r := bufio.NewReader(fromServer) - for { - m, err := read(r) - if err != nil { - return +// congruence says that equal indices select equal elements. +func (pr *prover) congruence() { + pairs := 0 + for _, list := range pr.sels { + for i := range list { + for j := i + 1; j < len(list); j++ { + d := list[i].idx.add(list[j].idx, -1) + if c, ok := d.isConst(); ok && c != 0 { + continue + } + if pairs++; pairs > 80 { + return + } + pr.axioms = append(pr.axioms, nOr(ne0(d), eq0(poly{list[i].name: 1, list[j].name: -1}))) } - c.incoming <- m } - }() - return c + } } +``` -func (c *client) send(method string, id int, params any) { - raw, ok := params.(string) - if !ok { - b, _ := json.Marshal(params) - raw = string(b) +The search keeps the constraints of the branch it is in and the disjunctions +it has not decided. Before it splits, it drops every disjunct that already +contradicts the branch; a disjunction with one disjunct left is a fact and one +with none closes the branch, which is unit propagation, and it is what keeps +the instances of a quantifier from multiplying the work. + +```go +// refute reports whether the constraints, the boolean assignment and the +// formulas still to do are unsatisfiable together. +func (pr *prover) refute(cons []poly2, bools map[string]bool, todo []*node) bool { + if pr.steps++; pr.steps > maxSteps { + pr.gaveUp = true + return false } - body := fmt.Sprintf(`{"jsonrpc":"2.0","method":%q,"params":%s}`, method, raw) - if id > 0 { - body = fmt.Sprintf(`{"jsonrpc":"2.0","id":%d,"method":%q,"params":%s}`, id, method, raw) + cons = cons[:len(cons):len(cons)] + var ors []*node + copied := false + for len(todo) > 0 { + n := todo[len(todo)-1] + todo = todo[: len(todo)-1 : len(todo)-1] + switch n.kind { + case 'f': + return true + case 'a': + todo = append(todo, n.kids...) + case 'o': + ors = append(ors, n) + case 'l', 'e': + cons = append(cons, poly2{n.p, n.kind == 'e'}) + case 'b': + if v, ok := bools[n.atom]; ok && v != n.truth { + return true + } + if !copied { + c := make(map[string]bool, len(bools)+1) + for k, v := range bools { + c[k] = v + } + bools, copied = c, true + } + bools[n.atom] = n.truth + } } - fmt.Fprintf(c.to, "Content-Length: %d\r\n\r\n%s", len(body), body) -} - -func (c *client) next() *message { - select { - case m := <-c.incoming: - return m - case <-time.After(60 * time.Second): - c.t.Fatal("the server said nothing for a minute") - return nil + if pr.unsat(cons) { + return true } -} - -// diagnostics waits for a publication for the given version that satisfies ok. -func (c *client) diagnostics(version int, ok func(string) bool) string { - for { - m := c.next() - if m.Method == "textDocument/publishDiagnostics" && strings.Contains(string(m.Params), fmt.Sprintf(`"version":%d`, version)) && ok(string(m.Params)) { - return string(m.Params) + if len(ors) == 0 { + return false + } + // Drop the disjuncts that are already impossible; a disjunction with one + // disjunct left is a fact, and one with none closes the branch. + var rest []*node + var units []*node + for _, o := range ors { + var live []*node + for _, k := range o.kids { + if !pr.dead(cons, bools, k) { + live = append(live, k) + } + } + switch len(live) { + case 0: + return true + case 1: + units = append(units, live[0]) + default: + rest = append(rest, &node{kind: 'o', kids: live}) } } -} - -func TestDiagnosticsComeAndGo(t *testing.T) { - c := connect(t, "off") - c.send("initialize", 1, `{"capabilities":{}}`) - if m := c.next(); !strings.Contains(fmt.Sprint(m.Result), "utf-16") { - t.Fatalf("a client that offers nothing gets UTF-16: %v", m.Result) + if len(units) > 0 { + return pr.refute(cons, bools, append(rest, units...)) } - uri := "file://" + t.TempDir() + "/hello.lit.md" - c.send("textDocument/didOpen", 0, map[string]any{"textDocument": map[string]any{"uri": uri, "version": 1, "text": program}}) - got := c.diagnostics(1, func(p string) bool { return strings.Contains(p, "undefined: whom") }) - // Line 11, after fmt.Println(" (13 bytes), a clef that is 4 bytes but 2 UTF-16 units, and `", `. - if !strings.Contains(got, `"start":{"line":11,"character":18}`) || !strings.Contains(got, `"end":{"line":11,"character":22}`) { - t.Errorf("the error should underline whom, in UTF-16 units: %s", got) + sort.SliceStable(rest, func(i, j int) bool { return len(rest[i].kids) < len(rest[j].kids) }) + for _, k := range rest[0].kids { + if !pr.refute(cons, bools, append(rest[1:len(rest):len(rest)], k)) { + return false + } } + return true +} - c.send("textDocument/didChange", 0, map[string]any{ - "textDocument": map[string]any{"uri": uri, "version": 2}, - "contentChanges": []any{map[string]any{"text": strings.Replace(program, "whom", `"world"`, 1)}}, - }) - c.diagnostics(2, func(p string) bool { return strings.Contains(p, `"diagnostics":[]`) }) - - c.send("textDocument/formatting", 2, `{}`) - if m := c.next(); m.Error == nil || m.Error.Code != -32601 { - t.Errorf("an unknown request should be refused politely: %+v", m) +// dead reports whether a disjunct is a constraint that contradicts what is +// already known. Anything more complicated is left for the branch to find. +func (pr *prover) dead(cons []poly2, bools map[string]bool, n *node) bool { + switch n.kind { + case 'f': + return true + case 'b': + v, ok := bools[n.atom] + return ok && v != n.truth + case 'l', 'e': + return pr.unsat(append(cons[:len(cons):len(cons)], poly2{n.p, n.kind == 'e'})) } + return false } -``` -# Everything else gopls knows +// poly2 is one constraint: p <= 0, or p = 0. +type poly2 struct { + p poly + eq bool +} +``` - - - +### Fourier-Motzkin elimination -Showing errors is the least an editor does for a Go programmer. It also -completes names, shows documentation, jumps to definitions, lists references -and renames things. gopls, Go's language server, does all of that, and none of -it should be written twice. But gopls reads Go files, and you are in a Markdown -file where the Go is scattered around in the wrong order. - -The source map works in both directions, so litgo can sit between the editor -and gopls. It starts its own gopls and shows it the tangled files as -*overlays*. An overlay is the protocol's word for the content of a file that is -open in an editor, and it takes precedence over what is on disk. Nothing gets -written. A question about a position in the document turns into the same -question about a position in a tangled file, and every position in the answer -gets translated back. - -```mermaid -sequenceDiagram - participant E as editor - participant S as litgo lsp - participant G as gopls - E->>S: didChange: the whole buffer - S->>G: didChange: pool.go, tangled, never written - E->>S: definition? pool.lit.md:67:15 - S->>G: definition? pool.go:31:14 - G-->>S: pool.go:12:6 - S-->>E: pool.lit.md:40:6 -``` - -The editor only sees one server, and to the editor a `.lit.md` buffer is a Go -buffer wherever the cursor is on Go. This works in any editor, because nothing -here is specific to Neovim. - -## A server of our own - -litgo looks for gopls on the `$PATH` first, and then in the directory where -`go install` puts it, because not everyone has that directory on their `$PATH`. -Without gopls, litgo is just the server from the last chapter. +To decide whether linear inequalities can hold together, pick a variable, +pair every constraint that bounds it from above with every one that bounds it +from below, and keep the sums, in which the variable cancels. Repeat until no +variables are left; the constraints are contradictory exactly when some +$c \le 0$ with a positive $c$ remains. That is complete over the rationals. +Over the integers it is only sound, and two small things make it sharper: +equalities are solved and substituted first, and every constraint is divided +by the gcd of its coefficients with the constant rounded, so that +$2x \le 1$ becomes $x \le 0$. ```go -// findGopls locates the gopls to run: the one named, or the one that can be -// found. "off" means none. -func findGopls(named string) string { - if named == "off" { - return "" - } - if named != "" { - p, _ := exec.LookPath(named) - return p - } - if p, err := exec.LookPath("gopls"); err == nil { - return p +// unsat decides whether linear constraints have no rational solution, with +// the rounding that makes it sharper over the integers. Every monomial is a +// variable here, so a product is just another unknown. +func (pr *prover) unsat(cons []poly2) bool { + var eqs, les []poly + for _, c := range cons { + if c.eq { + eqs = append(eqs, c.p) + } else { + les = append(les, c.p) + } } - gopath := os.Getenv("GOPATH") - if home, err := os.UserHomeDir(); gopath == "" && err == nil { - gopath = filepath.Join(home, "go") + eqs, les = squeeze(eqs, les) + eqs, les = unfoldProducts(eqs, les) + // Equalities first: solve each for one variable and substitute. + for len(eqs) > 0 { + e := eqs[0] + eqs = eqs[1:] + e, ok := tighten(e, true) + if !ok { + return true + } + if len(e) == 0 { + continue + } + v := pivot(e) + for i := range eqs { + eqs[i] = eliminate(eqs[i], e, v) + } + for i := range les { + les[i] = eliminate(les[i], e, v) + } } - for _, dir := range []string{os.Getenv("GOBIN"), filepath.Join(gopath, "bin")} { - if p, err := exec.LookPath(filepath.Join(dir, "gopls")); dir != "" && err == nil { - return p + for { + // Normalise, drop what is trivially true, and keep the tightest of the + // constraints that differ only in their constant. + best := map[string]poly{} + for _, p := range les { + p, _ := tighten(p, false) + if c, ok := p.isConst(); ok { + if c > 0 { + return true + } + continue + } + k := key(p) + if old, ok := best[k]; !ok || p[""] > old[""] { + best[k] = p + } + } + if len(best) == 0 { + return false + } + if len(best) > maxConstraints { + pr.gaveUp = true + return false + } + les = les[:0] + keys := make([]string, 0, len(best)) + for k := range best { + keys = append(keys, k) } + sort.Strings(keys) + for _, k := range keys { + les = append(les, best[k]) + } + // Eliminate the variable that multiplies the fewest pairs. + up, down := map[string]int{}, map[string]int{} + for _, p := range les { + for m, c := range p { + if m == "" { + continue + } + if c > 0 { + up[m]++ + } else { + down[m]++ + } + } + } + v, cost := "", -1 + for _, p := range les { + for m := range p { + if m == "" { + continue + } + if c := up[m] * down[m]; cost < 0 || c < cost || c == cost && m < v { + v, cost = m, c + } + } + } + var next, pos, neg []poly + for _, p := range les { + switch c := p[v]; { + case c > 0: + pos = append(pos, p) + case c < 0: + neg = append(neg, p) + default: + next = append(next, p) + } + } + for _, p := range pos { + for _, n := range neg { + a, b := p[v], -n[v] + if big(a) || big(b) { + pr.gaveUp = true + return false + } + c := poly{}.add(p, b).add(n, a) + for _, k := range c { + if big(k) { + pr.gaveUp = true + return false + } + } + next = append(next, c) + } + } + les = next } - return "" } ``` -From gopls's side, litgo is the client. Requests are numbered, and `pending` -holds what to do with each answer until the answer arrives. The editor might -cancel a request before then, so litgo also keeps the editor's own number for -the request. +```go +// squeeze finds the equalities hidden in pairs of inequalities: p <= 0 and +// -p <= 0. +func squeeze(eqs, les []poly) ([]poly, []poly) { + byKey := map[string]bool{} + for _, p := range les { + byKey[p.String()] = true + } + for _, p := range les { + if len(p) > 0 && byKey[poly{}.add(p, -1).String()] { + eqs = append(eqs, p) + } + } + return eqs, les +} + +// unfoldProducts uses an equality x = e inside the products x occurs in. +// Elimination cannot do that, because to it x*y is one unknown and x another; +// but i = n + 1 is exactly what turns i*i into something about n. +func unfoldProducts(eqs, les []poly) ([]poly, []poly) { + for round := 0; round < 8; round++ { + x, by := "", poly(nil) + for _, e := range eqs { + for m, c := range e { + if m == "" || strings.Contains(m, sep) || c != 1 && c != -1 || !inProduct(m, eqs, les) { + continue + } + rest := poly{}.add(e, -c) // x = rest, once x's own term is taken out + delete(rest, m) + if mentions(rest, m) { + continue + } + if x == "" || m < x { + x, by = m, rest + } + } + } + if x == "" { + break + } + for i := range eqs { + eqs[i] = substitute(eqs[i], x, by) + } + for i := range les { + les[i] = substitute(les[i], x, by) + } + eqs = append(eqs, poly{x: 1}.add(by, -1)) // what substituting made trivial + } + return eqs, les +} -gopls adds two goroutines to the server, and no locks. One writes to gopls's -standard input and one reads its standard output. The reader sends each message -to the server's loop over `msgs`. The loop does all the bookkeeping and runs -every `done` function, so an answer from gopls gets translated by the same -goroutine that owns the documents and the overlays. +func mentions(p poly, x string) bool { + for m := range p { + for _, a := range strings.Split(m, sep) { + if a == x { + return true + } + } + } + return false +} -```mermaid -sequenceDiagram - participant L as the loop - participant W as writer - participant G as gopls - participant R as reader - L->>W: out - W->>G: standard input - G->>R: standard output - R->>L: msgs +func inProduct(x string, lists ...[]poly) bool { + for _, list := range lists { + for _, p := range list { + for m := range p { + if strings.Contains(m, sep) && mentions(poly{m: 1}, x) { + return true + } + } + } + } + return false +} + +func substitute(p poly, x string, by poly) poly { + if !mentions(p, x) { + return p + } + out := poly{} + for m, c := range p { + term := poly{"": c} + for _, a := range strings.Split(m, sep) { + switch { + case a == x: + term = term.mul(by) + case a != "": + term = term.mul(poly{a: 1}) + } + } + out = out.add(term, 1) + } + return out +} ``` ```go -// gopls is the Go language server, run as a child process. Like the rest of -// the server's state, it belongs to the loop in Serve. -type gopls struct { - cmd *exec.Cmd - in io.WriteCloser - out chan []byte // to the goroutine that writes to gopls - msgs chan *message // from the goroutine that reads from it - next int - pending map[int]call - ids map[string]int // the editor's id for a request in flight β†’ ours - dead bool - caps map[string]any // what gopls said it can do -} +func big(v int64) bool { return v > 1<<30 || v < -(1<<30) } -// call is a request in flight. done gets the answer, or nil if gopls died. -type call struct { - done func(*message) - editor string +func key(p poly) string { + ms := make([]string, 0, len(p)) + for m := range p { + if m != "" { + ms = append(ms, m) + } + } + sort.Strings(ms) + var b strings.Builder + for _, m := range ms { + fmt.Fprintf(&b, "%d*%s;", p[m], m) + } + return b.String() } -func (g *gopls) write(m *message) { g.out <- frame(m) } +// tighten divides a constraint by the gcd of its coefficients. Over the +// integers the constant of an inequality rounds up, and an equality whose +// constant does not divide has no solution. +func tighten(p poly, eq bool) (poly, bool) { + var g int64 + for m, c := range p { + if m != "" { + g = gcd(g, c) + } + } + if g == 0 { + c := p[""] + return p, !eq || c == 0 + } + if g == 1 { + return p, true + } + out := poly{} + for m, c := range p { + if m != "" { + out[m] = c / g + } + } + c := p[""] + if eq { + if c%g != 0 { + return p, false + } + c /= g + } else { + c = ceilDiv(c, g) + } + if c != 0 { + out[""] = c + } + return out, true +} -func (g *gopls) notify(method string, params any) { - raw, _ := json.Marshal(params) - g.write(&message{Method: method, Params: raw}) +func gcd(a, b int64) int64 { + if a < 0 { + a = -a + } + if b < 0 { + b = -b + } + for b != 0 { + a, b = b, a%b + } + return a } -// request asks gopls something on behalf of the editor's request id (nil if -// the question is litgo's own). -func (g *gopls) request(id *json.RawMessage, method string, params any, done func(*message)) { - if g.dead { - done(nil) - return +func ceilDiv(a, b int64) int64 { + q := a / b + if a%b != 0 && (a < 0) == (b < 0) { + q++ } - raw, _ := json.Marshal(params) - c := call{done: done} - g.next++ - if id != nil { - c.editor = string(*id) - g.ids[c.editor] = g.next + return q +} + +// pivot picks the variable of an equality to solve for: one with coefficient +// Β±1 if there is one, so that substituting it loses nothing over the integers. +func pivot(e poly) string { + v := "" + for m, c := range e { + if m == "" { + continue + } + unit := c == 1 || c == -1 + bestUnit := v != "" && (e[v] == 1 || e[v] == -1) + if v == "" || unit && !bestUnit || unit == bestUnit && m < v { + v = m + } } - g.pending[g.next] = c - ours := json.RawMessage(strconv.Itoa(g.next)) - g.write(&message{ID: &ours, Method: method, Params: raw}) + return v } -// cancel passes on the editor's loss of interest in one of its requests. -func (g *gopls) cancel(params json.RawMessage) { - var p struct { - ID json.RawMessage `json:"id"` +// eliminate removes v from p using the equality e = 0. +func eliminate(p, e poly, v string) poly { + c := p[v] + if c == 0 { + return p } - json.Unmarshal(params, &p) - if n, ok := g.ids[string(p.ID)]; ok { - g.notify("$/cancelRequest", map[string]any{"id": n}) + a := e[v] + // |a|*p - sign(a)*c*e has no v, and scaling an inequality by |a| > 0 is fine. + if a < 0 { + return poly{}.add(p, -a).add(e, c) } + return poly{}.add(p, a).add(e, -c) } ``` -gopls sends three kinds of message. An answer goes to whoever is waiting for -it. A request of its own (to register a capability, or to read configuration) -gets `null` back, and gopls takes that to mean "use your defaults". Of its -notifications, litgo only passes error messages on. The progress reports are -about files you never see. The diagnostics would repeat the ones the type -checker from the last chapter already placed, or in the worst case contradict -them. And the one warning gopls has is that someone is editing a generated -file. +## Tests -The reader drops the notifications nobody wants before they get to the loop, -because gopls sends a lot of them. It closes `msgs` when gopls hangs up. The -loop then marks gopls as dead and tells everyone who was waiting for an answer -that there won't be one. + + + + +A verifier has two ways to be wrong, and they are not equally bad. If it fails +to prove something true, somebody strengthens an invariant. If it proves +something false, it is worse than useless. So every program here that is +proved is also broken, one small change at a time, and each broken version +has to be rejected for the right reason. ```go -// listen reads what gopls sends until it hangs up or the server stops. -func (g *gopls) listen(out io.Reader, quit <-chan struct{}) { - defer close(g.msgs) - defer g.cmd.Wait() - r := bufio.NewReader(out) - for { - m, err := read(r) - if err != nil { - return - } - if m.ID == nil && m.Method != "window/showMessage" { - continue +func verifySrc(t *testing.T, src string) ([]Finding, Summary) { + t.Helper() + return Verify([]Source{{"t.go", []byte("package p\n" + src)}}) +} + +func expectProved(t *testing.T, src string, n int) { + t.Helper() + fs, sum := verifySrc(t, src) + for _, f := range fs { + if !f.Warning { + t.Errorf("unexpected: %d:%d %s", f.Line+1, f.Col+1, f.Message) } - select { - case g.msgs <- m: - case <-quit: + } + if sum.Proved != n { + t.Errorf("proved %d of %d, want %d", sum.Proved, sum.Functions, n) + } +} + +func expectFail(t *testing.T, src, want string) { + t.Helper() + fs, _ := verifySrc(t, src) + for _, f := range fs { + if !f.Warning && strings.Contains(f.Message, want) { return } } + t.Errorf("no error containing %q in %v", want, fs) } -// heard is where messages from gopls arrive. Without a gopls it is nil, and -// a nil channel never delivers anything. -func (s *server) heard() <-chan *message { - if s.gopls == nil || s.gopls.dead { - return nil +const divide = ` +//@ Requires n >= 0 ^ d > 0 +func Divide(n, d int) (q, r int) { + r = n + //@ Invariant n = q*d + r ^ r >= 0 + //@ Variant r + for r >= d { + q++ + r -= d } - return s.gopls.msgs + return q, r } +//@ Ensures n = q*d + r +//@ Ensures 0 <= r < d +` -// hear handles one message from gopls. -func (g *gopls) hear(m *message, s *server) { - switch { - case m.Method == "" && m.ID != nil: - n, _ := strconv.Atoi(string(*m.ID)) - c, ok := g.pending[n] - delete(g.pending, n) - delete(g.ids, c.editor) - if ok { - c.done(m) - } - case m.ID != nil: - g.write(&message{ID: m.ID, Result: json.RawMessage("null")}) - case m.Method == "window/showMessage": - var p struct { - Type int `json:"type"` +func TestDivide(t *testing.T) { expectProved(t, divide, 1) } + +// gofmt moves annotations around: //@ becomes // @ in a doc comment, and an +// Ensures gets a blank line above it. The proof has to come out the same, and +// so does the refusal. +func TestSurvivesGofmt(t *testing.T) { + for _, src := range []string{divide, search} { + formatted, err := format.Source([]byte("package p\n" + src)) + if err != nil { + t.Fatal(err) } - if json.Unmarshal(m.Params, &p); p.Type == 1 { - s.write(&message{Method: m.Method, Params: m.Params}) + text := strings.TrimPrefix(string(formatted), "package p\n") + if !strings.Contains(text, "// @ Requires") || !strings.Contains(text, "}\n\n//@ Ensures") { + t.Fatalf("gofmt no longer does what this test is about:\n%s", text) } + expectProved(t, text, 1) } + formatted, _ := format.Source([]byte("package p\n" + strings.Replace(divide, "r -= d", "r -= 1", 1))) + expectFail(t, strings.TrimPrefix(string(formatted), "package p\n"), "Invariant is not maintained") } -// hangUp is what happens when gopls is gone: nobody gets an answer any more. -func (g *gopls) hangUp() { - g.dead = true - orphans := g.pending - g.pending = map[int]call{} - for _, c := range orphans { - c.done(nil) +// A contract may also be written whole above its function. +func TestContractAbove(t *testing.T) { + const above = ` +func First() (x int) { return 1 } + +//@ Requires n >= 0 +//@ Ensures r = n + 1 +func Next(n int) (r int) { return n + 1 } +` + expectProved(t, above, 1) + expectFail(t, strings.Replace(above, "return n + 1", "return n", 1), "Ensures is not proved") +} + +// What the verifier cannot follow, it refuses. +func TestRefusals(t *testing.T) { + expectFail(t, ` +//@ Requires 0 <= i ^ i < |A| +func Set(A []int, i int) (x int) { + A[i] = 0 + return A[i] +} +//@ Ensures x = 0 +`, "only variables can be assigned to") + expectFail(t, ` +func scramble(A []int) { + for i := range A { + A[i] = 0 + } +} + +//@ Requires |A| > 0 ^ A[0] = 1 +func Keeps(A []int) (x int) { + scramble(A) + return A[0] +} +//@ Ensures x = 1 +`, "scramble has no annotations") + expectFail(t, ` +//@ Requires n >= 0 +func Bad(n int) (x int) { + //@ Assert n + 1 + return n +} +`, "this has to be a formula") + expectFail(t, ` +//@ Requires n >= 0 +func Bad(n int) (x int) { + //@ Variant n > 0 + for n > 0 { + n-- } + return n +} +`, "this has to be an integer expression") + fs, sum := verifySrc(t, ` +type Counter interface { + //@ Requires self.Value >= 0 + Increment() } -``` -litgo starts gopls while the editor is waiting for the answer to its -`initialize`, because what litgo can promise the editor depends on what gopls -promises litgo. litgo tells gopls what the editor told litgo (the workspace, -and what the editor can display, like snippets in completions and Markdown in -hovers). It leaves out the capabilities that would make gopls send requests of -its own. +//@ Requires n >= 0 +func Id(n int) (x int) { return n } +//@ Ensures x = n +`) + if sum.Proved != 1 || len(fs) != 1 || !fs[0].Warning { + t.Errorf("findings = %+v, sum = %+v", fs, sum) + } +} -The loop is the goroutine that runs `start`, so it can't be in its `select` -while it waits for gopls to answer. `start` receives from `msgs` itself until -the answer has come. +func TestDivideWrong(t *testing.T) { + expectFail(t, strings.Replace(divide, "r -= d", "r -= 1", 1), "Invariant is not maintained") + expectFail(t, strings.Replace(divide, "d > 0", "d >= 0", 1), "Variant is not shown to decrease") + expectFail(t, strings.Replace(divide, "0 <= r < d", "0 <= r <= d - 2", 1), "Ensures is not proved") +} -```go -// start runs gopls and introduces litgo to it the way the editor introduced -// itself to litgo. On any failure s.gopls stays nil. -func (s *server) start(initialize json.RawMessage) { - bin := findGopls(s.goplsBin) - if bin == "" { - return - } - cmd := exec.Command(bin) - in, _ := cmd.StdinPipe() - out, _ := cmd.StdoutPipe() - if cmd.Start() != nil { - return +func TestGauss(t *testing.T) { + expectProved(t, ` +//@ Requires n >= 0 +func Sum(n int) (s int) { + //@ Invariant 0 <= i <= n + 1 ^ 2*s = i*(i-1) + //@ Variant n + 1 - i + for i := 0; i <= n; i++ { + s += i } - g := &gopls{cmd: cmd, in: in, msgs: make(chan *message), pending: map[int]call{}, ids: map[string]int{}} - g.out, _ = writer(in) - go g.listen(out, s.quit) + return s +} +//@ Ensures 2*s = n*(n+1) +`, 1) +} - var p map[string]any - if json.Unmarshal(initialize, &p); p == nil { - p = map[string]any{} +func TestSqrt(t *testing.T) { + expectProved(t, ` +//@ Requires n >= 0 +func Isqrt(n int) (r int) { + //@ Invariant r >= 0 ^ r*r <= n + //@ Variant n - r*r + for (r+1)*(r+1) <= n { + r++ } - theirs, _ := p["capabilities"].(map[string]any) - p["capabilities"] = map[string]any{"textDocument": theirs["textDocument"]} - var reply *message - answered := false - g.request(nil, "initialize", p, func(m *message) { reply, answered = m, true }) - timeout := time.After(30 * time.Second) - for !answered { - select { - case m, ok := <-g.msgs: - if ok { - g.hear(m, s) - } else { - g.hangUp() // which answers the request, with nil - } - case <-timeout: - answered = true + return r +} +//@ Ensures r*r <= n ^ n < (r+1)*(r+1) +`, 1) +} + +const search = ` +//@ Predicate Sorted(A) ::= Forall i in [0, |A|) : Forall j in [i, |A|) : A[i] <= A[j] + +//@ Requires Sorted(A) +func Search(A []int, x int) (at int, found bool) { + lo, hi := 0, len(A) + //@ Invariant 0 <= lo <= hi <= |A| + //@ Invariant A[0:lo) < x ^ x < A[hi:|A|) + //@ Variant hi - lo + for lo < hi { + mid := lo + (hi-lo)/2 + if A[mid] == x { + return mid, true + } else if A[mid] < x { + lo = mid + 1 + } else { + hi = mid } } - if result, _ := reply.result().(map[string]any); result != nil { - g.caps, _ = result["capabilities"].(map[string]any) - g.notify("initialized", map[string]any{}) - s.gopls = g - return - } - g.kill() + return 0, false } +//@ Ensures found -> 0 <= at < |A| ^ A[at] = x +//@ Ensures ~found -> x <> A[:] +` -// kill stops gopls, and the goroutine that writes to it. -func (g *gopls) kill() { - close(g.out) - g.in.Close() - g.cmd.Process.Kill() +func TestSearch(t *testing.T) { expectProved(t, search, 1) } + +func TestSearchWrong(t *testing.T) { + expectFail(t, strings.Replace(search, "lo = mid + 1", "lo = mid", 1), "Variant is not shown to decrease") + expectFail(t, strings.Replace(search, "hi = mid\n", "hi = mid - 1\n", 1), "Invariant is not maintained") + expectFail(t, strings.Replace(search, "//@ Requires Sorted(A)\n", "", 1), "Invariant is not maintained") } -// result is the result of a successful answer, and nil otherwise. -func (m *message) result() any { - if m == nil || m.Error != nil { - return nil - } - return m.Result +func TestSafety(t *testing.T) { + expectFail(t, ` +//@ Requires i >= 0 +func At(A []int, i int) (x int) { return A[i] } +`, "index may be out of range") + expectFail(t, ` +//@ Requires a >= 0 +func Div(a, b int) (x int) { return a / b } +`, "divisor may be zero") } -func (s *server) stop() { - if g := s.gopls; g != nil { - g.kill() +func TestCallsAndRecursion(t *testing.T) { + expectProved(t, ` +//@ Requires n >= 0 +//@ Measure n +func Fact(n int) (f int) { + //@ BaseCase n = 0 + if n == 0 { + return 1 } + return n * Fact(n-1) } -``` - -## Overlays - -gopls has to see the latest text before it gets asked about it. A completion -request comes right after the keystroke that triggered it, sooner than the -delay that diagnostics wait for. So tangling doesn't wait. Whoever needs the -tangled files, whether that is the analysis or a request, calls `sync`, and -`sync` tangles at most once per version of the text. Both of them run on the -loop, so they can't get in each other's way. A document with a broken -chunk structure still gets tangled as well as it can be, because you are in the -middle of something and would still like completions. +//@ Ensures f >= 1 -```go -// overlay is a tangled file as gopls was last shown it. -type overlay struct { - owner string // URI of the document it is tangled from - version int - text string +//@ Requires a >= 0 ^ b >= 0 +func Use(a, b int) (x int) { + return Fact(a) + Fact(b) } - -// sync tangles a document, unless that has been done for this version of -// its text, and shows gopls the result. It returns a nil result for a -// document that is not open. -func (s *server) sync(uri string) (*lit.Result, int) { - d := s.docs[uri] - if d == nil { - return nil, -1 - } - if d.res == nil || d.tangled != d.version { - d.res, d.tangled = lit.Parse(path(uri), []byte(d.text)).Tangle(), d.version - s.overlay(uri, d.res) +//@ Ensures x >= 2 +`, 2) + expectFail(t, ` +//@ Requires n >= 0 +//@ Measure n +func F(n int) (f int) { + if n == 0 { + return 1 } - return d.res, d.version + return F(n) } - -// forget closes what a closed document tangled to. -func (s *server) forget(uri string) { - s.overlay(uri, nil) +`, "Measure is not shown to get smaller") } -``` -A document like this one tangles to dozens of files, and a keystroke only -changes one of them. litgo only sends that one. +func TestPrimesAndPreserves(t *testing.T) { + expectProved(t, ` +//@ Requires a = a +func P(a int) (x int) { + x = 0 + x = x + 1 + x = x + 2 + //@ Assert x = 0 ^ x' = 1 ^ x'' = 3 + y := a + //@ Preserves x + y >= a + x++ + y-- + x++ + return x + y +} +//@ Ensures x >= a +`, 1) +} -```go -// overlay brings gopls up to date with what a document tangles to: new -// files are opened, changed ones changed, and ones no longer written closed. -func (s *server) overlay(uri string, res *lit.Result) { - g := s.gopls - if g == nil { - return - } - live := map[string]bool{} - for _, o := range outputs(res) { - live[o.Path] = true - text := string(o.Bytes()) - ov := s.overlays[o.Path] - switch { - case ov == nil: - s.overlays[o.Path] = &overlay{owner: uri, version: 1, text: text} - g.notify("textDocument/didOpen", map[string]any{ - "textDocument": map[string]any{"uri": fileURI(o.Path), "languageId": "go", "version": 1, "text": text}}) - case ov.text != text: - ov.owner, ov.text, ov.version = uri, text, ov.version+1 - g.notify("textDocument/didChange", map[string]any{ - "textDocument": map[string]any{"uri": fileURI(o.Path), "version": ov.version}, - "contentChanges": []any{map[string]any{"text": text}}}) +// The examples are documentation, and documentation that claims a proof had +// better have one. +func TestExamples(t *testing.T) { + for name, want := range map[string]int{"isqrt": 1, "vego-tutorial": 15, "heap": 10} { + src, err := os.ReadFile("../../examples/" + name + ".lit.md") + if err != nil { + t.Skip(err) } - } - for p, ov := range s.overlays { - if ov.owner == uri && !live[p] { - delete(s.overlays, p) - g.notify("textDocument/didClose", map[string]any{ - "textDocument": map[string]any{"uri": fileURI(p)}}) + diags, sum := Prove(lit.Parse(name+".lit.md", src).Tangle()) + for _, d := range diags { + if d.Severity == lit.SevError { + t.Errorf("%s:%d: %s", name, d.Line+1, d.Message) + } + } + if sum.Proved != want || sum.Functions != want { + t.Errorf("%s: proved %d of %d, want %d", name, sum.Proved, sum.Functions, want) } } } -// outputs lists the Go files of a result, which may be nil. -func outputs(res *lit.Result) []*lit.Output { - var out []*lit.Output - if res != nil { - for _, o := range res.Files { - if o.File.Lang == "go" { - out = append(out, o) - } +// A failure is reported where the annotation is in the Markdown, and the +// line it mentions is a line of the Markdown too. +// litgo proves the arithmetic of its own source map. The test names the two +// functions, so that deleting their annotations does not pass for a proof. +func TestLitgoProvesItself(t *testing.T) { + src, err := os.ReadFile("../../litgo.lit.md") + if err != nil { + t.Skip(err) + } + diags, sum := Prove(lit.Parse("litgo.lit.md", src).Tangle()) + for _, d := range diags { + if d.Severity == lit.SevError { + t.Errorf("litgo.lit.md:%d: %s", d.Line+1, d.Message) } } - return out + proved := strings.Join(sum.Names, " ") + if !strings.Contains(proved, "Seg.out") || !strings.Contains(proved, "Seg.src") || sum.Proved != sum.Functions { + t.Errorf("proved %d of %d: %s", sum.Proved, sum.Functions, proved) + } } -func fileURI(path string) string { - return (&url.URL{Scheme: "file", Path: filepath.ToSlash(path)}).String() +func TestProveMapsToTheDocument(t *testing.T) { + doc := "\n# Halving\n\n```go\nfunc Halve(n int) (h int) {\n\t// <>\n}\n//@ Ensures 2*h = n\n```\n\n" + + "\n```go\nreturn n / 2\n```\n" + diags, sum := Prove(lit.Parse("halve.lit.md", []byte(doc)).Tangle()) + if sum.Proved != 0 || len(diags) != 1 { + t.Fatalf("diags = %+v, sum = %+v", diags, sum) + } + d := diags[0] + if d.Line != 7 || d.Source != "vego" || !strings.Contains(d.Message, "the return at line 13") { + t.Errorf("diag = %+v", d) + } } -``` -When gopls names a file in an answer, it might be one of these overlays. -`output` finds it, along with the document it came from, and prefers the -tangling that was current when the question was asked. +func TestOneLine(t *testing.T) { + expectProved(t, ` +type Seg struct{ OutCol, Len, SrcLine, SrcCol int } -```go -func (s *server) output(uri string, asked *lit.Result) (o *lit.Output, doc *lit.Doc, owner string) { - p := filepath.Clean(path(uri)) - res := asked - if ov := s.overlays[p]; ov != nil { - owner = ov.owner - if d := s.docs[owner]; d != nil && (asked == nil || asked.Doc.Path != path(owner)) { - res = d.res - } - } - if owner != "" { - for _, o := range outputs(res) { - if filepath.Clean(o.Path) == p { - return o, res.Doc, owner - } - } - } - return nil, nil, "" +//@ Requires s.SrcCol <= col ^ col <= s.SrcCol + s.Len +func (s Seg) out(col int) (c int) { return s.OutCol + col - s.SrcCol } +//@ Ensures s.OutCol <= c ^ c <= s.OutCol + s.Len +//@ Ensures s.src(c) = col + +//@ Requires s.OutCol <= col ^ col <= s.OutCol + s.Len +func (s Seg) src(col int) (c int) { return s.SrcCol + col - s.OutCol } +//@ Ensures s.SrcCol <= c ^ c <= s.SrcCol + s.Len +//@ Ensures s.out(c) = col +`, 2) } ``` -## Questions +# The command -These are the requests that ask about a position in a document, each with the -capability that announces it. litgo offers the editor whatever gopls offers -litgo, as long as it is in this list. It always offers four of them, because it -can answer those itself when the position is on a chunk. + + + + +Half of the commands are for people. The other half are the editor protocol, +where the document arrives on standard input and JSON goes out. The document +comes in on standard input because the buffer being edited is rarely the same +as the file on disk. ```go -var positional = map[string]string{ - "textDocument/hover": "hoverProvider", - "textDocument/definition": "definitionProvider", - "textDocument/typeDefinition": "typeDefinitionProvider", - "textDocument/implementation": "implementationProvider", - "textDocument/references": "referencesProvider", - "textDocument/documentHighlight": "documentHighlightProvider", - "textDocument/completion": "completionProvider", - "textDocument/signatureHelp": "signatureHelpProvider", - "textDocument/prepareRename": "renameProvider", - "textDocument/rename": "renameProvider", -} +const usage = `litgo β€” literate Go -func (s *server) capabilities() map[string]any { - c := map[string]any{} - if s.gopls != nil { - for _, name := range positional { - if v, ok := s.gopls.caps[name]; ok { - c[name] = v - } - } - c["workspaceSymbolProvider"] = s.gopls.caps["workspaceSymbolProvider"] - } - c["hoverProvider"], c["definitionProvider"], c["referencesProvider"] = true, true, true - c["renameProvider"] = map[string]any{"prepareProvider": true} - return c -} -``` + litgo tangle [--force] [--stdout] FILE.lit.md... write the Go source + litgo run [--vet] [--no-prove] [--timeout 30s] FILE [-- ARGS] + tangle, prove, compile and run + litgo weave [--to pdf|html|typ|md] [-o OUT] FILE write a PDF through Typst, or a web page + litgo check FILE.lit.md... chunk, syntax, type and proof errors; writes nothing + litgo prove [--json] FILE.lit.md... check the //@ annotations: contracts, invariants, termination + litgo cat FILE.lit.md print with math and diagrams rendered + litgo fmt [-w] FILE.lit.md gofmt the code blocks -```go -// proxy handles the requests that gopls has a say in, and reports whether -// the message was one. -func (s *server) proxy(m *message) bool { - switch { - case positional[m.Method] != "" && m.ID != nil: - s.ask(m) - case m.Method == "$/cancelRequest": - if s.gopls != nil { - s.gopls.cancel(m.Params) +Editor protocol (document on stdin, JSON on stdout): + litgo analyze [--path P] [--upright] + litgo untangle --start L --end L [--start-col C --end-col C] [--name N] + litgo rename --from NAME --to NAME + litgo run --json FILE one JSON event per line + litgo lsp [--gopls PATH|off] language server on stdin/stdout: errors as you type, + and gopls for the Go in the code blocks + +Positions in the editor protocol are 0-based; columns are bytes. +` + +const VERSION = "0.2.1" + +func main() { + if len(os.Args) < 2 { + fmt.Fprint(os.Stderr, usage) + os.Exit(2) + } + cmd, args := os.Args[1], os.Args[2:] + switch cmd { + case "tangle": + os.Exit(cmdTangle(args)) + case "run": + os.Exit(cmdRun(args)) + case "weave": + os.Exit(cmdWeave(args)) + case "check": + os.Exit(cmdCheck(args)) + case "prove": + os.Exit(cmdProve(args)) + case "cat": + os.Exit(cmdCat(args)) + case "fmt": + os.Exit(cmdFmt(args)) + case "analyze": + os.Exit(cmdAnalyze(args)) + case "untangle": + os.Exit(cmdUntangle(args)) + case "version": + os.Exit(cmdVersion(args)) + case "rename": + os.Exit(cmdRename(args)) + case "lsp": + fs := flag.NewFlagSet("lsp", flag.ExitOnError) + gopls := fs.String("gopls", "", "the gopls to put behind the server; found when empty, none when \"off\"") + fs.Parse(args) + if err := lsp.Serve(os.Stdin, os.Stdout, *gopls); err != nil { + fmt.Fprintln(os.Stderr, "litgo lsp:", err) + os.Exit(1) } - case m.Method == "workspace/symbol" && m.ID != nil && s.gopls != nil: - // No position to translate on the way in, but the symbols found - // may well be in tangled files. - var params any - json.Unmarshal(m.Params, ¶ms) - s.gopls.request(m.ID, m.Method, params, func(reply *message) { - s.relay(m, reply, &translation{s: s}) - }) + case "help", "-h", "--help": + fmt.Print(usage) default: - return false + fmt.Fprintf(os.Stderr, "litgo: unknown command %q\n\n%s", cmd, usage) + os.Exit(2) } - return true } ``` -`Locate` translates the position in the question. If the position is in prose, -or in a Lua block, or on a line that an exclude pattern drops, then it isn't -anywhere in a Go file and the answer is `null`. Every one of these requests -allows `null`, and editors accept it silently. - ```go -// position is the protocol's: a line, and a column in the agreed units. -type position struct { - Line int `json:"line"` - Character int `json:"character"` -} - -func (s *server) ask(m *message) { - var p struct { - TextDocument struct { - URI string `json:"uri"` - } `json:"textDocument"` - Position position `json:"position"` - NewName string `json:"newName"` - } - json.Unmarshal(m.Params, &p) - uri := p.TextDocument.URI - res, _ := s.sync(uri) - if res == nil || p.Position.Line < 0 || p.Position.Line >= len(res.Doc.Lines) { - s.respond(m, nil) - return - } - line := p.Position.Line - col := offset(s.utf8, res.Doc.Lines[line], p.Position.Character) - if mention := res.Doc.MentionAt(line, col); mention != nil { - // <> - return +func load(path string) (*lit.Doc, error) { + abs, err := filepath.Abs(path) + if err != nil { + return nil, err } - o, outLine, outCol, ok := res.Locate(line, col) - if !ok || o.File.Lang != "go" || s.gopls == nil { - s.respond(m, nil) - return + src, err := os.ReadFile(abs) + if err != nil { + return nil, err } - var params map[string]any - json.Unmarshal(m.Params, ¶ms) - params["textDocument"] = map[string]any{"uri": fileURI(o.Path)} - params["position"] = position{outLine, units(false, o.Lines[outLine].Text, outCol)} - s.gopls.request(m.ID, m.Method, params, func(reply *message) { - s.relay(m, reply, &translation{s: s, res: res, out: o, uri: uri}) - }) + return lit.Parse(abs, src), nil } -// respond answers a request; a nil result is the protocol's null. -func (s *server) respond(to *message, result any) { - if result == nil { - result = json.RawMessage("null") +func stdinDoc(path string) *lit.Doc { + src, _ := io.ReadAll(os.Stdin) + if path == "" { + path = "stdin.lit.md" } - s.write(&message{ID: to.ID, Result: result}) + return lit.Parse(path, src) } -``` - -gopls counts columns in UTF-16 code units and doesn't offer anything else. That -is why litgo stops agreeing to bytes with the editor when gopls is there. It -means positions in files litgo knows nothing about (the standard library, a -dependency) can pass through untouched. litgo has the text of the documents and -the tangled files, so it converts the columns for those. -```go -// offset converts a column in UTF-16 code units (or in bytes) to a byte -// offset in the line: the inverse of units. -func offset(utf8 bool, line string, n int) int { - if utf8 { - return min(max(n, 0), len(line)) - } - for i, r := range line { - if n <= 0 { - return i - } - n-- - if r >= 0x10000 { - n-- - } +func printJSON(v any) int { + enc := json.NewEncoder(os.Stdout) + enc.SetEscapeHTML(false) + if err := enc.Encode(v); err != nil { + fmt.Fprintln(os.Stderr, "litgo:", err) + return 1 } - return len(line) + return 0 } -``` - -## Questions about chunks -A chunk reference is a comment, so gopls has nothing to say about it. But the -questions still make sense, and litgo knows the answers. A reference is -*defined* by the blocks of its chunk, hovering shows those blocks, the -*references* to a chunk are its references, and renaming it is `Rename`. If you -ask for the definition of a definition, litgo answers with the uses, so one key -takes you back and forth. All of this works with or without gopls. +func jsonError(err error) int { + printJSON(map[string]string{"error": err.Error()}) + return 1 +} - -```go -result, err := s.mention(m.Method, uri, res.Doc, mention, p.NewName) -if err != nil { - s.write(&message{ID: m.ID, Error: &rpcError{requestFailed, err.Error()}}) -} else { - s.respond(m, result) +func report(diags []lit.Diag) (errors int) { + r := &reporter{seen: map[string]bool{}} + for _, d := range diags { + r.diag(d) + if d.Severity == lit.SevError { + errors++ + } + } + return errors } ``` ```go -const requestFailed = -32803 - -func (s *server) mention(method, uri string, doc *lit.Doc, m *lit.Mention, newName string) (any, error) { - span := func(line, col, end int) map[string]any { - line = min(line, len(doc.Lines)-1) - text := doc.Lines[line] - return map[string]any{"start": position{line, s.column(text, col)}, "end": position{line, s.column(text, end)}} - } - c := m.Chunk - var uses, blocks []any - for _, r := range c.Refs { - uses = append(uses, map[string]any{"uri": uri, "range": span(r.Line, r.Col, r.EndCol)}) - } - for _, b := range c.Blocks { - blocks = append(blocks, map[string]any{"uri": uri, "range": span(b.First(), 0, 0)}) +func cmdTangle(args []string) int { + fs := flag.NewFlagSet("tangle", flag.ExitOnError) + force := fs.Bool("force", false, "overwrite files litgo did not generate") + stdout := fs.Bool("stdout", false, "print instead of writing") + fs.Parse(args) + if fs.NArg() == 0 { + fmt.Fprintln(os.Stderr, "litgo tangle: no input files") + return 2 } - switch method { - case "textDocument/definition": - if m.Ref { - return blocks, nil - } - return uses, nil - case "textDocument/references": - return uses, nil - case "textDocument/hover": - if !m.Ref || len(c.Blocks) == 0 { - return nil, nil - } - text := "⟨" + c.Name + "⟩ ≑\n" - for _, b := range c.Blocks { - text += "```" + b.Lang + "\n" + strings.Join(doc.Lines[b.First():b.Last()+1], "\n") + "\n```\n" - } - return map[string]any{"contents": map[string]any{"kind": "markdown", "value": text}, "range": span(m.Line, m.Col, m.EndCol)}, nil - case "textDocument/prepareRename": - return map[string]any{"range": span(m.Line, m.Col, m.EndCol), "placeholder": c.Name}, nil - case "textDocument/rename": - edits, err := doc.Rename(c.Name, newName) + status := 0 + for _, path := range fs.Args() { + doc, err := load(path) if err != nil { - return nil, err + fmt.Fprintln(os.Stderr, "litgo:", err) + status = 1 + continue } - var changes []any - for _, e := range edits { // Rename replaces lines one for one - changes = append(changes, map[string]any{"range": span(e.Start, 0, len(doc.Lines[e.Start])), "newText": e.Lines[0]}) + res := doc.Tangle() + if report(res.Diags) > 0 { + status = 1 + continue } - return map[string]any{"changes": map[string]any{uri: changes}}, nil - } - return nil, nil -} -``` - -## Answers - -The answers vary as much as the questions do, but the positions in them only -come in a few shapes. So litgo does the translation on the JSON, without a type -for every kind of answer: - -* A *location* is a `uri` with a `range`, and a *location link* has a - `targetUri`. If the file is a tangled one, both get rewritten to point at the - document. Otherwise it is a file gopls found on disk, and it is left alone. -* A bare *range* anywhere else (the word a hover is about, an occurrence to - highlight) is in the file the question was about. -* A *text edit* is a range with a `newText`, and it gets the careful treatment - described below. - -A chunk that is used twice gets tangled twice, and gopls finds a reference in -each copy. Both translate back to the same place, so litgo removes the -duplicates from lists. - -```go -// translation brings the answer to one question back to the document. -type translation struct { - s *server - res *lit.Result // the tangling gopls was asked about, - out *lit.Output // the file in it that the question was about, - uri string // and the document -} - -func (t *translation) value(v any) any { - switch v := v.(type) { - case []any: - if len(v) > 0 && t.out != nil && isEdit(asMap(v[0])) { - return t.edits(t.out, t.res.Doc, v) - } - out, seen := []any{}, map[string]bool{} - for _, e := range v { - if e = t.value(e); e == nil { + for _, o := range res.Files { + if *stdout { + if len(res.Files) > 1 { + fmt.Printf("==> %s <==\n", relPath(o.Path)) + } + os.Stdout.Write(o.Bytes()) continue } - if key := positioned(e); key == "" || !seen[key] { - out = append(out, e) - seen[key] = true + if err := writeTangled(o, *force); err != nil { + fmt.Fprintln(os.Stderr, "litgo:", err) + status = 1 + continue } + fmt.Fprintf(os.Stderr, "litgo: %s β†’ %s (%d lines)\n", relPath(doc.Path), relPath(o.Path), len(o.Lines)) } - return out - case map[string]any: - return t.object(v) } - return v + return status } -func (t *translation) object(v map[string]any) any { - if uri, ok := v["uri"].(string); ok && v["range"] != nil { - if o, doc, owner := t.s.output(uri, t.res); o != nil { - if v["range"], ok = t.span(o, doc, v["range"], false); !ok { - return nil - } - v["uri"] = owner - } - return v - } - if uri, ok := v["targetUri"].(string); ok { - if t.out != nil && v["originSelectionRange"] != nil { - v["originSelectionRange"], _ = t.span(t.out, t.res.Doc, v["originSelectionRange"], false) - } - if o, doc, owner := t.s.output(uri, t.res); o != nil { - for _, k := range []string{"targetRange", "targetSelectionRange"} { - if v[k], ok = t.span(o, doc, v[k], false); !ok { - return nil - } - } - v["targetUri"] = owner - } - return v - } - if t.out != nil && isEdit(v) { // on its own, as the textEdit of a completion - if t.edit(t.out, t.res.Doc, v) { - return v - } - return nil - } - if t.out != nil && isRange(v) { - r, _ := t.span(t.out, t.res.Doc, v, false) - return r +func cmdRun(args []string) int { + fs := flag.NewFlagSet("run", flag.ExitOnError) + o := runOptions{} + fs.BoolVar(&o.json, "json", false, "emit one JSON event per line") + fs.BoolVar(&o.force, "force", false, "overwrite files litgo did not generate") + fs.BoolVar(&o.vet, "vet", false, "also run go vet") + fs.BoolVar(&o.noProve, "no-prove", false, "run even if the //@ annotations are not proved") + fs.DurationVar(&o.timeout, "timeout", 0, "kill the program after this long") + fs.Parse(args) + if fs.NArg() == 0 { + fmt.Fprintln(os.Stderr, "litgo run: no input file") + return 2 } - for k, e := range v { - if e = t.value(e); e == nil { - delete(v, k) // in the protocol, null and absent are the same - } else { - v[k] = e - } + o.args = fs.Args()[1:] + if len(o.args) > 0 && o.args[0] == "--" { + o.args = o.args[1:] } - return v -} - -func isRange(v map[string]any) bool { - _, start := v["start"].(map[string]any) - _, end := v["end"].(map[string]any) - return start && end -} - -func isEdit(v map[string]any) bool { - _, ok := v["newText"].(string) - return ok -} - -func asMap(v any) map[string]any { - m, _ := v.(map[string]any) - return m -} - -func asList(v any) []any { - l, _ := v.([]any) - return l + return runFile(fs.Arg(0), o) } -// positioned gives a location, a highlight or an edit a key to be compared -// by, and anything else none. -func positioned(v any) string { - if m, ok := v.(map[string]any); ok && (m["range"] != nil || m["targetUri"] != nil) { - b, _ := json.Marshal(m) - return string(b) +func cmdWeave(args []string) int { + fs := flag.NewFlagSet("weave", flag.ExitOnError) + to := fs.String("to", "", "pdf, html, typ (Typst source) or md (woven Markdown); default pdf, or what -o ends in") + out := fs.String("o", "", "output file") + fs.Parse(args) + if fs.NArg() != 1 { + fmt.Fprintln(os.Stderr, "litgo weave: expected one input file") + return 2 } - return "" -} -``` - -A range gets translated one end at a time, exactly if possible. A range that is -only for looking at can fall back on `Map`'s approximation. For example, the -definition of a name in a generated `import` is close enough to the `imports:` -directive. A range to *replace* can't fall back like that. It also has to have -the same shape in the document as in the tangled file, and it doesn't if it -starts in a host line and ends in a chunk spliced into that line. - -```go -// span translates a range in a tangled file to the document it came from. -func (t *translation) span(o *lit.Output, doc *lit.Doc, v any, edit bool) (any, bool) { - r, _ := v.(map[string]any) - l1, c1, ok1 := outPosition(o, r["start"]) - l2, c2, ok2 := outPosition(o, r["end"]) - if !ok1 || !ok2 && edit { - return v, false + doc, err := load(fs.Arg(0)) + if err != nil { + fmt.Fprintln(os.Stderr, "litgo:", err) + return 1 } - if !ok2 { - l2, c2 = l1, c1 + format := *to + if format == "" { + format = strings.TrimPrefix(filepath.Ext(*out), ".") } - sl, sc, ok1 := o.Exact(l1, c1, false) - el, ec, ok2 := o.Exact(l2, c2, l1 != l2 || c1 != c2) - if edit && !(ok1 && ok2 && el-sl == l2-l1 && (l1 != l2 || ec-sc == c2-c1)) { - return v, false + if format == "" { + format = "pdf" } - if !ok1 { - if sl, sc, _, ok1 = o.Map(l1, c1); !ok1 { - return v, false + if *out == "" { + ext := map[string]string{"md": "woven.md"}[format] + if ext == "" { + ext = format } + *out = strings.TrimSuffix(strings.TrimSuffix(doc.Path, ".md"), ".lit") + "." + ext } - if !ok2 { - el, ec, _, ok2 = o.Map(l2, c2) + switch format { + case "pdf": + err = weave.PDF(doc, *out) + case "html": + err = weave.HTML(doc, *out) + case "typ": + var src string + if src, err = weave.Typst(doc); err == nil { + err = os.WriteFile(*out, []byte(src), 0o644) + } + case "md": + err = os.WriteFile(*out, []byte(weave.Markdown(doc)), 0o644) + default: + err = fmt.Errorf("cannot weave to %q: pdf, html, typ and md are what there is", format) } - if !ok2 || el < sl || el == sl && ec < sc { - el, ec = sl, sc + if err != nil { + fmt.Fprintln(os.Stderr, "litgo:", err) + return 1 } - return map[string]any{ - "start": position{sl, units(t.s.utf8, doc.Lines[sl], sc)}, - "end": position{el, units(t.s.utf8, doc.Lines[el], ec)}, - }, true + abs, _ := filepath.Abs(*out) + fmt.Println(abs) + return 0 } -// outPosition decodes a position in a tangled file, in bytes. -func outPosition(o *lit.Output, v any) (line, col int, ok bool) { - p, _ := v.(map[string]any) - l, ok1 := p["line"].(float64) - c, ok2 := p["character"].(float64) - if !ok1 || !ok2 || l < 0 || int(l) >= len(o.Lines) { - return 0, 0, false +func cmdCheck(args []string) int { + status, cache := 0, &check.Cache{} + for _, path := range args { + doc, err := load(path) + if err != nil { + fmt.Fprintln(os.Stderr, "litgo:", err) + status = 1 + continue + } + res := doc.Tangle() + diags := append(res.Diags, res.SyntaxCheck()...) + if len(diags) == len(res.Diags) { + diags = append(diags, check.Types(res, cache)...) + } + if len(diags) == len(res.Diags) { + proofs, _ := vego.Prove(res) + diags = append(diags, proofs...) + } + if report(diags) > 0 { + status = 1 + } } - return int(l), offset(false, o.Lines[int(l)].Text, int(c)), true + return status } ``` -An edit that can't be translated gets dropped, with one exception. Completing -`strings.ToUpper` in a file that doesn't import `strings` comes with edits to -the `import (` block, and litgo generates that block from the `imports:` -directive. The edits are useless as they are, because gopls sends the smallest -difference it can find, and between `"fmt"` and `"fmt"`, newline, `"strings"` -that is a few characters in the middle of a line. But it is easy to find out -what the edits are *for*. litgo carries them out on the tangled file and looks -at what the file imports afterwards that it didn't import before. `AddImport` -knows how to add that to a document. +`prove` is the part of `check` that is about the `//@` annotations, with a +count at the end, because after a proof the interesting news is good news and +silence would not tell it. It checks the structure and the syntax first, since +a proof about a program that does not parse is a proof about nothing. It does +not wait for the type checker, though: the verifier reads types off the +declarations, and a proof is worth having while an import is still missing. ```go -// edit translates a text edit in place, if it has a place in the document. -func (t *translation) edit(o *lit.Output, doc *lit.Doc, v map[string]any) bool { - ok := true - for _, k := range []string{"range", "insert", "replace"} { // a completion may offer both an insertion and a replacement - if r, has := v[k]; has && ok { - if l, _, _ := outPosition(o, asMap(r)["start"]); l < o.Header { - return false - } - v[k], ok = t.span(o, doc, r, true) - } +func cmdProve(args []string) int { + fs := flag.NewFlagSet("prove", flag.ExitOnError) + asJSON := fs.Bool("json", false, "print the findings and the count as JSON") + fs.Parse(args) + if fs.NArg() == 0 { + fmt.Fprintln(os.Stderr, "litgo prove: no input files") + return 2 } - return ok -} - -// edits translates a list of text edits. Those to the generated header -// become edits to the imports directive. -func (t *translation) edits(o *lit.Output, doc *lit.Doc, list []any) []any { - out, header := []any{}, []map[string]any{} - for _, e := range list { - m := asMap(e) - if l, _, ok := outPosition(o, asMap(m["range"])["start"]); ok && l < o.Header { - header = append(header, m) - } else if t.edit(o, doc, m) { - out = append(out, m) + status := 0 + for _, path := range fs.Args() { + doc, err := load(path) + if err != nil { + fmt.Fprintln(os.Stderr, "litgo:", err) + status = 1 + continue } - } - if len(header) == 0 { - return out - } - // <> - file, _ := parser.ParseFile(token.NewFileSet(), "", text, parser.ImportsOnly) - if file == nil { - return out - } - for _, spec := range file.Imports { - alias, path := "", strings.Trim(spec.Path.Value, "\"`") - if spec.Name != nil { - alias = spec.Name.Name + res := doc.Tangle() + diags := append(res.Diags, res.SyntaxCheck()...) + var sum vego.Summary + if len(diags) == len(res.Diags) && !res.HasErrors() { + var proofs []lit.Diag + proofs, sum = vego.Prove(res) + diags = append(diags, proofs...) } - known := false - for _, imp := range o.File.Imports { - known = known || imp.Path == path + failed := false + for _, d := range diags { + failed = failed || d.Severity == lit.SevError } - if line, col, entry, ok := doc.AddImport(o.File, alias, path); ok && !known { - at := position{line, units(t.s.utf8, doc.Lines[min(line, len(doc.Lines)-1)], col)} - out = append(out, map[string]any{"range": map[string]any{"start": at, "end": at}, "newText": entry}) + if failed { + status = 1 + } + if *asJSON { + if diags == nil { + diags = []lit.Diag{} + } + printJSON(map[string]any{"file": doc.Path, "diagnostics": diags, "functions": sum.Functions, + "proved": sum.Proved, "obligations": sum.Obligations, "names": sum.Names}) + continue + } + report(diags) + switch { + case failed && sum.Functions == 0: + case sum.Functions == 0: + fmt.Fprintf(os.Stderr, "litgo: %s has no //@ annotations, so there is nothing to prove\n", relPath(doc.Path)) + default: + fmt.Fprintf(os.Stderr, "litgo: %s: proved %d of %d annotated functions (%d obligations)\n", + relPath(doc.Path), sum.Proved, sum.Functions, sum.Obligations) + for _, name := range sum.Names { + fmt.Fprintf(os.Stderr, " βœ“ %s\n", name) + } } } - return out + return status } ``` -The edits in a list all refer to the text as it was before any of them, so they -get carried out from the bottom up. +`cat` prints a document the way the editor shows it, so it is the quickest way +to see what the renderers do. - ```go -text := string(o.Bytes()) -starts := []int{0} -for _, l := range o.Lines { - starts = append(starts, starts[len(starts)-1]+len(l.Text)+1) -} -at := func(v any) int { - l, c, ok := outPosition(o, v) - if !ok { - return -1 +// cmdCat prints the document the way the editor shows it: math as Unicode, +// diagrams drawn, chunk syntax prettified. +func cmdCat(args []string) int { + if len(args) != 1 { + fmt.Fprintln(os.Stderr, "litgo cat: expected one input file") + return 2 } - return starts[l] + c -} -sort.SliceStable(header, func(i, j int) bool { - return at(asMap(header[i]["range"])["start"]) > at(asMap(header[j]["range"])["start"]) -}) -for _, m := range header { - r := asMap(m["range"]) - if from, to := at(r["start"]), at(r["end"]); from >= 0 && to >= from { - text = text[:from] + m["newText"].(string) + text[to:] + doc, err := load(args[0]) + if err != nil { + fmt.Fprintln(os.Stderr, "litgo:", err) + return 1 + } + a := doc.Analyze(lit.AnalyzeOptions{}) + blocks := map[int]lit.Item{} + inline := map[int][]lit.Item{} + for _, it := range a.Items { + if it.Kind == "table" || it.Kind == "vego" { + continue // lining up a table and colouring an annotation are the editor's business + } else if len(it.Lines) > 0 { + blocks[it.Line] = it + } else { + inline[it.Line] = append(inline[it.Line], it) + } + } + for i := 0; i < len(doc.Lines); i++ { + if it, ok := blocks[i]; ok { + for _, l := range it.Lines { + fmt.Println(" " + l) + } + i = it.EndLine + continue + } + line := doc.Lines[i] + items := inline[i] + sort.Slice(items, func(a, b int) bool { return items[a].Col < items[b].Col }) + for k := len(items) - 1; k >= 0; k-- { // right to left keeps columns valid + it := items[k] + line = line[:it.Col] + it.Text + line[min(it.EndCol, len(line)):] + } + fmt.Println(line) } + return 0 } ``` -Renaming is the one answer that is made entirely of edits, in any number of -files. It is also the one where dropping an edit is not an option, because a -rename that is only partly carried out leaves a program that doesn't compile. -So litgo refuses the whole rename if any part of it can't be done. That -includes the case where gopls wants to change a tangled file whose document -isn't open. The change would be lost at the next tangle, and the right way to -make it is to open that document and rename there. - -The edits come back grouped by tangled file, and they go out grouped by -document. litgo strips the version numbers gopls put on them, because those are -versions of overlays the editor has never heard of. +`go/format` accepts fragments (a list of statements as well as a whole file), +so most blocks can be formatted right where they are. A block that only parses +after its chunks are spliced in gets left alone. ```go -// workspaceEdit translates the answer to a rename. -func (t *translation) workspaceEdit(v any) (any, error) { - w, _ := v.(map[string]any) - if w == nil { - return nil, nil +func cmdFmt(args []string) int { + fs := flag.NewFlagSet("fmt", flag.ExitOnError) + write := fs.Bool("w", false, "write the result back instead of printing it") + fs.Parse(args) + if fs.NArg() != 1 { + fmt.Fprintln(os.Stderr, "litgo fmt: expected one input file") + return 2 } - changes := map[string][]any{} - seen := map[string]bool{} - add := func(uri string, edits []any) error { - o, doc, owner := t.s.output(uri, t.res) - if o == nil { - if b, err := os.ReadFile(path(uri)); err == nil && lit.Generated(b) { - return fmt.Errorf("%s is tangled from a document that is not open: open it, and rename there", filepath.Base(path(uri))) - } - changes[uri] = append(changes[uri], edits...) - return nil + doc, err := load(fs.Arg(0)) + if err != nil { + fmt.Fprintln(os.Stderr, "litgo:", err) + return 1 + } + lines := append([]string{}, doc.Lines...) + for i := len(doc.Blocks) - 1; i >= 0; i-- { + b := doc.Blocks[i] + if !b.Tangled() || b.Lang != "go" || b.Last() < b.First() || b.Strip > 0 { + continue } - for _, e := range edits { - m := asMap(e) - at, _ := json.Marshal(m["range"]) - if !t.edit(o, doc, m) { - return fmt.Errorf("this would change text that litgo generates, in %s at %s", filepath.Base(o.Path), at) - } - if key := positioned(m); !seen[owner+key] { - seen[owner+key] = true - changes[owner] = append(changes[owner], m) - } + // format.Source accepts fragments; blocks that only make sense once + // their chunks are spliced in fail to parse and are left alone. So are + // blocks with inline references: gofmt would tidy away the comma that + // precedes a reference standing for the rest of an argument list. + if !b.Verbatim && inlineRefs(doc.Lines[b.First():b.Last()+1]) { + continue } - return nil - } - for uri, edits := range asMap(w["changes"]) { - if err := add(uri, asList(edits)); err != nil { - return nil, err + src := strings.Join(doc.Lines[b.First():b.Last()+1], "\n") + "\n" + out, err := format.Source([]byte(src)) + if err != nil { + continue } + formatted := strings.Split(strings.TrimRight(string(out), "\n"), "\n") + lines = append(append(append([]string{}, lines[:b.First()]...), formatted...), lines[b.Last()+1:]...) } - for _, c := range asList(w["documentChanges"]) { - change := asMap(c) - if change["kind"] != nil { - return nil, fmt.Errorf("this would %s a file, which a document's file directives decide", change["kind"]) - } - uri, _ := asMap(change["textDocument"])["uri"].(string) - if err := add(uri, asList(change["edits"])); err != nil { - return nil, err + text := strings.Join(lines, "\n") + "\n" + if *write { + if err := os.WriteFile(doc.Path, []byte(text), 0o644); err != nil { + fmt.Fprintln(os.Stderr, "litgo:", err) + return 1 } + return 0 } - return map[string]any{"changes": changes}, nil + fmt.Print(text) + return 0 } ``` ```go -// relay passes gopls's answer to a request on to the editor. -func (s *server) relay(to *message, reply *message, t *translation) { - switch { - case reply != nil && reply.Error != nil: - s.write(&message{ID: to.ID, Error: reply.Error}) - case to.Method == "textDocument/rename": - edit, err := t.workspaceEdit(reply.result()) - if err != nil { - s.write(&message{ID: to.ID, Error: &rpcError{requestFailed, err.Error()}}) - return +func inlineRefs(lines []string) bool { + for i, l := range lines { + for _, r := range lit.RefsIn(l, i) { + if r.Inline { + return true + } } - s.respond(to, edit) - default: - s.respond(to, t.value(reply.result())) } + return false } -``` - - - - -The test needs gopls and gets skipped without it. Its document has a function -in one block, a call to that function in a chunk, and a clef character in front -of the call, which is two UTF-16 units and four bytes. Every position in the -test gets translated twice on the way in and twice on the way out. +func cmdAnalyze(args []string) int { + fs := flag.NewFlagSet("analyze", flag.ExitOnError) + path := fs.String("path", "", "the document's file name") + upright := fs.Bool("upright", false, "do not italicise math letters") + fs.Parse(args) + return printJSON(stdinDoc(*path).Analyze(lit.AnalyzeOptions{Upright: *upright})) +} - -```go -const greeter = "\n\n\n```go\nfunc greet(whom string) string {\n\treturn \"hello, \" + whom\n}\n\nfunc main() {\n\t// <>\n}\n```\n\n\n```go\nfmt.Println(\"π„ž\", greet(\"world\"))\n```\n" - -// ask sends a request about a position and returns the result as JSON. -func (c *client) ask(id int, method, uri string, line, character int, more map[string]any) string { - params := map[string]any{"textDocument": map[string]any{"uri": uri}, "position": map[string]any{"line": line, "character": character}} - for k, v := range more { - params[k] = v +func cmdUntangle(args []string) int { + fs := flag.NewFlagSet("untangle", flag.ExitOnError) + path := fs.String("path", "", "the document's file name") + start := fs.Int("start", -1, "first selected line") + end := fs.Int("end", -1, "last selected line") + startCol := fs.Int("start-col", 0, "first selected byte (charwise)") + endCol := fs.Int("end-col", -1, "byte after the selection (charwise)") + name := fs.String("name", "", "chunk name; suggested when empty") + fs.Parse(args) + res, err := stdinDoc(*path).Untangle(lit.Selection{StartLine: *start, EndLine: *end, StartCol: *startCol, EndCol: *endCol}, *name) + if err != nil { + return jsonError(err) } - c.send("textDocument/"+method, id, params) - return c.result(id) + return printJSON(res) } -// result waits for the answer to a request. -func (c *client) result(id int) string { - for { - if m := c.next(); m.ID != nil && string(*m.ID) == strconv.Itoa(id) { - if m.Error != nil { - return "error: " + m.Error.Message - } - var sb strings.Builder - enc := json.NewEncoder(&sb) - enc.SetEscapeHTML(false) - enc.Encode(m.Result) - return strings.TrimSpace(sb.String()) - } +func cmdVersion(args []string) int { + fmt.Printf("%s\n", VERSION) + return 0 +} + +func cmdRename(args []string) int { + fs := flag.NewFlagSet("rename", flag.ExitOnError) + path := fs.String("path", "", "the document's file name") + from := fs.String("from", "", "current chunk name") + to := fs.String("to", "", "new chunk name") + fs.Parse(args) + edits, err := stdinDoc(*path).Rename(*from, *to) + if err != nil { + return jsonError(err) } + return printJSON(map[string]any{"edits": edits, "name": lit.NormName(*to)}) } +``` -// start spells the start of a range the way encoding/json does: keys sorted. -func start(line, character int) string { - return fmt.Sprintf(`"start":{"character":%d,"line":%d}`, character, line) +# The feedback loop + + + + + +`litgo run` is the reason everything above exists. It tangles, proves, +compiles and runs the program, and reports every position in terms of the +document. The stages are in order of cost, and each one stops the run if it +fails: a chunk that does not exist costs microseconds to find, a broken +invariant milliseconds, a type error a compiler, and a wrong answer a whole +execution and somebody paying attention. + +```mermaid +sequenceDiagram + participant E as editor + participant L as litgo run + participant G as go build + participant P as program + E->>L: pool.lit.md + L->>L: prove the //@ annotations of pool.go + L-->>E: pool.lit.md:80:1: Invariant is not maintained + L->>G: pool.go + G-->>L: pool.go:31:14: undefined: jobb + L-->>E: pool.lit.md:67:15: undefined: jobb + L->>P: start + P-->>L: output, or a panic with a stack trace + L-->>E: output, positions translated +``` + +Progress goes to a person as text, or to an editor as one JSON event per line. +The events are `tangle`, `prove`, `diagnostic`, `build`, `output` and `exit`. +A failed proof is a `diagnostic` like a compile error, told apart by its +`source`, which is `vego`. An editor can use that to show the two differently, +and the Neovim plugin does. + +The reporter has no lock, so only one goroutine is allowed to use it. That is +the goroutine that runs `runFile`. When a program's output has to be read from +two pipes at once, the goroutines that read the pipes send their lines to that +one goroutine over a channel, and it does all of the reporting. + +```go +// reporter carries the run's progress to a person (plain text) or to an +// editor (one JSON event per line). It belongs to one goroutine. +type reporter struct { + json bool + seen map[string]bool } -func TestGoplsThroughTheDocument(t *testing.T) { - if findGopls("") == "" { - t.Skip("no gopls") - } - dir := t.TempDir() - os.WriteFile(dir+"/go.mod", []byte("module example.com/greeter\n\ngo 1.22\n"), 0o644) - uri := "file://" + dir + "/greeter.lit.md" - c := connect(t, "") - c.send("initialize", 1, map[string]any{"rootUri": "file://" + dir, "capabilities": map[string]any{ - "general": map[string]any{"positionEncodings": []string{"utf-8", "utf-16"}}}}) - if got := c.result(1); !strings.Contains(got, "completionProvider") || !strings.Contains(got, `"utf-16"`) { - t.Fatalf("with gopls behind it the server completes, and counts as gopls does: %s", got) +func (r *reporter) emit(event string, fields map[string]any) { + if !r.json { + return } - c.send("initialized", 0, `{}`) - c.send("textDocument/didOpen", 0, map[string]any{"textDocument": map[string]any{"uri": uri, "version": 1, "text": greeter}}) + fields["event"] = event + b, _ := json.Marshal(fields) + fmt.Println(string(b)) +} - // Println, in the chunk. gopls may still be loading the first time. - hover := "" - for i := 0; i < 50 && !strings.Contains(hover, "func fmt.Println"); i++ { - if hover = c.ask(10+i, "hover", uri, 15, 5, nil); hover == "null" { - time.Sleep(200 * time.Millisecond) - } - } - if !strings.Contains(hover, "func fmt.Println") || !strings.Contains(hover, start(15, 4)) { - t.Errorf("hover in a chunk: %s", hover) - } - // greet, after the clef: from the chunk to the block that defines it. - if got := c.ask(2, "definition", uri, 15, 19, nil); !strings.Contains(got, uri) || !strings.Contains(got, start(4, 5)) { - t.Errorf("definition across blocks: %s", got) - } - if got := c.ask(3, "references", uri, 4, 6, map[string]any{"context": map[string]any{"includeDeclaration": true}}); strings.Count(got, uri) != 2 || !strings.Contains(got, start(15, 18)) { - t.Errorf("references, in UTF-16 units: %s", got) - } - if got := c.ask(4, "rename", uri, 4, 6, map[string]any{"newName": "hail"}); strings.Count(got, `"newText":"hail"`) != 2 || - !strings.Contains(got, start(15, 18)) || strings.Contains(got, "greeter.go") { - t.Errorf("rename edits the document, not the tangled file: %s", got) +func (r *reporter) diag(d lit.Diag) { + key := fmt.Sprintf("%s:%d:%d:%s", d.File, d.Line, d.Col, d.Message) + if r.seen[key] { + return } - if got := c.ask(5, "hover", uri, 0, 3, nil); got != "null" { - t.Errorf("prose is not Go: %s", got) + r.seen[key] = true + if r.json { + b, _ := json.Marshal(d) + var fields map[string]any + json.Unmarshal(b, &fields) + r.emit("diagnostic", fields) + return } + sev := map[int]string{lit.SevError: "error", lit.SevWarning: "warning", lit.SevInfo: "note"}[d.Severity] + fmt.Fprintf(os.Stderr, "%s:%d:%d: %s: %s\n", relPath(d.File), d.Line+1, d.Col+1, sev, d.Message) +} - // The chunk reference is litgo's own business. - if got := c.ask(6, "definition", uri, 9, 6, nil); !strings.Contains(got, start(15, 0)) { - t.Errorf("a reference is defined by its chunk: %s", got) - } - if got := c.ask(7, "hover", uri, 9, 6, nil); !strings.Contains(got, "⟨say it⟩") || !strings.Contains(got, "greet(") { - t.Errorf("hovering over a reference shows the chunk: %s", got) +func (r *reporter) output(stream, text string) { + if r.json { + r.emit("output", map[string]any{"stream": stream, "text": text}) + return } - if got := c.ask(8, "rename", uri, 13, 14, map[string]any{"newName": "speak"}); !strings.Contains(got, "") || !strings.Contains(got, "// <>") { - t.Errorf("renaming a chunk: %s", got) + if stream == "stderr" { + fmt.Fprintln(os.Stderr, text) + } else { + fmt.Println(text) } +} - // Completing from a package that is not imported adds it to the directive. - c.send("textDocument/didChange", 0, map[string]any{ - "textDocument": map[string]any{"uri": uri, "version": 2}, - "contentChanges": []any{map[string]any{"text": strings.Replace(greeter, `greet("world")`, "strings.", 1)}}, - }) - got := c.ask(9, "completion", uri, 15, 26, nil) - if !strings.Contains(got, `"label":"ToUpper"`) { - t.Fatalf("completion: %.300s", got) +func (r *reporter) status(format string, args ...any) { + if !r.json { + fmt.Fprintf(os.Stderr, "litgo: "+format+"\n", args...) } - if !strings.Contains(got, `"newText":", strings"`) || !strings.Contains(got, start(1, 17)) { - i := strings.Index(got, `"label":"ToUpper"`) - t.Errorf("the import belongs in the imports directive: %.600s", got[max(i-300, 0):]) +} + +func relPath(p string) string { + if wd, err := os.Getwd(); err == nil { + if rel, err := filepath.Rel(wd, p); err == nil && !strings.HasPrefix(rel, "..") { + return rel + } } + return p } ``` -# The Neovim plugin +## Translating positions -The plugin lives in this repository, at the root, where plugin managers look -for `plugin/` and `lua/`. It is a client of the protocol above, so the two have -to change together. With lazy.nvim: +Anything that looks like `file.go:12:5` and names a file litgo tangled gets +rewritten, wherever it shows up. That can be in a compiler error, in the middle +of an error message that mentions a second position, or in a stack trace. -```lua -{ - "tlehman/litgo", - build = "go build -o bin/litgo .", - event = { "BufReadPre *.lit.md", "BufNewFile *.lit.md" }, - opts = {}, +```go +// remapper rewrites positions in tangled files back to the literate source. +type remapper struct { + dir string + outputs map[string]*mapped // by absolute output path } -``` - -The plugin is thin on purpose. Everything that knows about the format is in the -binary, and the Lua just turns answers into extmarks, edits and diagnostics. -Another editor would need the same four calls and nothing else. - -| Command | Key | | -|---|---|---| -| *(nothing)* | | Chunk, syntax and type errors show up under the offending line as you type, from `litgo lsp`. | -| *(what Neovim has for any language server)* | `K`, `grr`, `grn`, ``, `` | In a Go block the buffer acts like a Go buffer. You get gopls's completion, documentation, references, rename and signature help, through `litgo lsp`. | -| `:TangleCompileAndRun [args]` | `\r` | Saves, tangles, compiles and runs. Panics and failing tests land in the Markdown too, and the cursor goes to the first error. | -| `:'<,'>Untangle [name]` | `\u` | Moves the selection into a chunk at the bottom and leaves a reference. | -| `:RenameChunk [name]` | `\n` | Renames the chunk under the cursor, everywhere. | -| `:Tangle[!]`, `:Weave [html]` | `\t`, `\w` | `:Tangle` writes the sources. `:Weave` writes and opens the PDF, or the web page. | -| `gd`, `K` | | On a chunk, `gd` goes from a reference to the definition and from a definition to the uses, and `K` previews the chunk. On Go they do what they do in Go. | -| `:LitWatch`, `:LitStop`, `:LitClear`, `:LitRender` | | Run on save, kill the program, clear diagnostics, toggle rendering. | - +// mapped is a tangled file together with the document it came from. +type mapped struct { + *lit.Output + doc *lit.Doc +} -```lua --- Loaded automatically when this directory is on the runtimepath. Calling --- require("litgo").setup({...}) yourself (before or after) overrides options. -if vim.g.loaded_litgo then - return -end +var posRe = regexp.MustCompile(`((?:[A-Za-z]:)?[\w./\\-]*\.go):(\d+)(?::(\d+))?`) -vim.g.loaded_litgo = true +// lookup finds the tangled file a tool is talking about. Compilers name +// files relative to where they ran; `go test` names them relative to the +// package, so a bare file name is matched if it is unambiguous. +func (m *remapper) lookup(path string) *mapped { + abs := path + if !filepath.IsAbs(abs) { + abs = filepath.Join(m.dir, abs) + } + if o := m.outputs[filepath.Clean(abs)]; o != nil || filepath.Base(path) != path { + return o + } + var found *mapped + for p, o := range m.outputs { + if filepath.Base(p) == path { + if found != nil { + return nil + } + found = o + } + } + return found +} -if vim.fn.has("nvim-0.11") == 0 then - vim.notify("litgo needs Neovim 0.11 or newer (inline virtual text, conceal_lines)", vim.log.levels.WARN) - return +// line rewrites every `file.go:line:col` in a line of tool output and returns +// the first position it translated. +func (m *remapper) line(s string) (string, *lit.Diag) { + var first *lit.Diag + out := posRe.ReplaceAllStringFunc(s, func(match string) string { + g := posRe.FindStringSubmatch(match) + res := m.lookup(g[1]) + if res == nil { + return match + } + ln, _ := strconv.Atoi(g[2]) + col := 1 + if g[3] != "" { + col, _ = strconv.Atoi(g[3]) + } + srcLine, srcCol, _, ok := res.Map(ln-1, col-1) + if !ok { + return match + } + if first == nil { + first = &lit.Diag{File: res.doc.Path, Line: srcLine, Col: srcCol} + } + if g[3] == "" { + return fmt.Sprintf("%s:%d", relPath(res.doc.Path), srcLine+1) + } + return fmt.Sprintf("%s:%d:%d", relPath(res.doc.Path), srcLine+1, srcCol+1) + }) + return out, first +} +``` + +```go +var compilerLineRe = regexp.MustCompile(`^(?:\./)?([^\s:]+\.go):(\d+)(?::(\d+))?:\s+(.*)$`) + +// toolOutput turns compiler or vet output into diagnostics, and reports +// whatever it could not place (linker errors, package-level complaints). +func (m *remapper) toolOutput(out []byte, sev int, source string, r *reporter) { + var last *lit.Diag + flush := func() { + if last != nil { + r.diag(*last) + last = nil + } + } + for _, raw := range strings.Split(strings.TrimRight(string(out), "\n"), "\n") { + if raw == "" || strings.HasPrefix(raw, "# ") { + continue + } + if g := compilerLineRe.FindStringSubmatch(raw); g != nil { + flush() + if _, pos := m.line(g[1] + ":" + g[2] + ":" + firstNonEmpty(g[3], "1")); pos != nil { + msg, _ := m.line(g[4]) + pos.Severity, pos.Message, pos.Source = sev, msg, source + last = pos + continue + } + } + if last != nil && (strings.HasPrefix(raw, "\t") || strings.HasPrefix(raw, " ")) { + cont, _ := m.line(strings.TrimSpace(raw)) + last.Message += "\n" + cont + continue + } + flush() + text, _ := m.line(raw) + r.output("stderr", text) + } + flush() +} + +func firstNonEmpty(a, b string) string { + if a != "" { + return a + } + return b +} +``` + +```go +// writeTangled writes a file unless that would clobber a hand-written one. +func writeTangled(o *lit.Output, force bool) error { + if old, err := os.ReadFile(o.Path); err == nil { + if bytes.Equal(old, o.Bytes()) { + return nil // leave the timestamp alone so builds stay cached + } + if !force && !lit.Generated(old) && len(bytes.TrimSpace(old)) > 0 && lit.CommentsFor(o.File.Lang).Line != "" { + return fmt.Errorf("%s exists and was not generated by litgo; use --force to overwrite it, or tangle to another ", relPath(o.Path)) + } + } + if err := os.MkdirAll(filepath.Dir(o.Path), 0o755); err != nil { + return err + } + return os.WriteFile(o.Path, o.Bytes(), 0o644) +} +``` + +## The pipeline + +```go +type runOptions struct { + json bool + force bool + vet bool + noProve bool + timeout time.Duration + args []string +} +``` + +```go +// runFile is the feedback loop: tangle, compile, run, with every position in +// every message pointing at the .lit.md source. +func runFile(path string, o runOptions) int { + r := &reporter{json: o.json, seen: map[string]bool{}} + // <> + + m := &remapper{dir: dir, outputs: map[string]*mapped{}} + // <> + // <> + + // <> + + // <> + return execute(r, m, exec.Command(bin, o.args...), dir, o.timeout, false) +} +``` + + +```go +abs, err := filepath.Abs(path) +if err != nil { + return fail(r, err) +} +dir := filepath.Dir(abs) +src, err := os.ReadFile(abs) +if err != nil { + return fail(r, err) +} +doc := lit.Parse(abs, src) + +// A custom run command may build a whole package, so tangle the siblings too. +docs := []*lit.Doc{doc} +if doc.Run != "" { + siblings, _ := filepath.Glob(filepath.Join(dir, "*.lit.md")) + for _, s := range siblings { + if s == abs { + continue + } + if b, err := os.ReadFile(s); err == nil { + docs = append(docs, lit.Parse(s, b)) + } + } +} +``` + +Every file that gets written is registered with the remapper. If the document's +structure has problems, the run stops here, because there is no point compiling +half a program. + + +```go +ok, unproved := true, false +for _, d := range docs { + res := d.Tangle() + for _, dg := range res.Diags { + r.diag(dg) + } + if res.HasErrors() { + ok = false + continue + } + for _, out := range res.Files { + if err := writeTangled(out, o.force); err != nil { + r.diag(lit.Diag{File: d.Path, Line: max(out.File.Line, 0), Severity: lit.SevError, Message: err.Error(), Source: "litgo"}) + ok = false + continue + } + m.outputs[filepath.Clean(out.Path)] = &mapped{out, d} + } + if n := len(res.Files); n == 1 { + r.emit("tangle", map[string]any{"file": d.Path, "out": res.Files[0].Path}) + r.status("tangled %s β†’ %s", relPath(d.Path), relPath(res.Files[0].Path)) + } else if n > 1 { + r.emit("tangle", map[string]any{"file": d.Path, "out": fmt.Sprintf("%d files", n)}) + r.status("tangled %s β†’ %d files", relPath(d.Path), n) + } + // <> +} +if !ok { + stage := "tangle" + if unproved { + stage = "prove" + r.status("not proved, so not run (--no-prove runs it anyway)") + } + r.emit("build", map[string]any{"ok": false, "ms": 0, "stage": stage}) + return 1 +} +``` + +A document with `//@` annotations is proved before it is compiled. The order +is deliberate. A proof needs no compiler and takes milliseconds, and a program +whose contract does not hold is not a program anyone asked to see run: the +output of a wrong `Divide` is a distraction from the line that says its +invariant is not maintained. So a failed proof stops the run, the same way a +compile error does, and is reported the same way, as a diagnostic at a line of +the Markdown with `vego` as its source. `--no-prove` is for the times when +watching it run is how you find out what the invariant should have been. + + +```go +if !o.noProve { + start := time.Now() + proofs, sum := vego.Prove(res) + for _, dg := range proofs { + r.diag(dg) + if dg.Severity == lit.SevError { + ok, unproved = false, true + } + } + if sum.Functions > 0 { + r.emit("prove", map[string]any{"file": d.Path, "functions": sum.Functions, "proved": sum.Proved, + "obligations": sum.Obligations, "ms": time.Since(start).Milliseconds()}) + r.status("proved %d of %d annotated functions in %s (%d obligations, %d ms)", sum.Proved, sum.Functions, + relPath(d.Path), sum.Obligations, time.Since(start).Milliseconds()) + } +} +``` + +This document takes the first branch. Its `run:` directive vets, tests and +rebuilds litgo. + + +```go +// A document that writes several files is a package or a module, and +// only it knows how it is built +command := doc.Run +if command == "" && len(doc.Files) > 1 { + command = "go build ./..." +} +if command != "" { + r.status("$ %s", command) + return execute(r, m, exec.Command("sh", "-c", command), dir, o.timeout, true) +} +def := doc.Files[0] +main := m.outputs[filepath.Clean(doc.OutPath(def))] +if def.Lang != "go" || main == nil { + r.status("no run step for tangler %q; add ", def.Lang) + r.emit("build", map[string]any{"ok": true, "ms": 0}) + return 0 +} +``` + +Building and running are separate steps. That way litgo can tell a compile +error apart from a program that exits non-zero, and it can time both steps. + + +```go +tmp, err := os.MkdirTemp("", "litgo-run") +if err != nil { + return fail(r, err) +} +defer os.RemoveAll(tmp) +bin := filepath.Join(tmp, "prog") + +start := time.Now() +build := exec.Command("go", /*<>*/) +build.Dir = dir +out, err := build.CombinedOutput() +ms := time.Since(start).Milliseconds() +m.toolOutput(out, lit.SevError, "go build", r) +if err != nil { + r.emit("build", map[string]any{"ok": false, "ms": ms, "stage": "compile"}) + r.status("build failed (%d ms)", ms) + return 1 +} +r.emit("build", map[string]any{"ok": true, "ms": ms}) +r.status("built in %d ms", ms) +``` + +The compiler stops after ten errors unless you tell it not to, and in a tight +loop it is better to see all of them. + + +```go +"build", "-gcflags=-e", "-o", bin, filepath.Base(main.Path) +``` + + +```go +if o.vet { + vet := exec.Command("go", "vet", filepath.Base(main.Path)) + vet.Dir = dir + out, _ := vet.CombinedOutput() + m.toolOutput(out, lit.SevWarning, "go vet", r) +} +if def.Package != "main" && def.Package != "" { + r.status("package %s compiles; nothing to run", def.Package) + r.emit("exit", map[string]any{"code": 0, "ms": 0, "ran": false}) + return 0 +} +``` + +## Running + +Output gets streamed line by line, and positions are translated on the way +through. A panic becomes a diagnostic on the innermost stack frame that is in +the document. A failing test does too, when the document's own command runs +`go test`. + +The program's stdout and stderr both have to be read while it runs, so each +pipe gets a goroutine. Those goroutines don't report anything themselves. They +send every line down one channel, and the loop in `execute` receives the lines +and does the reporting. The same loop waits for the timeout, so the reporter, +the panic bookkeeping and the `timedOut` flag all belong to one goroutine and +none of them needs a lock. + +```mermaid +flowchart LR + out[read stdout] --> loop[the loop in execute] + err[read stderr] --> loop + timer[timeout] --> loop + loop --> reporter +``` + +```go +func fail(r *reporter, err error) int { + r.diag(lit.Diag{Severity: lit.SevError, Message: err.Error(), Source: "litgo"}) + return 1 +} + +var ( + frameRe = regexp.MustCompile(`^\s+(\S+\.go):(\d+)`) + // reportRe is looser than compilerLineRe: `go test` indents its failures. + reportRe = regexp.MustCompile(`^\s*(?:\./)?[^\s:]+\.go:\d+(?::\d+)?:\s+(.*)$`) +) + +// outputLine is one line of a program's output, or the end of a stream. +type outputLine struct { + stream, text string + eof bool +} + +// pump sends the lines of one of a program's streams, and then its end. +func pump(stream string, rd io.Reader, lines chan<- outputLine) { + sc := bufio.NewScanner(rd) + sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024) + for sc.Scan() { + lines <- outputLine{stream: stream, text: sc.Text()} + } + lines <- outputLine{stream: stream, eof: true} +} + +// execute streams a program's output, translating positions as it goes. A +// panic becomes a diagnostic on the innermost frame that lives in the +// literate source. +func execute(r *reporter, m *remapper, cmd *exec.Cmd, dir string, timeout time.Duration, parseTools bool) int { + cmd.Dir = dir + stdout, _ := cmd.StdoutPipe() + stderr, _ := cmd.StderrPipe() + ownGroup(cmd) + start := time.Now() + if err := cmd.Start(); err != nil { + return fail(r, err) + } + defer killOnSignal(cmd)() + var timeUp <-chan time.Time // stays nil, which never delivers, without a timeout + if timeout > 0 { + timer := time.NewTimer(timeout) + defer timer.Stop() + timeUp = timer.C + } + + lines := make(chan outputLine) + go pump("stdout", stdout, lines) + go pump("stderr", stderr, lines) + timedOut := false + var panicMsg string + panicPlaced := false + for open := 2; open > 0; { + select { + case <-timeUp: + timedOut = true + killGroup(cmd) + case l := <-lines: + if l.eof { + open-- + continue + } + raw := l.text + text, pos := m.line(raw) + if l.stream == "stderr" { + switch { + case strings.HasPrefix(raw, "panic: ") || strings.HasPrefix(raw, "fatal error: "): + panicMsg, panicPlaced = raw, false + case panicMsg != "" && !panicPlaced && pos != nil && frameRe.MatchString(raw): + pos.Severity, pos.Message, pos.Source = lit.SevError, panicMsg, "runtime" + r.diag(*pos) + panicPlaced = true + case parseTools && pos != nil && reportRe.MatchString(raw): + msg, _ := m.line(reportRe.FindStringSubmatch(raw)[1]) + pos.Severity, pos.Message, pos.Source = lit.SevError, msg, "run" + r.diag(*pos) + } + } else if parseTools && pos != nil && reportRe.MatchString(raw) { + msg, _ := m.line(reportRe.FindStringSubmatch(raw)[1]) + pos.Severity, pos.Message, pos.Source = lit.SevError, msg, "run" + r.diag(*pos) + } + r.output(l.stream, text) + } + } + err := cmd.Wait() + code := 0 + if err != nil { + code = 1 + if ee, ok := err.(*exec.ExitError); ok && ee.ExitCode() >= 0 { + code = ee.ExitCode() + } + } + ms := time.Since(start).Milliseconds() + if timedOut { + r.output("stderr", fmt.Sprintf("litgo: killed after %s", timeout)) + } + r.emit("exit", map[string]any{"code": code, "ms": ms, "ran": true, "timed_out": timedOut}) + r.status("exit %d (%d ms)", code, ms) + return code +} +``` + +The program runs in its own process group. Stopping litgo stops the program and +anything it started, and the timeout does the same. + + + +```go +//go:build unix + +package main + +import ( + "os" + "os/exec" + "os/signal" + "syscall" +) + +// ownGroup puts the child in its own process group so that it, and anything +// it spawns, can be killed as one. +func ownGroup(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} +} + +func killGroup(cmd *exec.Cmd) { + if cmd.Process != nil { + syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + } +} + +// killOnSignal makes stopping litgo (the editor's :LitStop) stop the program +// it is running. The returned function ends the watch. +func killOnSignal(cmd *exec.Cmd) func() { + sigs := make(chan os.Signal, 1) + done := make(chan struct{}) + signal.Notify(sigs, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP) + go func() { + select { + case <-sigs: + killGroup(cmd) + case <-done: + } + }() + return func() { + signal.Stop(sigs) + close(done) + } +} +``` + + + +```go +//go:build !unix + +package main + +import "os/exec" + +func ownGroup(cmd *exec.Cmd) {} + +func killGroup(cmd *exec.Cmd) { + if cmd.Process != nil { + cmd.Process.Kill() + } +} + +func killOnSignal(cmd *exec.Cmd) func() { return func() {} } +``` + +# Errors as you type + +`litgo run` finds every error, but only when you ask, and only after it writes +files and starts a compiler. Most errors can be found sooner than that. The Go +type checker is in the standard library. If you give it the tangled files *in +memory*, it reports what the compiler would report (an undefined name, a wrong +argument, an unused variable, an unused import) in a few milliseconds, and the +source map puts each report on the right line of the Markdown. A language +server runs that check after every change, so the errors show up under the code +that caused them before you have even thought about running anything. The +verifier works the same way, on the same files in memory, so a document with +`//@` annotations is proved after every change too. + +```mermaid +sequenceDiagram + participant E as editor + participant S as litgo lsp + participant T as go/types + E->>S: didChange: the whole buffer + S-->>E: chunk and syntax errors, at once + S->>T: the tangled files, never written + T-->>S: pool.go:31:14: undefined: jobb + S-->>E: pool.lit.md:67:15: undefined: jobb + S->>S: prove the //@ annotations, once the types are right + S-->>E: pool.lit.md:80:1: Invariant is not maintained +``` + +## Type-checking what is not on disk + + + + + +A check works on packages. A package's files are the document's outputs in one +directory. When the document writes a module or has its own `run:` command, the +other Go files already in that directory are included too. A document that is a +single program gets checked by itself, the same way `litgo run` compiles it by +itself. This matters because a directory of examples is full of `main` +functions that can't be in the same package. + +```go +// pkg is one package about to be checked. +type pkg struct { + dir, name, path string + files []*ast.File // what an importer of the package sees + tests []*ast.File // _test.go files of the same package + types *types.Package + checking bool +} + +type checker struct { + res *lit.Result + fset *token.FileSet + outputs map[string]*lit.Output // by path + pkgs []*pkg + byPath map[string]*pkg // packages this document writes, by import path + exports map[string]string // import path to export data file, for all others + gc types.Importer + goVersion string + seen map[string]bool + diags []lit.Diag +} + +// Cache carries what is worth keeping between checks: where the compiled +// standard library lives. It has no lock, so whoever owns a Cache runs one +// check at a time with it. +type Cache struct { + std map[string]string +} + +// Types type-checks the Go files a document tangles to, without writing +// them, and reports errors at their positions in the document. It reports +// nothing if the files do not parse (the syntax check covers that) or if +// there is no go command to ask about imports. +func Types(res *lit.Result, cache *Cache) []lit.Diag { + if res.HasErrors() { + return nil + } + c := &checker{res: res, fset: token.NewFileSet(), outputs: map[string]*lit.Output{}, + byPath: map[string]*pkg{}, exports: map[string]string{}, seen: map[string]bool{}} + if !c.parse() { + return nil + } + c.locate() + if !c.resolve(cache) { + return nil + } + c.gc = importer.ForCompiler(c.fset, "gc", c.lookup) + for _, p := range c.pkgs { + c.check(p, true) + } + return c.diags +} +``` + +Build constraints matter here. litgo itself has two files that define the same +functions for different operating systems. `go/build` decides which files +belong, and it gets the in-memory content through its `OpenFile` hook. + +```go +// parse sorts the files into packages. It fails if an output does not parse. +func (c *checker) parse() bool { + doc := c.res.Doc + for _, o := range c.res.Files { + if o.File.Lang == "go" { + c.outputs[o.Path] = o + } + } + ctx := build.Default + ctx.OpenFile = func(path string) (io.ReadCloser, error) { + if o := c.outputs[path]; o != nil { + return io.NopCloser(bytes.NewReader(o.Bytes())), nil + } + return os.Open(path) + } + groups := map[string]*pkg{} + add := func(path string, src any) error { + dir, base := filepath.Dir(path), filepath.Base(path) + if ok, _ := ctx.MatchFile(dir, base); !ok { + return nil + } + f, err := parser.ParseFile(c.fset, path, src, parser.SkipObjectResolution) + if err != nil { + return err + } + test := strings.HasSuffix(base, "_test.go") + name := f.Name.Name + p := groups[dir+"\x00"+name] + if p == nil { + p = &pkg{dir: dir, name: name} + groups[dir+"\x00"+name] = p + c.pkgs = append(c.pkgs, p) + } + if test && !strings.HasSuffix(name, "_test") { + p.tests = append(p.tests, f) + } else { + p.files = append(p.files, f) + } + return nil + } + + dirs := map[string]bool{} + for path, o := range c.outputs { + if add(path, o.Bytes()) != nil { + return false + } + dirs[filepath.Dir(path)] = true + } + if doc.Run != "" || len(doc.Files) > 1 { + // <> + } + sort.Slice(c.pkgs, func(i, j int) bool { + return c.pkgs[i].dir+c.pkgs[i].name < c.pkgs[j].dir+c.pkgs[j].name + }) + return true +} +``` + +If a hand-written file doesn't parse, that is its author's problem and not this +document's, so the file gets left out. + + +```go +for dir := range dirs { + entries, _ := os.ReadDir(dir) + for _, e := range entries { + path := filepath.Join(dir, e.Name()) + if e.IsDir() || !strings.HasSuffix(path, ".go") || c.outputs[path] != nil { + continue + } + add(path, nil) + } +} +``` + +The packages a document writes can import each other. For example, litgo's +`main` imports its `internal/lit`. Those imports have to resolve to the version +in the buffer and not to whatever was last tangled, so each package needs its +import path, which is the module path plus the directory. The `go.mod` can be +one of the outputs too. + +```go +// locate works out the import path of every package. +func (c *checker) locate() { + modDir, mod := "", "" + for _, o := range c.res.Files { + if filepath.Base(o.Path) == "go.mod" { + modDir, mod = filepath.Dir(o.Path), string(o.Bytes()) + } + } + for dir := filepath.Dir(c.res.Doc.Path); mod == ""; dir = filepath.Dir(dir) { + if b, err := os.ReadFile(filepath.Join(dir, "go.mod")); err == nil { + modDir, mod = dir, string(b) + } + if dir == filepath.Dir(dir) { + break + } + } + module := "" + for _, line := range strings.Split(mod, "\n") { + if f := strings.Fields(line); len(f) == 2 && f[0] == "module" { + module = strings.Trim(f[1], `"`) + } else if len(f) == 2 && f[0] == "go" { + c.goVersion = "go" + f[1] + } + } + for _, p := range c.pkgs { + p.path = p.name + if rel, err := filepath.Rel(modDir, p.dir); module != "" && err == nil && !strings.HasPrefix(rel, "..") { + p.path = strings.TrimSuffix(module+"/"+filepath.ToSlash(rel), "/.") + if strings.HasSuffix(p.name, "_test") { + p.path += "_test" + } + } + if !strings.HasSuffix(p.name, "_test") && p.name != "main" { + c.byPath[p.path] = p + } + } +} +``` + +Everything else gets imported the way `go vet` does it, from the export data +the compiler left in the build cache. `go list -export` finds that data, and +compiles first if it has to. This is exact, and it is fast. Reading the export +data of `fmt` takes a fraction of the time it would take to type-check its +source. One `go list` call covers all the imports of a check. The standard +library gets remembered for the life of the process, because it doesn't change +while an editor is running. + +```go +// resolve finds the export data of every package that is imported but not +// written by this document. It fails if the go command cannot be run. +func (c *checker) resolve(cache *Cache) bool { + if cache.std == nil { + cache.std = map[string]string{} + } + var missing []string + for _, p := range c.pkgs { + for _, f := range append(append([]*ast.File{}, p.files...), p.tests...) { + for _, imp := range f.Imports { + path := strings.Trim(imp.Path.Value, "\"`") + if c.byPath[path] != nil || path == "C" || path == "unsafe" || c.seen[path] { + continue + } + c.seen[path] = true + if file, ok := cache.std[path]; ok && exists(file) { + c.exports[path] = file + } else { + missing = append(missing, path) + } + } + } + } + c.seen = map[string]bool{} + if len(missing) == 0 { + return true + } + args := append([]string{"list", "-e", "-export", "-deps", "-f", "{{if .Export}}{{.ImportPath}}\t{{.Export}}{{end}}", "--"}, missing...) + cmd := exec.Command("go", args...) + cmd.Dir = filepath.Dir(c.res.Doc.Path) + out, err := cmd.Output() + if _, broken := err.(*exec.Error); broken { + return false + } + for _, line := range strings.Split(string(out), "\n") { + if path, file, ok := strings.Cut(line, "\t"); ok { + c.exports[path] = file + if !strings.Contains(strings.Split(path, "/")[0], ".") { + cache.std[path] = file + } + } + } + return true +} + +func exists(file string) bool { + _, err := os.Stat(file) + return err == nil +} + +func (c *checker) lookup(path string) (io.ReadCloser, error) { + if file := c.exports[path]; file != "" { + return os.Open(file) + } + return nil, os.ErrNotExist +} +``` + +The checker acts as its own importer. A package from this document gets checked +on demand, once, without its tests. Anything else comes from the export data. + +```go +// Import implements types.Importer. +func (c *checker) Import(path string) (*types.Package, error) { + if p := c.byPath[path]; p != nil { + return c.check(p, false) + } + return c.gc.Import(path) +} + +// check type-checks a package: as a root, with its tests and with its +// errors reported, or as the import of another package, silently. +func (c *checker) check(p *pkg, root bool) (*types.Package, error) { + files := p.files + if root { + files = append(append([]*ast.File{}, p.files...), p.tests...) + } else if p.types != nil { + return p.types, nil + } else if p.checking { + return nil, os.ErrInvalid // an import cycle; the root check of it says so + } + conf := types.Config{Importer: c, GoVersion: c.goVersion, FakeImportC: true, Error: func(err error) { + if root { + c.report(err) + } + }} + p.checking = true + tp, _ := conf.Check(p.path, c.fset, files, nil) + p.checking = false + if !root { + p.types = tp + } + return tp, nil +} +``` + +If a chunk is used twice, an error in it gets reported once, not twice. + +```go +func (c *checker) report(err error) { + te, ok := err.(types.Error) + if !ok || len(c.diags) >= 100 { + return + } + pos := c.fset.Position(te.Pos) + o := c.outputs[pos.Filename] + if o == nil { + return + } + line, col, _, ok := o.Map(pos.Line-1, pos.Column-1) + key := fmt.Sprintf("%d:%d:%s", line, col, te.Msg) + if !ok || c.seen[key] { + return + } + c.seen[key] = true + c.diags = append(c.diags, lit.Diag{File: c.res.Doc.Path, Line: line, Col: col, Severity: lit.SevError, Message: te.Msg, Source: "types"}) +} +``` + + + + + +The test document writes a small module with a library, a program that uses it, +and a test. None of it exists on disk, and each error is in a different kind of +place. + + +```go +const module = "\n```gomod\nmodule example.com/m\n\ngo 1.22\n```\n\n" + + "\n\n\n" + + "```go\nfunc Shout(s string) string {\n\t// <>\n}\n```\n\n" + + "\n```go\nreturn strings.ToUpper(s) + suffix\n```\n\n" + + "\n\n\n" + + "```go\nfunc TestShout(t *testing.T) {\n\tif Shout(1) != \"A\" {\n\t\tt.Fail()\n\t}\n}\n```\n\n" + + "\n\n\n" + + "```go\nfunc main() {\n\tn := 1\n\tfmt.Println(lib.Shout(\"a\"), lib.Whisper)\n}\n```\n" + +func TestTypes(t *testing.T) { + path := filepath.Join(t.TempDir(), "m.lit.md") + os.WriteFile(path, []byte(module), 0o644) + doc := lit.Parse(path, []byte(module)) + diags := Types(doc.Tangle(), &Cache{}) + + var got []string + for _, d := range diags { + got = append(got, strings.TrimSpace(doc.Lines[d.Line][d.Col:])+" ← "+d.Message) + } + for _, want := range []string{ + `os, strings --> ← "os" imported and not used`, // on its entry in the directive + "suffix ← undefined: suffix", // inside a chunk + "1) != \"A\" { ← cannot use 1", // in a test file of the package + "n := 1 ← declared and not used: n", + "Whisper) ← undefined: lib.Whisper", // through an import of a package that is not on disk + } { + text, msg, _ := strings.Cut(want, " ← ") + found := false + for _, g := range got { + found = found || strings.HasPrefix(g, text) && strings.Contains(g, msg) + } + if !found { + t.Errorf("missing %q in:\n%s", want, strings.Join(got, "\n")) + } + } + if len(diags) != 5 { + t.Errorf("%d diagnostics, want 5:\n%s", len(diags), strings.Join(got, "\n")) + } +} + +func TestSingleProgramStandsAlone(t *testing.T) { + dir := t.TempDir() + os.WriteFile(filepath.Join(dir, "other.go"), []byte("package main\n\nfunc main() {}\n"), 0o644) + src := "\n```go\nfunc main() {}\n```\n" + doc := lit.Parse(filepath.Join(dir, "one.lit.md"), []byte(src)) + if diags := Types(doc.Tangle(), &Cache{}); len(diags) != 0 { + t.Errorf("a second main in the directory is not this program's business: %+v", diags) + } +} +``` + +## The language server + + + + + +The Language Server Protocol is JSON-RPC over standard input and output, with a +`Content-Length` header in front of each message. The part litgo needs for +publishing diagnostics is small enough to write out here. + +```go +type message struct { + JSONRPC string `json:"jsonrpc"` + ID *json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result any `json:"result,omitempty"` + Error *rpcError `json:"error,omitempty"` +} + +type rpcError struct { + Code int `json:"code"` + Message string `json:"message"` +} + +func read(r *bufio.Reader) (*message, error) { + length := 0 + for { + line, err := r.ReadString('\n') + if err != nil { + return nil, err + } + line = strings.TrimSpace(line) + if line == "" { + break + } + if name, value, ok := strings.Cut(line, ":"); ok && strings.EqualFold(name, "Content-Length") { + length, _ = strconv.Atoi(strings.TrimSpace(value)) + } + } + body := make([]byte, length) + if _, err := io.ReadFull(r, body); err != nil { + return nil, err + } + m := &message{} + return m, json.Unmarshal(body, m) +} + +// frame spells a message the way it goes over the wire. +func frame(m *message) []byte { + m.JSONRPC = "2.0" + body, _ := json.Marshal(m) + return []byte(fmt.Sprintf("Content-Length: %d\r\n\r\n%s", len(body), body)) +} +``` + +The server has no locks. One goroutine, the loop in `Serve`, owns all of the +server's state: the open documents, the overlays, and the bookkeeping for +gopls. Every other goroutine does one job that is slow or that blocks, and +talks to the loop over a channel. One goroutine reads messages from the editor +and one writes messages to the editor. Each document that changed has a timer +that tells the loop when the typing has paused. At most one goroutine runs the +type checker. Since only the loop touches the state, nothing needs a mutex. + +```mermaid +flowchart LR + reader[read the editor] --> loop + timers --> loop + checker[type checker] --> loop + loop --> writer[write to the editor] +``` + +A pipe gets written by one goroutine that owns it. Messages are framed by +whoever sends them, so the writer never sees a value that someone else might +still change. The channel has a buffer, so the loop can keep going while a +reader on the other end of the pipe is slow. Closing the channel says there is +nothing more to write, and `done` closes after the last message has gone out. + +```go +// writer starts the goroutine that writes to a pipe. +func writer(w io.Writer) (messages chan []byte, done chan struct{}) { + messages, done = make(chan []byte, 64), make(chan struct{}) + go func() { + defer close(done) + for b := range messages { + w.Write(b) + } + }() + return messages, done +} + +func (s *server) write(m *message) { s.out <- frame(m) } +``` + +The server keeps the text of every open document, and the whole text arrives +with each change. A change starts a short timer instead of an analysis, so a +burst of typing costs one analysis instead of one per keystroke. + +```go +type document struct { + text string + version int + timer *time.Timer + res *lit.Result // what the text tangles to, + tangled int // and the version of the text that was tangled +} + +// server is the state of the loop in Serve. Only that goroutine touches it. +type server struct { + out chan<- []byte // to the goroutine that writes to the editor + docs map[string]*document + utf8 bool // the client counts columns in bytes, as litgo does + delay time.Duration + + due chan due // a document's timer went off + checked chan *job // the type checker is done with a job + quit chan struct{} // closed when the server stops + waiting map[string]*job + checking bool // the type checker is busy, and has the cache + cache check.Cache + + // The rest is for the chapter after this one. + goplsBin string + gopls *gopls + overlays map[string]*overlay // by path of the tangled file +} + +// due says that a document has not changed for a while since this version. +type due struct { + uri string + version int +} + +// Serve speaks the protocol until the client says exit or hangs up. It puts +// the gopls named (or found, if none is named; "off" means none) behind it. +func Serve(in io.Reader, out io.Writer, gopls string) error { + toEditor, written := writer(out) + s := &server{out: toEditor, docs: map[string]*document{}, delay: 120 * time.Millisecond, + due: make(chan due), checked: make(chan *job), quit: make(chan struct{}), waiting: map[string]*job{}, + goplsBin: gopls, overlays: map[string]*overlay{}} + defer func() { + close(s.quit) + s.stop() + close(toEditor) + <-written + }() + editor, failed := make(chan *message), make(chan error, 1) + go func() { + r := bufio.NewReader(in) + for { + m, err := read(r) + if err != nil { + failed <- err + return + } + select { + case editor <- m: + case <-s.quit: + return + } + } + }() + for { + select { + case m := <-editor: + if m.Method == "exit" { + return nil + } + s.handle(m) + case err := <-failed: + if err == io.EOF { + return nil + } + return err + case d := <-s.due: + if _, now := s.current(d.uri); now == d.version { + s.analyze(d.uri) + } + case j := <-s.checked: + s.finish(j) + case m, ok := <-s.heard(): + if ok { + s.gopls.hear(m, s) + } else { + s.gopls.hangUp() + } + } + } +} +``` + +Every goroutine that sends to the loop also watches `quit`, so none of them is +left waiting for a loop that has gone away. + +```go +func (s *server) handle(m *message) { + switch m.Method { + case "initialize": + // <> + case "shutdown": + s.write(&message{ID: m.ID, Result: json.RawMessage("null")}) + case "textDocument/didOpen": + var p struct { + TextDocument struct { + URI string `json:"uri"` + Text string `json:"text"` + Version int `json:"version"` + } `json:"textDocument"` + } + json.Unmarshal(m.Params, &p) + s.change(p.TextDocument.URI, p.TextDocument.Text, p.TextDocument.Version) + case "textDocument/didChange": + var p struct { + TextDocument struct { + URI string `json:"uri"` + Version int `json:"version"` + } `json:"textDocument"` + ContentChanges []struct { + Text string `json:"text"` + } `json:"contentChanges"` + } + json.Unmarshal(m.Params, &p) + if n := len(p.ContentChanges); n > 0 { + s.change(p.TextDocument.URI, p.ContentChanges[n-1].Text, p.TextDocument.Version) + } + case "textDocument/didClose": + var p struct { + TextDocument struct { + URI string `json:"uri"` + } `json:"textDocument"` + } + json.Unmarshal(m.Params, &p) + if d := s.docs[p.TextDocument.URI]; d != nil && d.timer != nil { + d.timer.Stop() + } + delete(s.docs, p.TextDocument.URI) + delete(s.waiting, p.TextDocument.URI) + s.forget(p.TextDocument.URI) + default: + if !s.proxy(m) && m.ID != nil { // a request for something this server does not do + s.write(&message{ID: m.ID, Error: &rpcError{-32601, "litgo lsp does not implement " + m.Method}}) + } + } +} +``` + +The protocol counts columns in UTF-16 code units unless both sides agree on +something else. litgo counts bytes and so does Neovim, which offers `utf-8`. +For a client that doesn't offer it, litgo converts columns on the way out. +Documents are synchronised whole (`change: 1`), because they are small and +litgo parses the whole document anyway. + + +```go +var p struct { + Capabilities struct { + General struct { + PositionEncodings []string `json:"positionEncodings"` + } `json:"general"` + } `json:"capabilities"` +} +json.Unmarshal(m.Params, &p) +s.start(m.Params) +encoding := "utf-16" +for _, e := range p.Capabilities.General.PositionEncodings { + if e == "utf-8" && s.gopls == nil { // gopls counts in UTF-16, and that settles it + encoding, s.utf8 = e, true + } +} +capabilities := s.capabilities() +capabilities["positionEncoding"] = encoding +capabilities["textDocumentSync"] = map[string]any{"openClose": true, "change": 1} +s.write(&message{ID: m.ID, Result: map[string]any{ + "serverInfo": map[string]any{"name": "litgo"}, + "capabilities": capabilities, +}}) +``` + +```go +func (s *server) change(uri, text string, version int) { + if !strings.HasSuffix(uri, ".lit.md") { + return + } + d := s.docs[uri] + if d == nil { + d = &document{} + s.docs[uri] = d + } + if d.timer != nil { + d.timer.Stop() + } + d.text, d.version = text, version + d.timer = time.AfterFunc(s.delay, func() { + select { + case s.due <- due{uri, version}: + case <-s.quit: + } + }) +} + +// current returns the text and version of a document, or version -1. +func (s *server) current(uri string) (string, int) { + if d := s.docs[uri]; d != nil { + return d.text, d.version + } + return "", -1 +} +``` + +Diagnostics go out in two rounds. Problems with the document's structure and +with Go's syntax take microseconds to find, so the loop finds them itself and +publishes them right away. If there are none, the document needs a type check. +That can take a while, because the first one in a session has to wait for +`go list`, so it runs in a goroutine of its own and the loop stays free to +answer the editor. + +Only one type check runs at a time, because the cache is not locked, and +because a machine has better things to do than check five versions of one +document at once. While the checker is busy, the loop keeps the newest job for +each document in `waiting`, and a newer job replaces an older one. When the +checker sends its job back, the loop publishes the findings unless the document +has changed in the meantime, and then starts the next job. + +A document with `//@` annotations gets a third round, in the same goroutine, +once the types are in order: the [proofs](#proofs). A proof about a program +with a type error would be a second opinion about code that is about to +change, so it waits. It costs milliseconds, which is what makes it a thing to +do as you type rather than a thing to remember to do: strengthen an invariant, +and the violet line under the `Ensures` goes away before you have reached for +a command. + +```go +// job is a type check: what to check, and then what the check found. The +// loop and the checker pass it back and forth and never hold it together. +type job struct { + uri string + version int + res *lit.Result + diags []lit.Diag // published already + typed []lit.Diag // what the type checker adds +} + +func (s *server) analyze(uri string) { + res, version := s.sync(uri) + if res == nil { + return + } + diags := append(append([]lit.Diag{}, res.Diags...), res.SyntaxCheck()...) + s.publish(uri, version, res.Doc, diags) + if res.HasErrors() || len(diags) > len(res.Diags) { + return + } + s.waiting[uri] = &job{uri: uri, version: version, res: res, diags: diags} + s.check() +} + +// check gives the type checker its next job, if it is free and there is one +// that is still worth doing. The cache goes to the checker with the job and +// comes back with it. +func (s *server) check() { + if s.checking { + return + } + for uri, j := range s.waiting { + delete(s.waiting, uri) + if _, now := s.current(uri); now != j.version { + continue + } + s.checking = true + go func() { + j.typed = check.Types(j.res, &s.cache) + if len(j.typed) == 0 { + j.typed, _ = vego.Prove(j.res) + } + select { + case s.checked <- j: + case <-s.quit: + } + }() + return + } +} + +// finish publishes what a type check found, if it is still true. +func (s *server) finish(j *job) { + s.checking = false + if _, now := s.current(j.uri); now == j.version && len(j.typed) > 0 { + s.publish(j.uri, j.version, j.res.Doc, append(j.diags, j.typed...)) + } + s.check() +} + +func path(uri string) string { + if u, err := url.Parse(uri); err == nil && u.Scheme == "file" { + return u.Path + } + return uri +} +``` + +A diagnostic from litgo is a point, but an editor underlines a range. So litgo +stretches the range over the identifier at that point, when there is one. + +```go +func (s *server) publish(uri string, version int, doc *lit.Doc, diags []lit.Diag) { + type position struct { + Line int `json:"line"` + Character int `json:"character"` + } + type diagnostic struct { + Range struct { + Start position `json:"start"` + End position `json:"end"` + } `json:"range"` + Severity int `json:"severity"` + Source string `json:"source"` + Message string `json:"message"` + } + out := []diagnostic{} + for _, d := range diags { + line := "" + if d.Line >= 0 && d.Line < len(doc.Lines) { + line = doc.Lines[d.Line] + } + start := min(max(d.Col, 0), len(line)) + end := d.EndCol + if end <= start { + end = start + for end < len(line) && (line[end] == '_' || line[end] >= '0' && line[end] <= '9' || line[end] >= 'A' && line[end] <= 'Z' || line[end] >= 'a' && line[end] <= 'z' || line[end] >= 0x80) { + end++ + } + if end == start { + end = min(start+1, len(line)) + } + } + end = min(end, len(line)) + var ld diagnostic + ld.Range.Start = position{d.Line, s.column(line, start)} + ld.Range.End = position{d.Line, s.column(line, end)} + ld.Severity, ld.Source, ld.Message = d.Severity, "litgo", d.Message + if d.Source != "" && d.Source != "litgo" { + ld.Source = "litgo " + d.Source + } + out = append(out, ld) + } + params, _ := json.Marshal(map[string]any{"uri": uri, "version": version, "diagnostics": out}) + s.write(&message{Method: "textDocument/publishDiagnostics", Params: params}) +} + +// column converts a byte offset into the client's unit. +func (s *server) column(line string, offset int) int { + return units(s.utf8, line, offset) +} + +// units converts a byte offset in a line to UTF-16 code units, unless bytes +// are what is wanted. +func units(utf8 bool, line string, offset int) int { + offset = min(max(offset, 0), len(line)) + if utf8 { + return offset + } + n := 0 + for _, r := range line[:offset] { + n++ + if r >= 0x10000 { + n++ + } + } + return n +} +``` + + + + + +The test acts as a client. It opens a document that has a type error in a +chunk, fixes the error, and watches the diagnostics show up and then go away. + + +```go +const program = "\n\n\n```go\nfunc main() {\n\t// <>\n}\n```\n\n\n```go\nfmt.Println(\"π„ž\", whom)\n```\n" + +// client is the editor's end of a conversation with the server. +type client struct { + t *testing.T + to io.Writer + incoming chan *message +} + +func connect(t *testing.T, gopls string) *client { + toServer, fromClient := io.Pipe() + fromServer, toClient := io.Pipe() + go Serve(toServer, toClient, gopls) + t.Cleanup(func() { fromClient.Close() }) + c := &client{t: t, to: fromClient, incoming: make(chan *message, 64)} + go func() { + r := bufio.NewReader(fromServer) + for { + m, err := read(r) + if err != nil { + return + } + c.incoming <- m + } + }() + return c +} + +func (c *client) send(method string, id int, params any) { + raw, ok := params.(string) + if !ok { + b, _ := json.Marshal(params) + raw = string(b) + } + body := fmt.Sprintf(`{"jsonrpc":"2.0","method":%q,"params":%s}`, method, raw) + if id > 0 { + body = fmt.Sprintf(`{"jsonrpc":"2.0","id":%d,"method":%q,"params":%s}`, id, method, raw) + } + fmt.Fprintf(c.to, "Content-Length: %d\r\n\r\n%s", len(body), body) +} + +func (c *client) next() *message { + select { + case m := <-c.incoming: + return m + case <-time.After(60 * time.Second): + c.t.Fatal("the server said nothing for a minute") + return nil + } +} + +// diagnostics waits for a publication for the given version that satisfies ok. +func (c *client) diagnostics(version int, ok func(string) bool) string { + for { + m := c.next() + if m.Method == "textDocument/publishDiagnostics" && strings.Contains(string(m.Params), fmt.Sprintf(`"version":%d`, version)) && ok(string(m.Params)) { + return string(m.Params) + } + } +} + +func TestDiagnosticsComeAndGo(t *testing.T) { + c := connect(t, "off") + c.send("initialize", 1, `{"capabilities":{}}`) + if m := c.next(); !strings.Contains(fmt.Sprint(m.Result), "utf-16") { + t.Fatalf("a client that offers nothing gets UTF-16: %v", m.Result) + } + uri := "file://" + t.TempDir() + "/hello.lit.md" + c.send("textDocument/didOpen", 0, map[string]any{"textDocument": map[string]any{"uri": uri, "version": 1, "text": program}}) + got := c.diagnostics(1, func(p string) bool { return strings.Contains(p, "undefined: whom") }) + // Line 11, after fmt.Println(" (13 bytes), a clef that is 4 bytes but 2 UTF-16 units, and `", `. + if !strings.Contains(got, `"start":{"line":11,"character":18}`) || !strings.Contains(got, `"end":{"line":11,"character":22}`) { + t.Errorf("the error should underline whom, in UTF-16 units: %s", got) + } + + c.send("textDocument/didChange", 0, map[string]any{ + "textDocument": map[string]any{"uri": uri, "version": 2}, + "contentChanges": []any{map[string]any{"text": strings.Replace(program, "whom", `"world"`, 1)}}, + }) + c.diagnostics(2, func(p string) bool { return strings.Contains(p, `"diagnostics":[]`) }) + + c.send("textDocument/formatting", 2, `{}`) + if m := c.next(); m.Error == nil || m.Error.Code != -32601 { + t.Errorf("an unknown request should be refused politely: %+v", m) + } +} +``` + +# Everything else gopls knows + + + + + +Showing errors is the least an editor does for a Go programmer. It also +completes names, shows documentation, jumps to definitions, lists references +and renames things. gopls, Go's language server, does all of that, and none of +it should be written twice. But gopls reads Go files, and you are in a Markdown +file where the Go is scattered around in the wrong order. + +The source map works in both directions, so litgo can sit between the editor +and gopls. It starts its own gopls and shows it the tangled files as +*overlays*. An overlay is the protocol's word for the content of a file that is +open in an editor, and it takes precedence over what is on disk. Nothing gets +written. A question about a position in the document turns into the same +question about a position in a tangled file, and every position in the answer +gets translated back. + +```mermaid +sequenceDiagram + participant E as editor + participant S as litgo lsp + participant G as gopls + E->>S: didChange: the whole buffer + S->>G: didChange: pool.go, tangled, never written + E->>S: definition? pool.lit.md:67:15 + S->>G: definition? pool.go:31:14 + G-->>S: pool.go:12:6 + S-->>E: pool.lit.md:40:6 +``` + +The editor only sees one server, and to the editor a `.lit.md` buffer is a Go +buffer wherever the cursor is on Go. This works in any editor, because nothing +here is specific to Neovim. + +## A server of our own + +litgo looks for gopls on the `$PATH` first, and then in the directory where +`go install` puts it, because not everyone has that directory on their `$PATH`. +Without gopls, litgo is just the server from the last chapter. + +```go +// findGopls locates the gopls to run: the one named, or the one that can be +// found. "off" means none. +func findGopls(named string) string { + if named == "off" { + return "" + } + if named != "" { + p, _ := exec.LookPath(named) + return p + } + if p, err := exec.LookPath("gopls"); err == nil { + return p + } + gopath := os.Getenv("GOPATH") + if home, err := os.UserHomeDir(); gopath == "" && err == nil { + gopath = filepath.Join(home, "go") + } + for _, dir := range []string{os.Getenv("GOBIN"), filepath.Join(gopath, "bin")} { + if p, err := exec.LookPath(filepath.Join(dir, "gopls")); dir != "" && err == nil { + return p + } + } + return "" +} +``` + +From gopls's side, litgo is the client. Requests are numbered, and `pending` +holds what to do with each answer until the answer arrives. The editor might +cancel a request before then, so litgo also keeps the editor's own number for +the request. + +gopls adds two goroutines to the server, and no locks. One writes to gopls's +standard input and one reads its standard output. The reader sends each message +to the server's loop over `msgs`. The loop does all the bookkeeping and runs +every `done` function, so an answer from gopls gets translated by the same +goroutine that owns the documents and the overlays. + +```mermaid +sequenceDiagram + participant L as the loop + participant W as writer + participant G as gopls + participant R as reader + L->>W: out + W->>G: standard input + G->>R: standard output + R->>L: msgs +``` + +```go +// gopls is the Go language server, run as a child process. Like the rest of +// the server's state, it belongs to the loop in Serve. +type gopls struct { + cmd *exec.Cmd + in io.WriteCloser + out chan []byte // to the goroutine that writes to gopls + msgs chan *message // from the goroutine that reads from it + next int + pending map[int]call + ids map[string]int // the editor's id for a request in flight β†’ ours + dead bool + caps map[string]any // what gopls said it can do +} + +// call is a request in flight. done gets the answer, or nil if gopls died. +type call struct { + done func(*message) + editor string +} + +func (g *gopls) write(m *message) { g.out <- frame(m) } + +func (g *gopls) notify(method string, params any) { + raw, _ := json.Marshal(params) + g.write(&message{Method: method, Params: raw}) +} + +// request asks gopls something on behalf of the editor's request id (nil if +// the question is litgo's own). +func (g *gopls) request(id *json.RawMessage, method string, params any, done func(*message)) { + if g.dead { + done(nil) + return + } + raw, _ := json.Marshal(params) + c := call{done: done} + g.next++ + if id != nil { + c.editor = string(*id) + g.ids[c.editor] = g.next + } + g.pending[g.next] = c + ours := json.RawMessage(strconv.Itoa(g.next)) + g.write(&message{ID: &ours, Method: method, Params: raw}) +} + +// cancel passes on the editor's loss of interest in one of its requests. +func (g *gopls) cancel(params json.RawMessage) { + var p struct { + ID json.RawMessage `json:"id"` + } + json.Unmarshal(params, &p) + if n, ok := g.ids[string(p.ID)]; ok { + g.notify("$/cancelRequest", map[string]any{"id": n}) + } +} +``` + +gopls sends three kinds of message. An answer goes to whoever is waiting for +it. A request of its own (to register a capability, or to read configuration) +gets `null` back, and gopls takes that to mean "use your defaults". Of its +notifications, litgo only passes error messages on. The progress reports are +about files you never see. The diagnostics would repeat the ones the type +checker from the last chapter already placed, or in the worst case contradict +them. And the one warning gopls has is that someone is editing a generated +file. + +The reader drops the notifications nobody wants before they get to the loop, +because gopls sends a lot of them. It closes `msgs` when gopls hangs up. The +loop then marks gopls as dead and tells everyone who was waiting for an answer +that there won't be one. + +```go +// listen reads what gopls sends until it hangs up or the server stops. +func (g *gopls) listen(out io.Reader, quit <-chan struct{}) { + defer close(g.msgs) + defer g.cmd.Wait() + r := bufio.NewReader(out) + for { + m, err := read(r) + if err != nil { + return + } + if m.ID == nil && m.Method != "window/showMessage" { + continue + } + select { + case g.msgs <- m: + case <-quit: + return + } + } +} + +// heard is where messages from gopls arrive. Without a gopls it is nil, and +// a nil channel never delivers anything. +func (s *server) heard() <-chan *message { + if s.gopls == nil || s.gopls.dead { + return nil + } + return s.gopls.msgs +} + +// hear handles one message from gopls. +func (g *gopls) hear(m *message, s *server) { + switch { + case m.Method == "" && m.ID != nil: + n, _ := strconv.Atoi(string(*m.ID)) + c, ok := g.pending[n] + delete(g.pending, n) + delete(g.ids, c.editor) + if ok { + c.done(m) + } + case m.ID != nil: + g.write(&message{ID: m.ID, Result: json.RawMessage("null")}) + case m.Method == "window/showMessage": + var p struct { + Type int `json:"type"` + } + if json.Unmarshal(m.Params, &p); p.Type == 1 { + s.write(&message{Method: m.Method, Params: m.Params}) + } + } +} + +// hangUp is what happens when gopls is gone: nobody gets an answer any more. +func (g *gopls) hangUp() { + g.dead = true + orphans := g.pending + g.pending = map[int]call{} + for _, c := range orphans { + c.done(nil) + } +} +``` + +litgo starts gopls while the editor is waiting for the answer to its +`initialize`, because what litgo can promise the editor depends on what gopls +promises litgo. litgo tells gopls what the editor told litgo (the workspace, +and what the editor can display, like snippets in completions and Markdown in +hovers). It leaves out the capabilities that would make gopls send requests of +its own. + +The loop is the goroutine that runs `start`, so it can't be in its `select` +while it waits for gopls to answer. `start` receives from `msgs` itself until +the answer has come. + +```go +// start runs gopls and introduces litgo to it the way the editor introduced +// itself to litgo. On any failure s.gopls stays nil. +func (s *server) start(initialize json.RawMessage) { + bin := findGopls(s.goplsBin) + if bin == "" { + return + } + cmd := exec.Command(bin) + in, _ := cmd.StdinPipe() + out, _ := cmd.StdoutPipe() + if cmd.Start() != nil { + return + } + g := &gopls{cmd: cmd, in: in, msgs: make(chan *message), pending: map[int]call{}, ids: map[string]int{}} + g.out, _ = writer(in) + go g.listen(out, s.quit) + + var p map[string]any + if json.Unmarshal(initialize, &p); p == nil { + p = map[string]any{} + } + theirs, _ := p["capabilities"].(map[string]any) + p["capabilities"] = map[string]any{"textDocument": theirs["textDocument"]} + var reply *message + answered := false + g.request(nil, "initialize", p, func(m *message) { reply, answered = m, true }) + timeout := time.After(30 * time.Second) + for !answered { + select { + case m, ok := <-g.msgs: + if ok { + g.hear(m, s) + } else { + g.hangUp() // which answers the request, with nil + } + case <-timeout: + answered = true + } + } + if result, _ := reply.result().(map[string]any); result != nil { + g.caps, _ = result["capabilities"].(map[string]any) + g.notify("initialized", map[string]any{}) + s.gopls = g + return + } + g.kill() +} + +// kill stops gopls, and the goroutine that writes to it. +func (g *gopls) kill() { + close(g.out) + g.in.Close() + g.cmd.Process.Kill() +} + +// result is the result of a successful answer, and nil otherwise. +func (m *message) result() any { + if m == nil || m.Error != nil { + return nil + } + return m.Result +} + +func (s *server) stop() { + if g := s.gopls; g != nil { + g.kill() + } +} +``` + +## Overlays + +gopls has to see the latest text before it gets asked about it. A completion +request comes right after the keystroke that triggered it, sooner than the +delay that diagnostics wait for. So tangling doesn't wait. Whoever needs the +tangled files, whether that is the analysis or a request, calls `sync`, and +`sync` tangles at most once per version of the text. Both of them run on the +loop, so they can't get in each other's way. A document with a broken +chunk structure still gets tangled as well as it can be, because you are in the +middle of something and would still like completions. + +```go +// overlay is a tangled file as gopls was last shown it. +type overlay struct { + owner string // URI of the document it is tangled from + version int + text string +} + +// sync tangles a document, unless that has been done for this version of +// its text, and shows gopls the result. It returns a nil result for a +// document that is not open. +func (s *server) sync(uri string) (*lit.Result, int) { + d := s.docs[uri] + if d == nil { + return nil, -1 + } + if d.res == nil || d.tangled != d.version { + d.res, d.tangled = lit.Parse(path(uri), []byte(d.text)).Tangle(), d.version + s.overlay(uri, d.res) + } + return d.res, d.version +} + +// forget closes what a closed document tangled to. +func (s *server) forget(uri string) { + s.overlay(uri, nil) +} +``` + +A document like this one tangles to dozens of files, and a keystroke only +changes one of them. litgo only sends that one. + +```go +// overlay brings gopls up to date with what a document tangles to: new +// files are opened, changed ones changed, and ones no longer written closed. +func (s *server) overlay(uri string, res *lit.Result) { + g := s.gopls + if g == nil { + return + } + live := map[string]bool{} + for _, o := range outputs(res) { + live[o.Path] = true + text := string(o.Bytes()) + ov := s.overlays[o.Path] + switch { + case ov == nil: + s.overlays[o.Path] = &overlay{owner: uri, version: 1, text: text} + g.notify("textDocument/didOpen", map[string]any{ + "textDocument": map[string]any{"uri": fileURI(o.Path), "languageId": "go", "version": 1, "text": text}}) + case ov.text != text: + ov.owner, ov.text, ov.version = uri, text, ov.version+1 + g.notify("textDocument/didChange", map[string]any{ + "textDocument": map[string]any{"uri": fileURI(o.Path), "version": ov.version}, + "contentChanges": []any{map[string]any{"text": text}}}) + } + } + for p, ov := range s.overlays { + if ov.owner == uri && !live[p] { + delete(s.overlays, p) + g.notify("textDocument/didClose", map[string]any{ + "textDocument": map[string]any{"uri": fileURI(p)}}) + } + } +} + +// outputs lists the Go files of a result, which may be nil. +func outputs(res *lit.Result) []*lit.Output { + var out []*lit.Output + if res != nil { + for _, o := range res.Files { + if o.File.Lang == "go" { + out = append(out, o) + } + } + } + return out +} + +func fileURI(path string) string { + return (&url.URL{Scheme: "file", Path: filepath.ToSlash(path)}).String() +} +``` + +When gopls names a file in an answer, it might be one of these overlays. +`output` finds it, along with the document it came from, and prefers the +tangling that was current when the question was asked. + +```go +func (s *server) output(uri string, asked *lit.Result) (o *lit.Output, doc *lit.Doc, owner string) { + p := filepath.Clean(path(uri)) + res := asked + if ov := s.overlays[p]; ov != nil { + owner = ov.owner + if d := s.docs[owner]; d != nil && (asked == nil || asked.Doc.Path != path(owner)) { + res = d.res + } + } + if owner != "" { + for _, o := range outputs(res) { + if filepath.Clean(o.Path) == p { + return o, res.Doc, owner + } + } + } + return nil, nil, "" +} +``` + +## Questions + +These are the requests that ask about a position in a document, each with the +capability that announces it. litgo offers the editor whatever gopls offers +litgo, as long as it is in this list. It always offers four of them, because it +can answer those itself when the position is on a chunk. + +```go +var positional = map[string]string{ + "textDocument/hover": "hoverProvider", + "textDocument/definition": "definitionProvider", + "textDocument/typeDefinition": "typeDefinitionProvider", + "textDocument/implementation": "implementationProvider", + "textDocument/references": "referencesProvider", + "textDocument/documentHighlight": "documentHighlightProvider", + "textDocument/completion": "completionProvider", + "textDocument/signatureHelp": "signatureHelpProvider", + "textDocument/prepareRename": "renameProvider", + "textDocument/rename": "renameProvider", +} + +func (s *server) capabilities() map[string]any { + c := map[string]any{} + if s.gopls != nil { + for _, name := range positional { + if v, ok := s.gopls.caps[name]; ok { + c[name] = v + } + } + c["workspaceSymbolProvider"] = s.gopls.caps["workspaceSymbolProvider"] + } + c["hoverProvider"], c["definitionProvider"], c["referencesProvider"] = true, true, true + c["renameProvider"] = map[string]any{"prepareProvider": true} + return c +} +``` + +```go +// proxy handles the requests that gopls has a say in, and reports whether +// the message was one. +func (s *server) proxy(m *message) bool { + switch { + case positional[m.Method] != "" && m.ID != nil: + s.ask(m) + case m.Method == "$/cancelRequest": + if s.gopls != nil { + s.gopls.cancel(m.Params) + } + case m.Method == "workspace/symbol" && m.ID != nil && s.gopls != nil: + // No position to translate on the way in, but the symbols found + // may well be in tangled files. + var params any + json.Unmarshal(m.Params, ¶ms) + s.gopls.request(m.ID, m.Method, params, func(reply *message) { + s.relay(m, reply, &translation{s: s}) + }) + default: + return false + } + return true +} +``` + +`Locate` translates the position in the question. If the position is in prose, +or in a Lua block, or on a line that an exclude pattern drops, then it isn't +anywhere in a Go file and the answer is `null`. Every one of these requests +allows `null`, and editors accept it silently. + +```go +// position is the protocol's: a line, and a column in the agreed units. +type position struct { + Line int `json:"line"` + Character int `json:"character"` +} + +func (s *server) ask(m *message) { + var p struct { + TextDocument struct { + URI string `json:"uri"` + } `json:"textDocument"` + Position position `json:"position"` + NewName string `json:"newName"` + } + json.Unmarshal(m.Params, &p) + uri := p.TextDocument.URI + res, _ := s.sync(uri) + if res == nil || p.Position.Line < 0 || p.Position.Line >= len(res.Doc.Lines) { + s.respond(m, nil) + return + } + line := p.Position.Line + col := offset(s.utf8, res.Doc.Lines[line], p.Position.Character) + if mention := res.Doc.MentionAt(line, col); mention != nil { + // <> + return + } + o, outLine, outCol, ok := res.Locate(line, col) + if !ok || o.File.Lang != "go" || s.gopls == nil { + s.respond(m, nil) + return + } + var params map[string]any + json.Unmarshal(m.Params, ¶ms) + params["textDocument"] = map[string]any{"uri": fileURI(o.Path)} + params["position"] = position{outLine, units(false, o.Lines[outLine].Text, outCol)} + s.gopls.request(m.ID, m.Method, params, func(reply *message) { + s.relay(m, reply, &translation{s: s, res: res, out: o, uri: uri}) + }) +} + +// respond answers a request; a nil result is the protocol's null. +func (s *server) respond(to *message, result any) { + if result == nil { + result = json.RawMessage("null") + } + s.write(&message{ID: to.ID, Result: result}) +} +``` + +gopls counts columns in UTF-16 code units and doesn't offer anything else. That +is why litgo stops agreeing to bytes with the editor when gopls is there. It +means positions in files litgo knows nothing about (the standard library, a +dependency) can pass through untouched. litgo has the text of the documents and +the tangled files, so it converts the columns for those. + +```go +// offset converts a column in UTF-16 code units (or in bytes) to a byte +// offset in the line: the inverse of units. +func offset(utf8 bool, line string, n int) int { + if utf8 { + return min(max(n, 0), len(line)) + } + for i, r := range line { + if n <= 0 { + return i + } + n-- + if r >= 0x10000 { + n-- + } + } + return len(line) +} +``` + +## Questions about chunks + +A chunk reference is a comment, so gopls has nothing to say about it. But the +questions still make sense, and litgo knows the answers. A reference is +*defined* by the blocks of its chunk, hovering shows those blocks, the +*references* to a chunk are its references, and renaming it is `Rename`. If you +ask for the definition of a definition, litgo answers with the uses, so one key +takes you back and forth. All of this works with or without gopls. + + +```go +result, err := s.mention(m.Method, uri, res.Doc, mention, p.NewName) +if err != nil { + s.write(&message{ID: m.ID, Error: &rpcError{requestFailed, err.Error()}}) +} else { + s.respond(m, result) +} +``` + +```go +const requestFailed = -32803 + +func (s *server) mention(method, uri string, doc *lit.Doc, m *lit.Mention, newName string) (any, error) { + span := func(line, col, end int) map[string]any { + line = min(line, len(doc.Lines)-1) + text := doc.Lines[line] + return map[string]any{"start": position{line, s.column(text, col)}, "end": position{line, s.column(text, end)}} + } + c := m.Chunk + var uses, blocks []any + for _, r := range c.Refs { + uses = append(uses, map[string]any{"uri": uri, "range": span(r.Line, r.Col, r.EndCol)}) + } + for _, b := range c.Blocks { + blocks = append(blocks, map[string]any{"uri": uri, "range": span(b.First(), 0, 0)}) + } + switch method { + case "textDocument/definition": + if m.Ref { + return blocks, nil + } + return uses, nil + case "textDocument/references": + return uses, nil + case "textDocument/hover": + if !m.Ref || len(c.Blocks) == 0 { + return nil, nil + } + text := "⟨" + c.Name + "⟩ ≑\n" + for _, b := range c.Blocks { + text += "```" + b.Lang + "\n" + strings.Join(doc.Lines[b.First():b.Last()+1], "\n") + "\n```\n" + } + return map[string]any{"contents": map[string]any{"kind": "markdown", "value": text}, "range": span(m.Line, m.Col, m.EndCol)}, nil + case "textDocument/prepareRename": + return map[string]any{"range": span(m.Line, m.Col, m.EndCol), "placeholder": c.Name}, nil + case "textDocument/rename": + edits, err := doc.Rename(c.Name, newName) + if err != nil { + return nil, err + } + var changes []any + for _, e := range edits { // Rename replaces lines one for one + changes = append(changes, map[string]any{"range": span(e.Start, 0, len(doc.Lines[e.Start])), "newText": e.Lines[0]}) + } + return map[string]any{"changes": map[string]any{uri: changes}}, nil + } + return nil, nil +} +``` + +## Answers + +The answers vary as much as the questions do, but the positions in them only +come in a few shapes. So litgo does the translation on the JSON, without a type +for every kind of answer: + +* A *location* is a `uri` with a `range`, and a *location link* has a + `targetUri`. If the file is a tangled one, both get rewritten to point at the + document. Otherwise it is a file gopls found on disk, and it is left alone. +* A bare *range* anywhere else (the word a hover is about, an occurrence to + highlight) is in the file the question was about. +* A *text edit* is a range with a `newText`, and it gets the careful treatment + described below. + +A chunk that is used twice gets tangled twice, and gopls finds a reference in +each copy. Both translate back to the same place, so litgo removes the +duplicates from lists. + +```go +// translation brings the answer to one question back to the document. +type translation struct { + s *server + res *lit.Result // the tangling gopls was asked about, + out *lit.Output // the file in it that the question was about, + uri string // and the document +} + +func (t *translation) value(v any) any { + switch v := v.(type) { + case []any: + if len(v) > 0 && t.out != nil && isEdit(asMap(v[0])) { + return t.edits(t.out, t.res.Doc, v) + } + out, seen := []any{}, map[string]bool{} + for _, e := range v { + if e = t.value(e); e == nil { + continue + } + if key := positioned(e); key == "" || !seen[key] { + out = append(out, e) + seen[key] = true + } + } + return out + case map[string]any: + return t.object(v) + } + return v +} + +func (t *translation) object(v map[string]any) any { + if uri, ok := v["uri"].(string); ok && v["range"] != nil { + if o, doc, owner := t.s.output(uri, t.res); o != nil { + if v["range"], ok = t.span(o, doc, v["range"], false); !ok { + return nil + } + v["uri"] = owner + } + return v + } + if uri, ok := v["targetUri"].(string); ok { + if t.out != nil && v["originSelectionRange"] != nil { + v["originSelectionRange"], _ = t.span(t.out, t.res.Doc, v["originSelectionRange"], false) + } + if o, doc, owner := t.s.output(uri, t.res); o != nil { + for _, k := range []string{"targetRange", "targetSelectionRange"} { + if v[k], ok = t.span(o, doc, v[k], false); !ok { + return nil + } + } + v["targetUri"] = owner + } + return v + } + if t.out != nil && isEdit(v) { // on its own, as the textEdit of a completion + if t.edit(t.out, t.res.Doc, v) { + return v + } + return nil + } + if t.out != nil && isRange(v) { + r, _ := t.span(t.out, t.res.Doc, v, false) + return r + } + for k, e := range v { + if e = t.value(e); e == nil { + delete(v, k) // in the protocol, null and absent are the same + } else { + v[k] = e + } + } + return v +} + +func isRange(v map[string]any) bool { + _, start := v["start"].(map[string]any) + _, end := v["end"].(map[string]any) + return start && end +} + +func isEdit(v map[string]any) bool { + _, ok := v["newText"].(string) + return ok +} + +func asMap(v any) map[string]any { + m, _ := v.(map[string]any) + return m +} + +func asList(v any) []any { + l, _ := v.([]any) + return l +} + +// positioned gives a location, a highlight or an edit a key to be compared +// by, and anything else none. +func positioned(v any) string { + if m, ok := v.(map[string]any); ok && (m["range"] != nil || m["targetUri"] != nil) { + b, _ := json.Marshal(m) + return string(b) + } + return "" +} +``` + +A range gets translated one end at a time, exactly if possible. A range that is +only for looking at can fall back on `Map`'s approximation. For example, the +definition of a name in a generated `import` is close enough to the `imports:` +directive. A range to *replace* can't fall back like that. It also has to have +the same shape in the document as in the tangled file, and it doesn't if it +starts in a host line and ends in a chunk spliced into that line. + +```go +// span translates a range in a tangled file to the document it came from. +func (t *translation) span(o *lit.Output, doc *lit.Doc, v any, edit bool) (any, bool) { + r, _ := v.(map[string]any) + l1, c1, ok1 := outPosition(o, r["start"]) + l2, c2, ok2 := outPosition(o, r["end"]) + if !ok1 || !ok2 && edit { + return v, false + } + if !ok2 { + l2, c2 = l1, c1 + } + sl, sc, ok1 := o.Exact(l1, c1, false) + el, ec, ok2 := o.Exact(l2, c2, l1 != l2 || c1 != c2) + if edit && !(ok1 && ok2 && el-sl == l2-l1 && (l1 != l2 || ec-sc == c2-c1)) { + return v, false + } + if !ok1 { + if sl, sc, _, ok1 = o.Map(l1, c1); !ok1 { + return v, false + } + } + if !ok2 { + el, ec, _, ok2 = o.Map(l2, c2) + } + if !ok2 || el < sl || el == sl && ec < sc { + el, ec = sl, sc + } + return map[string]any{ + "start": position{sl, units(t.s.utf8, doc.Lines[sl], sc)}, + "end": position{el, units(t.s.utf8, doc.Lines[el], ec)}, + }, true +} + +// outPosition decodes a position in a tangled file, in bytes. +func outPosition(o *lit.Output, v any) (line, col int, ok bool) { + p, _ := v.(map[string]any) + l, ok1 := p["line"].(float64) + c, ok2 := p["character"].(float64) + if !ok1 || !ok2 || l < 0 || int(l) >= len(o.Lines) { + return 0, 0, false + } + return int(l), offset(false, o.Lines[int(l)].Text, int(c)), true +} +``` + +An edit that can't be translated gets dropped, with one exception. Completing +`strings.ToUpper` in a file that doesn't import `strings` comes with edits to +the `import (` block, and litgo generates that block from the `imports:` +directive. The edits are useless as they are, because gopls sends the smallest +difference it can find, and between `"fmt"` and `"fmt"`, newline, `"strings"` +that is a few characters in the middle of a line. But it is easy to find out +what the edits are *for*. litgo carries them out on the tangled file and looks +at what the file imports afterwards that it didn't import before. `AddImport` +knows how to add that to a document. + +```go +// edit translates a text edit in place, if it has a place in the document. +func (t *translation) edit(o *lit.Output, doc *lit.Doc, v map[string]any) bool { + ok := true + for _, k := range []string{"range", "insert", "replace"} { // a completion may offer both an insertion and a replacement + if r, has := v[k]; has && ok { + if l, _, _ := outPosition(o, asMap(r)["start"]); l < o.Header { + return false + } + v[k], ok = t.span(o, doc, r, true) + } + } + return ok +} + +// edits translates a list of text edits. Those to the generated header +// become edits to the imports directive. +func (t *translation) edits(o *lit.Output, doc *lit.Doc, list []any) []any { + out, header := []any{}, []map[string]any{} + for _, e := range list { + m := asMap(e) + if l, _, ok := outPosition(o, asMap(m["range"])["start"]); ok && l < o.Header { + header = append(header, m) + } else if t.edit(o, doc, m) { + out = append(out, m) + } + } + if len(header) == 0 { + return out + } + // <> + file, _ := parser.ParseFile(token.NewFileSet(), "", text, parser.ImportsOnly) + if file == nil { + return out + } + for _, spec := range file.Imports { + alias, path := "", strings.Trim(spec.Path.Value, "\"`") + if spec.Name != nil { + alias = spec.Name.Name + } + known := false + for _, imp := range o.File.Imports { + known = known || imp.Path == path + } + if line, col, entry, ok := doc.AddImport(o.File, alias, path); ok && !known { + at := position{line, units(t.s.utf8, doc.Lines[min(line, len(doc.Lines)-1)], col)} + out = append(out, map[string]any{"range": map[string]any{"start": at, "end": at}, "newText": entry}) + } + } + return out +} +``` + +The edits in a list all refer to the text as it was before any of them, so they +get carried out from the bottom up. + + +```go +text := string(o.Bytes()) +starts := []int{0} +for _, l := range o.Lines { + starts = append(starts, starts[len(starts)-1]+len(l.Text)+1) +} +at := func(v any) int { + l, c, ok := outPosition(o, v) + if !ok { + return -1 + } + return starts[l] + c +} +sort.SliceStable(header, func(i, j int) bool { + return at(asMap(header[i]["range"])["start"]) > at(asMap(header[j]["range"])["start"]) +}) +for _, m := range header { + r := asMap(m["range"]) + if from, to := at(r["start"]), at(r["end"]); from >= 0 && to >= from { + text = text[:from] + m["newText"].(string) + text[to:] + } +} +``` + +Renaming is the one answer that is made entirely of edits, in any number of +files. It is also the one where dropping an edit is not an option, because a +rename that is only partly carried out leaves a program that doesn't compile. +So litgo refuses the whole rename if any part of it can't be done. That +includes the case where gopls wants to change a tangled file whose document +isn't open. The change would be lost at the next tangle, and the right way to +make it is to open that document and rename there. + +The edits come back grouped by tangled file, and they go out grouped by +document. litgo strips the version numbers gopls put on them, because those are +versions of overlays the editor has never heard of. + +```go +// workspaceEdit translates the answer to a rename. +func (t *translation) workspaceEdit(v any) (any, error) { + w, _ := v.(map[string]any) + if w == nil { + return nil, nil + } + changes := map[string][]any{} + seen := map[string]bool{} + add := func(uri string, edits []any) error { + o, doc, owner := t.s.output(uri, t.res) + if o == nil { + if b, err := os.ReadFile(path(uri)); err == nil && lit.Generated(b) { + return fmt.Errorf("%s is tangled from a document that is not open: open it, and rename there", filepath.Base(path(uri))) + } + changes[uri] = append(changes[uri], edits...) + return nil + } + for _, e := range edits { + m := asMap(e) + at, _ := json.Marshal(m["range"]) + if !t.edit(o, doc, m) { + return fmt.Errorf("this would change text that litgo generates, in %s at %s", filepath.Base(o.Path), at) + } + if key := positioned(m); !seen[owner+key] { + seen[owner+key] = true + changes[owner] = append(changes[owner], m) + } + } + return nil + } + for uri, edits := range asMap(w["changes"]) { + if err := add(uri, asList(edits)); err != nil { + return nil, err + } + } + for _, c := range asList(w["documentChanges"]) { + change := asMap(c) + if change["kind"] != nil { + return nil, fmt.Errorf("this would %s a file, which a document's file directives decide", change["kind"]) + } + uri, _ := asMap(change["textDocument"])["uri"].(string) + if err := add(uri, asList(change["edits"])); err != nil { + return nil, err + } + } + return map[string]any{"changes": changes}, nil +} +``` + +```go +// relay passes gopls's answer to a request on to the editor. +func (s *server) relay(to *message, reply *message, t *translation) { + switch { + case reply != nil && reply.Error != nil: + s.write(&message{ID: to.ID, Error: reply.Error}) + case to.Method == "textDocument/rename": + edit, err := t.workspaceEdit(reply.result()) + if err != nil { + s.write(&message{ID: to.ID, Error: &rpcError{requestFailed, err.Error()}}) + return + } + s.respond(to, edit) + default: + s.respond(to, t.value(reply.result())) + } +} +``` + + + + + +The test needs gopls and gets skipped without it. Its document has a function +in one block, a call to that function in a chunk, and a clef character in front +of the call, which is two UTF-16 units and four bytes. Every position in the +test gets translated twice on the way in and twice on the way out. + + +```go +const greeter = "\n\n\n```go\nfunc greet(whom string) string {\n\treturn \"hello, \" + whom\n}\n\nfunc main() {\n\t// <>\n}\n```\n\n\n```go\nfmt.Println(\"π„ž\", greet(\"world\"))\n```\n" + +// ask sends a request about a position and returns the result as JSON. +func (c *client) ask(id int, method, uri string, line, character int, more map[string]any) string { + params := map[string]any{"textDocument": map[string]any{"uri": uri}, "position": map[string]any{"line": line, "character": character}} + for k, v := range more { + params[k] = v + } + c.send("textDocument/"+method, id, params) + return c.result(id) +} + +// result waits for the answer to a request. +func (c *client) result(id int) string { + for { + if m := c.next(); m.ID != nil && string(*m.ID) == strconv.Itoa(id) { + if m.Error != nil { + return "error: " + m.Error.Message + } + var sb strings.Builder + enc := json.NewEncoder(&sb) + enc.SetEscapeHTML(false) + enc.Encode(m.Result) + return strings.TrimSpace(sb.String()) + } + } +} + +// start spells the start of a range the way encoding/json does: keys sorted. +func start(line, character int) string { + return fmt.Sprintf(`"start":{"character":%d,"line":%d}`, character, line) +} + +func TestGoplsThroughTheDocument(t *testing.T) { + if findGopls("") == "" { + t.Skip("no gopls") + } + dir := t.TempDir() + os.WriteFile(dir+"/go.mod", []byte("module example.com/greeter\n\ngo 1.22\n"), 0o644) + uri := "file://" + dir + "/greeter.lit.md" + c := connect(t, "") + c.send("initialize", 1, map[string]any{"rootUri": "file://" + dir, "capabilities": map[string]any{ + "general": map[string]any{"positionEncodings": []string{"utf-8", "utf-16"}}}}) + if got := c.result(1); !strings.Contains(got, "completionProvider") || !strings.Contains(got, `"utf-16"`) { + t.Fatalf("with gopls behind it the server completes, and counts as gopls does: %s", got) + } + c.send("initialized", 0, `{}`) + c.send("textDocument/didOpen", 0, map[string]any{"textDocument": map[string]any{"uri": uri, "version": 1, "text": greeter}}) + + // Println, in the chunk. gopls may still be loading the first time. + hover := "" + for i := 0; i < 50 && !strings.Contains(hover, "func fmt.Println"); i++ { + if hover = c.ask(10+i, "hover", uri, 15, 5, nil); hover == "null" { + time.Sleep(200 * time.Millisecond) + } + } + if !strings.Contains(hover, "func fmt.Println") || !strings.Contains(hover, start(15, 4)) { + t.Errorf("hover in a chunk: %s", hover) + } + // greet, after the clef: from the chunk to the block that defines it. + if got := c.ask(2, "definition", uri, 15, 19, nil); !strings.Contains(got, uri) || !strings.Contains(got, start(4, 5)) { + t.Errorf("definition across blocks: %s", got) + } + if got := c.ask(3, "references", uri, 4, 6, map[string]any{"context": map[string]any{"includeDeclaration": true}}); strings.Count(got, uri) != 2 || !strings.Contains(got, start(15, 18)) { + t.Errorf("references, in UTF-16 units: %s", got) + } + if got := c.ask(4, "rename", uri, 4, 6, map[string]any{"newName": "hail"}); strings.Count(got, `"newText":"hail"`) != 2 || + !strings.Contains(got, start(15, 18)) || strings.Contains(got, "greeter.go") { + t.Errorf("rename edits the document, not the tangled file: %s", got) + } + if got := c.ask(5, "hover", uri, 0, 3, nil); got != "null" { + t.Errorf("prose is not Go: %s", got) + } + + // The chunk reference is litgo's own business. + if got := c.ask(6, "definition", uri, 9, 6, nil); !strings.Contains(got, start(15, 0)) { + t.Errorf("a reference is defined by its chunk: %s", got) + } + if got := c.ask(7, "hover", uri, 9, 6, nil); !strings.Contains(got, "⟨say it⟩") || !strings.Contains(got, "greet(") { + t.Errorf("hovering over a reference shows the chunk: %s", got) + } + if got := c.ask(8, "rename", uri, 13, 14, map[string]any{"newName": "speak"}); !strings.Contains(got, "") || !strings.Contains(got, "// <>") { + t.Errorf("renaming a chunk: %s", got) + } + + // Completing from a package that is not imported adds it to the directive. + c.send("textDocument/didChange", 0, map[string]any{ + "textDocument": map[string]any{"uri": uri, "version": 2}, + "contentChanges": []any{map[string]any{"text": strings.Replace(greeter, `greet("world")`, "strings.", 1)}}, + }) + got := c.ask(9, "completion", uri, 15, 26, nil) + if !strings.Contains(got, `"label":"ToUpper"`) { + t.Fatalf("completion: %.300s", got) + } + if !strings.Contains(got, `"newText":", strings"`) || !strings.Contains(got, start(1, 17)) { + i := strings.Index(got, `"label":"ToUpper"`) + t.Errorf("the import belongs in the imports directive: %.600s", got[max(i-300, 0):]) + } +} +``` + +# The Neovim plugin + +The plugin lives in this repository, at the root, where plugin managers look +for `plugin/` and `lua/`. It is a client of the protocol above, so the two have +to change together. With lazy.nvim: + +```lua +{ + "tlehman/litgo", + build = "go build -o bin/litgo .", + event = { "BufReadPre *.lit.md", "BufNewFile *.lit.md" }, + opts = {}, +} +``` + +The plugin is thin on purpose. Everything that knows about the format is in the +binary, and the Lua just turns answers into extmarks, edits and diagnostics. +Another editor would need the same four calls and nothing else. + +| Command | Key | | +|---|---|---| +| *(nothing)* | | Chunk, syntax and type errors show up under the offending line as you type, from `litgo lsp`. So do failed proofs of `//@` annotations, in violet. The annotations themselves are violet in the code, and their formulas are typeset like the math in the prose: `lo*lo <= n ^ n < hi*hi` reads $lo^2 \leq n \land n < hi^2$ until the cursor is on the line. | +| *(what Neovim has for any language server)* | `K`, `grr`, `grn`, ``, `` | In a Go block the buffer acts like a Go buffer. You get gopls's completion, documentation, references, rename and signature help, through `litgo lsp`. | +| `:TangleCompileAndRun [args]` | `\r` | Saves, tangles, compiles and runs. Panics and failing tests land in the Markdown too, and the cursor goes to the first error. | +| `:'<,'>Untangle [name]` | `\u` | Moves the selection into a chunk at the bottom and leaves a reference. | +| `:RenameChunk [name]` | `\n` | Renames the chunk under the cursor, everywhere. | +| `:Prove` | `\p` | Saves and checks the `//@` annotations, without compiling or running. | +| `:Tangle[!]`, `:Weave [html]` | `\t`, `\w` | `:Tangle` writes the sources. `:Weave` writes and opens the PDF, or the web page. | +| `gd`, `K` | | On a chunk, `gd` goes from a reference to the definition and from a definition to the uses, and `K` previews the chunk. On Go they do what they do in Go. | +| `:LitWatch`, `:LitStop`, `:LitClear`, `:LitRender` | | Run on save, kill the program, clear diagnostics, toggle rendering. | + + + +```lua +-- Loaded automatically when this directory is on the runtimepath. Calling +-- require("litgo").setup({...}) yourself (before or after) overrides options. +if vim.g.loaded_litgo then + return +end + +vim.g.loaded_litgo = true + +if vim.fn.has("nvim-0.11") == 0 then + vim.notify("litgo needs Neovim 0.11 or newer (inline virtual text, conceal_lines)", vim.log.levels.WARN) + return +end + +require("litgo").setup() +``` + +## Setup + + + +```lua +-- litgo.nvim: a tight loop for literate Go in .lit.md files. +-- +-- The plugin is deliberately thin. Everything that knows about the format +-- lives in the `litgo` binary; Lua asks it questions (document on stdin, JSON +-- back) and turns the answers into extmarks, edits and diagnostics. +local M = {} + +M.config = { + bin = nil, -- path to the litgo binary; found on $PATH when nil + render = { + enabled = true, + math = true, + mermaid = true, + tables = true, -- line up the columns of Markdown tables + proofs = true, -- typeset the formulas of //@ annotations: ^ as ∧, <= as ≀, Forall as βˆ€ + chunks = true, -- show `// <>` as ⟨name⟩ and chunk directives as ⟨name⟩ ≑ + conceal_source = true, -- hide diagram/display-math source while the cursor is elsewhere + upright = false, -- true: keep math letters upright instead of π‘–π‘‘π‘Žπ‘™π‘–π‘ + indent = 4, + debounce_ms = 150, + }, + run = { + vet = false, -- also report `go vet` findings as warnings + timeout = "60s", -- kill runaway programs; "" to disable + jump = true, -- move the cursor to the first error + focus = true, -- put the cursor in the output panel when the run ends, so q closes it + on_save = false, -- :TangleCompileAndRun after every write (toggle with :LitWatch) + panel_height = 12, + }, + lsp = { + enabled = true, -- run `litgo lsp`: chunk, syntax, type and proof errors as you type + virtual_lines = true, -- show them under the offending line rather than beside it + gopls = true, -- completion, hover, definitions, references, rename in Go blocks; or a path, or false + settings = nil, -- gopls settings, e.g. { buildFlags = { "-tags=integration" } } + completion = "auto", -- "auto": Neovim's own completion popup as you type, unless a + -- completion plugin is loaded. true forces it on, false off. + }, + weave = { open = true }, + untangle = { jump = false }, -- true: go to the new chunk to document it right away + keymaps = { + run = "r", + stop = "s", + tangle = "t", + weave = "w", + prove = "p", + untangle = "u", -- visual mode + rename = "n", + toggle_render = "v", + goto_chunk = "gd", + hover = "K", + }, +} +``` + +The plugin looks for the binary in its own checkout first. That is where a +plugin manager's build step puts it, and a binary from there is sure to match +the Lua. + +```lua +local function plugin_root() + local src = debug.getinfo(1, "S").source:sub(2) + return vim.fn.fnamemodify(src, ":p:h:h:h") +end + +local warned = false + +--- Locate the litgo binary: explicit config, a build inside the plugin's own +--- checkout (what a plugin manager's build step produces), then $PATH. +function M.bin() + if M._bin then + return M._bin + end + local candidates = { plugin_root() .. "/bin/litgo", "litgo" } + if M.config.bin then + table.insert(candidates, 1, vim.fn.expand(M.config.bin)) + end + for _, c in ipairs(candidates) do + if c and vim.fn.executable(c) == 1 then + M._bin = c + return c + end + end + if not warned then + warned = true + vim.notify( + "litgo: binary not found. Run `go build -o bin/litgo .` in " .. plugin_root() .. ", or set `bin` in setup().", + vim.log.levels.ERROR + ) + end +end +``` + +The plugin sees the protocol as two halves: ask a question about the buffer, +then apply the edits that come back. Several `nvim_buf_set_lines` calls made +from one command count as a single undo step. + +```lua +--- Run litgo synchronously with the buffer on stdin; returns decoded JSON. +function M.query(buf, args) + local bin = M.bin() + if not bin then + return nil, "litgo binary not found" + end + local cmd = vim.list_extend({ bin }, args) + local text = table.concat(vim.api.nvim_buf_get_lines(buf, 0, -1, false), "\n") .. "\n" + local res = vim.system(cmd, { stdin = text, text = true }):wait(5000) + local ok, decoded = pcall(vim.json.decode, res.stdout or "") + if not ok or type(decoded) ~= "table" then + return nil, (res.stderr ~= "" and res.stderr) or "litgo returned no JSON" + end + if decoded.error then + return nil, decoded.error + end + return decoded +end + +--- Apply bottom-up line edits as a single undo step. +function M.apply_edits(buf, edits) + for _, e in ipairs(edits) do + vim.api.nvim_buf_set_lines(buf, e.start, e["end"], false, e.lines) + end +end + +function M.is_lit(buf) + return vim.api.nvim_buf_get_name(buf):match("%.lit%.md$") ~= nil +end + +local function define_highlights() + local links = { + LitgoMath = "Special", + LitgoDiagram = "Function", + LitgoTableRule = "Comment", + LitgoChunk = "Title", + LitgoFile = "Directory", + LitgoInfo = "Comment", + LitgoError = "DiagnosticError", + LitgoOk = "DiagnosticOk", + } + for name, target in pairs(links) do + vim.api.nvim_set_hl(0, name, { link = target, default = true }) + end + -- <> +end +``` + +Everything about proofs is violet: the `//@` annotations in the code, and what +the verifier has to say about them. No colour scheme has a group for that, and +the point of the colour is that it is not one of the others. A red line under +code says the compiler will not take it, a yellow one that `go vet` frowns. A +violet one says something of a different kind: this compiles, it runs, and it +has not been shown to do what it promises. The groups are defined with +`default`, so a colour scheme or a user who disagrees about violet wins. + + +```lua +local dark = vim.o.background ~= "light" +local violet = dark and "#c792ea" or "#7c3aed" +local faded = dark and "#a58fc4" or "#8b6fc0" +for name, spec in pairs({ + LitgoProofKeyword = { fg = violet, bold = true }, + LitgoProof = { fg = faded }, + LitgoProofError = { fg = violet }, + LitgoProofWarn = { fg = faded, italic = true }, + LitgoProofUnderline = { undercurl = true, sp = violet }, +}) do + spec.default = true + vim.api.nvim_set_hl(0, name, spec) +end +``` + +A completion plugin offers the same sources everywhere in the buffer, which in +a Go block means Markdown snippets and words from the prose alongside what +gopls has to say. The plugin can't reach into someone else's configuration, so +it answers the question instead: is this line inside a Go block? Counting +fences from the top is enough, and cheap, because a completion menu only asks +about the line the cursor is on. + +```lua +--- Is a line (0-based; the cursor's by default) inside a Go block? +function M.in_go_block(buf, line) + buf = buf or 0 + if not M.is_lit(buf) then + return false + end + line = line or (vim.api.nvim_win_get_cursor(0)[1] - 1) + local lang, fence = nil, nil + for _, l in ipairs(vim.api.nvim_buf_get_lines(buf, 0, line, false)) do + local marks, rest = l:match("^%s*(``+`)%s*(.*)$") + if marks then + if not fence then + fence, lang = marks, rest:match("^[%w_+-]*") + elseif #marks >= #fence then + fence, lang = nil, nil + end + end + end + return lang == "go" +end +``` + +The filetype stays `markdown`, so treesitter highlighting and everything else +that works for Markdown keeps working. The commands are local to the buffer. + +```lua +--- Set up one .lit.md buffer: commands, keymaps, rendering. +function M.attach(buf) + if vim.b[buf].litgo_attached or not M.is_lit(buf) then + return + end + vim.b[buf].litgo_attached = true + + local run = require("litgo.run") + local chunks = require("litgo.chunks") + local render = require("litgo.render") + + local function command(name, fn, opts) + vim.api.nvim_buf_create_user_command(buf, name, fn, opts or {}) + end + command("Tangle", function(o) + run.tangle(buf, o.bang) + end, { bang = true, desc = "Write the Go source (! overwrites files litgo did not generate)" }) + command("Weave", function(o) + run.weave(buf, o.args) + end, { nargs = "?", desc = "Write and open the woven PDF (or :Weave html)" }) + command("TangleCompileAndRun", function(o) + run.run(buf, o.fargs) + end, { nargs = "*", desc = "Tangle, compile and run; errors appear inline" }) + command("Prove", function() + run.prove(buf) + end, { desc = "Check the //@ annotations: contracts, invariants, termination" }) + command("LitStop", function() + run.stop() + end, { desc = "Kill the running program" }) + command("LitWatch", function() + M.config.run.on_save = not M.config.run.on_save + vim.notify("litgo: run on save " .. (M.config.run.on_save and "on" or "off")) + end, { desc = "Toggle :TangleCompileAndRun on every write" }) + command("LitClear", function() + run.clear(buf) + end, { desc = "Clear compile and run diagnostics" }) + command("Untangle", function(o) + chunks.untangle(buf, o) + end, { range = true, nargs = "*", desc = "Move the selection into a named chunk at the bottom" }) + command("RenameChunk", function(o) + chunks.rename(buf, o.args) + end, { nargs = "*", desc = "Rename the chunk under the cursor everywhere" }) + command("LitRender", function() + render.toggle(buf) + end, { desc = "Toggle math, diagram and chunk rendering" }) + + local keys = M.config.keymaps or {} + local function map(mode, lhs, rhs, desc) + if lhs and lhs ~= "" then + vim.keymap.set(mode, lhs, rhs, { buffer = buf, silent = true, desc = "litgo: " .. desc }) + end + end + map("n", keys.run, "TangleCompileAndRun", "tangle, compile and run") + map("n", keys.stop, "LitStop", "stop the program") + map("n", keys.tangle, "Tangle", "tangle") + map("n", keys.weave, "Weave", "weave") + map("n", keys.prove, "Prove", "prove the annotations") + map("x", keys.untangle, ":Untangle", "untangle selection into a chunk") + map("n", keys.rename, "RenameChunk", "rename chunk") + map("n", keys.toggle_render, "LitRender", "toggle rendering") + map("n", keys.goto_chunk, chunks.goto_chunk, "go to chunk definition / uses") + map("n", keys.hover, chunks.hover, "preview chunk") + + vim.api.nvim_create_autocmd("BufWritePost", { + buffer = buf, + callback = function() + if M.config.run.on_save then + run.run(buf, {}) + end + end, + }) + + run.attach(buf) + render.attach(buf) + require("litgo.lsp").start(buf) +end + +function M.setup(opts) + M.config = vim.tbl_deep_extend("force", M.config, opts or {}) + M._bin = nil + define_highlights() + require("litgo.proof").setup() + local group = vim.api.nvim_create_augroup("litgo", { clear = true }) + vim.api.nvim_create_autocmd({ "BufReadPost", "BufNewFile", "BufWinEnter" }, { + group = group, + pattern = "*.lit.md", + callback = function(ev) + M.attach(ev.buf) + end, + }) + vim.api.nvim_create_autocmd("ColorScheme", { group = group, callback = define_highlights }) + for _, buf in ipairs(vim.api.nvim_list_bufs()) do + if vim.api.nvim_buf_is_loaded(buf) then + M.attach(buf) + end + end +end + +return M +``` + +## Rendering + + + +```lua +-- In-buffer rendering: LaTeX as Unicode, Mermaid as box drawings, and chunk +-- syntax as ⟨names⟩. Whatever the cursor touches falls back to its source, +-- with block renderings kept underneath as a live preview. +local M = {} + +local ns = vim.api.nvim_create_namespace("litgo-render") + +local check_ns = vim.api.nvim_create_namespace("litgo-check") + +M.check_ns = check_ns + +-- bufnr -> { items, chunks, enabled, dirty, timer, cursor_row } +local state = {} + +local function litgo() + return require("litgo") +end + +function M.get(buf) + return state[buf] +end + +local function virt_lines(lines, hl, indent) + local pad = string.rep(" ", indent) + local out = {} + for _, l in ipairs(lines) do + out[#out + 1] = { { pad .. l, hl } } + end + return out +end + +local function mark(buf, row, col, opts) + opts.strict = false + pcall(vim.api.nvim_buf_set_extmark, buf, ns, row, col, opts) +end + +local function wanted(item, cfg) + if item.kind == "math" or item.kind == "math-block" then + return cfg.math + elseif item.kind == "mermaid" then + return cfg.mermaid + elseif item.kind == "table" then + return cfg.tables + elseif item.kind == "vego-math" then + return cfg.proofs + end + return cfg.chunks +end +``` + +Everything is drawn with extmarks, and whatever the cursor is on doesn't get +drawn, so the source is always one keystroke away from its rendering. + +```lua +--- Redraw every extmark. Cheap enough (one pass over the items) to run on +--- each cursor-line change, which keeps the reveal logic trivial. +function M.draw(buf) + local st = state[buf] + if not st or not vim.api.nvim_buf_is_valid(buf) then + return + end + vim.api.nvim_buf_clear_namespace(buf, ns, 0, -1) + if not st.enabled or not st.items then + return + end + local cfg = litgo().config.render + local cursor = -1 + if vim.api.nvim_get_current_buf() == buf then + cursor = vim.api.nvim_win_get_cursor(0)[1] - 1 + end + st.cursor_row = cursor + local last = vim.api.nvim_buf_line_count(buf) - 1 + + for _, it in ipairs(st.items) do + if it.kind == "vego" then + -- An annotation keeps its colour under the cursor too: nothing is + -- hidden, so there is nothing to reveal. + local key_end = it.col + 3 + if it.name and it.name ~= "" then + local line = vim.api.nvim_buf_get_lines(buf, it.line, it.line + 1, false)[1] or "" + local _, e = line:find(it.name, it.col + 4, true) + key_end = e or key_end + end + mark(buf, it.line, it.col, { end_col = key_end, hl_group = "LitgoProofKeyword", priority = 210 }) + mark(buf, it.line, key_end, { end_col = it.end_col, hl_group = "LitgoProof", priority = 210 }) + elseif wanted(it, cfg) then + local under_cursor = cursor >= it.line and cursor <= it.end_line + if it.lines then + -- <> + elseif it.table then + -- <> + else + -- <> + end + end + end +end +``` + +A block rendering replaces its source. The source lines get concealed, and the +drawing hangs off a neighbouring line as virtual lines. While the cursor is +inside the block, the source shows and the drawing stays underneath as a live +preview. + + +```lua +local hl = it.kind == "mermaid" and "LitgoDiagram" or "LitgoMath" +if it.error then + hl = "LitgoError" +end +local vl = virt_lines(it.lines, hl, cfg.indent) +if under_cursor or not cfg.conceal_source then + -- Editing: keep the source visible and preview underneath. Anchor to + -- the line after the block: treesitter conceals fence lines, and + -- virtual lines hanging off a concealed line disappear with it. + if it.end_line < last then + mark(buf, it.end_line + 1, 0, { virt_lines = vl, virt_lines_above = true }) + else + mark(buf, it.end_line, 0, { virt_lines = vl }) + end +elseif it.end_line < last then + mark(buf, it.line, 0, { end_row = it.end_line, conceal_lines = "" }) + mark(buf, it.end_line + 1, 0, { virt_lines = vl, virt_lines_above = true }) +elseif it.line > 0 then + mark(buf, it.line, 0, { end_row = it.end_line, conceal_lines = "" }) + mark(buf, it.line - 1, 0, { virt_lines = vl }) +else + mark(buf, it.end_line, 0, { virt_lines = vl }) +end +``` + +A table stays where it is, and its lines stay lines of the buffer. The only +things added are spaces, as inline virtual text, so that the pipes of every row +end up under the pipes of the header. A column is as wide as its widest cell +looks. A cell that looks narrower gets the difference as spaces: after its text +if the column is aligned left, before its text if it is aligned right, and half +on each side if it is centred. The rule under the header is the one thing that +gets replaced, by dashes that are as wide as the column. The row the cursor is +on is left exactly as it was typed, like everything else the cursor is on. + +The widths get measured with `strdisplaywidth`, because that is the editor's +own answer to how many screen cells a string takes up. They are kept with the +item, so moving the cursor doesn't measure every table again. + + +```lua +local t = it.table +local function looks(cell) + return (cell.col - cell.from) + vim.fn.strdisplaywidth(cell.text) + (cell.to - cell.end_col) +end +if not it.widths then + it.widths = {} + for _, row in ipairs(t.rows) do + for c, cell in ipairs(row.cells) do + it.widths[c] = math.max(it.widths[c] or 3, looks(cell)) + end + end +end +local function spaces(row, col, n) + if n > 0 then + mark(buf, row, col, { virt_text = { { string.rep(" ", n) } }, virt_text_pos = "inline" }) + end +end +for _, row in ipairs(t.rows) do + if row.line ~= cursor then + for _, h in ipairs(row.hide or {}) do + mark(buf, row.line, h[1], { end_col = h[2], conceal = "" }) + end + for c, cell in ipairs(row.cells) do + local gap = (it.widths[c] or 0) - looks(cell) + local before = ({ right = gap, center = math.floor(gap / 2) })[t.align[c]] or 0 + spaces(row.line, cell.from, before) + spaces(row.line, cell.to, gap - before) + end + end +end +if it.line + 1 ~= cursor then + for c, cell in ipairs(t.rule) do + local w, a = it.widths[c] or 3, t.align[c] + local dashes = (a == "center" and ":" or "-") .. string.rep("-", w - 2) .. (a == "left" and "-" or ":") + mark(buf, it.line + 1, cell.from, { end_col = cell.to, conceal = "" }) + mark(buf, it.line + 1, cell.from, { virt_text = { { dashes, "LitgoTableRule" } }, virt_text_pos = "inline" }) + end +end +``` + + +```lua +local hl = ({ math = "LitgoMath", file = "LitgoFile", ["vego-math"] = "LitgoProof" })[it.kind] or "LitgoChunk" +if it.info == "undefined" then + hl = "LitgoError" +end +if not under_cursor and it.text then + mark(buf, it.line, it.col, { end_col = it.end_col, conceal = "" }) + mark(buf, it.line, it.col, { virt_text = { { it.text, hl } }, virt_text_pos = "inline" }) +end +if it.info and it.info ~= "" then + mark(buf, it.line, 0, { virt_text = { { " " .. it.info, "LitgoInfo" } }, virt_text_pos = "eol" }) +end +``` + +Analysis is asynchronous and debounced. An answer only gets used if the buffer +hasn't changed since the question was asked. + +```lua +local function to_diagnostics(list, buf) + local name = vim.api.nvim_buf_get_name(buf) + local count = vim.api.nvim_buf_line_count(buf) + local out = {} + for _, d in ipairs(list or {}) do + if not d.file or d.file == "" or d.file == name then + out[#out + 1] = { + lnum = math.min(d.line, count - 1), + col = d.col or 0, + end_col = (d.end_col and d.end_col > 0) and d.end_col or nil, + severity = d.severity, + message = d.message, + source = d.source, + } + end + end + return out +end + +M.to_diagnostics = to_diagnostics + +--- Ask litgo about the buffer and redraw when the answer is still current. +function M.refresh(buf) + local st = state[buf] + local bin = litgo().bin() + if not st or not bin or not vim.api.nvim_buf_is_valid(buf) then + return + end + local tick = vim.api.nvim_buf_get_changedtick(buf) + local cmd = { bin, "analyze", "--path", vim.api.nvim_buf_get_name(buf) } + if litgo().config.render.upright then + cmd[#cmd + 1] = "--upright" + end + local text = table.concat(vim.api.nvim_buf_get_lines(buf, 0, -1, false), "\n") .. "\n" + vim.system(cmd, { stdin = text, text = true }, function(res) + vim.schedule(function() + if not vim.api.nvim_buf_is_valid(buf) then + return + end + if vim.api.nvim_buf_get_changedtick(buf) ~= tick then + -- The buffer moved on. Usually a debounced request is already queued; + -- if not (the tick also bumps while a file finishes loading), ask again. + if not st.dirty then + M.refresh(buf) + end + return + end + local ok, decoded = pcall(vim.json.decode, res.stdout or "") + if not ok or type(decoded) ~= "table" then + return + end + st.items, st.chunks, st.dirty = decoded.items, decoded.chunks, false + M.draw(buf) + -- Reporting problems is the language server's job. Without it, the + -- ones this analysis found are better than none; syntax errors while + -- typing are noise, so they wait until insert mode ends. + if require("litgo.lsp").active(buf) then + vim.diagnostic.reset(check_ns, buf) + elseif vim.api.nvim_get_mode().mode:sub(1, 1) ~= "i" then + vim.diagnostic.set(check_ns, buf, to_diagnostics(decoded.diagnostics, buf)) + end + end) + end) +end + +function M.schedule(buf) + local st = state[buf] + if not st then + return + end + st.dirty = true + if st.timer then + st.timer:stop() + else + st.timer = vim.uv.new_timer() + end + st.timer:start(litgo().config.render.debounce_ms, 0, function() + vim.schedule(function() + M.refresh(buf) + end) + end) +end + +function M.toggle(buf) + local st = state[buf] + if st then + st.enabled = not st.enabled + M.draw(buf) + end +end + +local function window_options(buf) + for _, win in ipairs(vim.fn.win_findbuf(buf)) do + vim.wo[win][0].conceallevel = 2 + vim.wo[win][0].concealcursor = "" + end +end + +function M.attach(buf) + state[buf] = { enabled = litgo().config.render.enabled } + local group = vim.api.nvim_create_augroup("litgo-render-" .. buf, { clear = true }) + vim.api.nvim_create_autocmd({ "TextChanged", "TextChangedI", "TextChangedP" }, { + group = group, + buffer = buf, + callback = function() + M.schedule(buf) + end, + }) + vim.api.nvim_create_autocmd("InsertLeave", { + group = group, + buffer = buf, + callback = function() + M.refresh(buf) + end, + }) + vim.api.nvim_create_autocmd({ "CursorMoved", "CursorMovedI", "BufEnter", "WinEnter" }, { + group = group, + buffer = buf, + callback = function() + local st = state[buf] + -- While an analysis is pending the item positions are stale; the + -- refresh that follows will redraw. + if st and not st.dirty and st.cursor_row ~= vim.api.nvim_win_get_cursor(0)[1] - 1 then + M.draw(buf) + end + end, + }) + vim.api.nvim_create_autocmd("BufWinEnter", { + group = group, + buffer = buf, + callback = function() + window_options(buf) + end, + }) + vim.api.nvim_create_autocmd("BufWipeout", { + group = group, + buffer = buf, + callback = function() + local st = state[buf] + if st and st.timer then + st.timer:stop() + st.timer:close() + end + state[buf] = nil + end, + }) + vim.diagnostic.config({ virtual_text = true, signs = true, underline = true, [require("litgo.proof").handler] = true }, check_ns) + window_options(buf) + M.refresh(buf) +end + +return M +``` + +## Tangle, weave, run + + + +```lua +-- :Tangle, :Weave and :TangleCompileAndRun. The run loop streams JSON events +-- from `litgo run --json`; diagnostics arrive already translated to .lit.md +-- positions, so they can be placed in the buffer the programmer is editing. +local M = {} + +local run_ns = vim.api.nvim_create_namespace("litgo-run") + +M.ns = run_ns + +local job -- the running vim.system object + +local panel = { buf = nil } + +local function litgo() + return require("litgo") +end +``` + +Program output streams into a panel at the bottom. The positions in it are +already in terms of the document, and pressing `` on one jumps there, as +`q` closes it again. + +While the program is running the cursor stays where the typing was, because a +panel that steals the cursor mid-run is a panel that eats keystrokes. So the +panel opens beside the work and gives the window back, and takes the cursor +only once the run is over β€” unless the run ended at an error, where the cursor +has somewhere better to be. + +```lua +local function panel_buf() + if panel.buf and vim.api.nvim_buf_is_valid(panel.buf) then + return panel.buf + end + local buf = vim.api.nvim_create_buf(false, true) + vim.api.nvim_buf_set_name(buf, "litgo://output") + vim.bo[buf].buftype = "nofile" + vim.bo[buf].bufhidden = "hide" + vim.bo[buf].swapfile = false + vim.bo[buf].filetype = "litgo-output" + vim.keymap.set("n", "q", "close", { buffer = buf, silent = true }) + -- on a `file.lit.md:12:3` reference jumps there. + vim.keymap.set("n", "", function() + local file, lnum, col = vim.api.nvim_get_current_line():match("([^%s:]+%.lit%.md):(%d+):?(%d*)") + if not file then + return + end + if vim.fn.filereadable(file) == 0 and panel.dir then + file = panel.dir .. "/" .. file + end + local target = vim.fn.bufnr(file) + for _, win in ipairs(vim.api.nvim_list_wins()) do + if vim.api.nvim_win_get_buf(win) == target then + vim.api.nvim_set_current_win(win) + vim.api.nvim_win_set_cursor(win, { tonumber(lnum), math.max((tonumber(col) or 1) - 1, 0) }) + return + end + end + vim.cmd("wincmd p | edit +" .. lnum .. " " .. vim.fn.fnameescape(file)) + end, { buffer = buf, silent = true }) + panel.buf = buf + return buf +end + +local function panel_win() + local buf = panel_buf() + for _, win in ipairs(vim.api.nvim_list_wins()) do + if vim.api.nvim_win_get_buf(win) == buf then + return win + end + end + local current = vim.api.nvim_get_current_win() + vim.cmd("botright " .. litgo().config.run.panel_height .. "split") + local win = vim.api.nvim_get_current_win() + vim.api.nvim_win_set_buf(win, buf) + vim.wo[win].number = false + vim.wo[win].relativenumber = false + vim.wo[win].signcolumn = "no" + vim.wo[win].winfixheight = true + vim.wo[win].wrap = true + vim.api.nvim_set_current_win(current) + return win +end + +--- Put the cursor in the panel, if it is on screen. +local function panel_focus() + local buf = panel.buf + if not (buf and vim.api.nvim_buf_is_valid(buf)) then + return + end + for _, win in ipairs(vim.api.nvim_list_wins()) do + if vim.api.nvim_win_get_buf(win) == buf then + vim.api.nvim_set_current_win(win) + return + end + end +end + +local function panel_clear() + local buf = panel_buf() + vim.api.nvim_buf_set_lines(buf, 0, -1, false, {}) + vim.api.nvim_buf_clear_namespace(buf, run_ns, 0, -1) +end + +local function panel_add(text, hl) + local buf = panel_buf() + if panel.dir then + text = text:gsub(vim.pesc(panel.dir .. "/"), "") -- positions read better without the directory + end + local lines = vim.split(text, "\n", { plain = true }) + local count = vim.api.nvim_buf_line_count(buf) + local empty = count == 1 and vim.api.nvim_buf_get_lines(buf, 0, 1, false)[1] == "" + local first = empty and 0 or count + vim.api.nvim_buf_set_lines(buf, first, -1, false, lines) + if hl then + for i = 0, #lines - 1 do + vim.api.nvim_buf_set_extmark(buf, run_ns, first + i, 0, { line_hl_group = hl }) + end + end + local win = panel_win() + vim.api.nvim_win_set_cursor(win, { vim.api.nvim_buf_line_count(buf), 0 }) +end +``` + +Diagnostics from a run are shown as virtual lines, right under the code that +caused them. They stay accurate while you edit. Changing a line dismisses the +error on it, and errors below an edit move with their text. + +```lua +--- Place diagnostics in every file they mention and mirror them in quickfix. +local function publish(diags, origin) + local by_file, qf = {}, {} + for _, d in ipairs(diags) do + local file = (d.file and d.file ~= "") and d.file or vim.api.nvim_buf_get_name(origin) + by_file[file] = by_file[file] or {} + table.insert(by_file[file], d) + qf[#qf + 1] = { + filename = file, + lnum = d.line + 1, + col = d.col + 1, + text = d.message, + type = d.severity == 1 and "E" or "W", + } + end + for file, list in pairs(by_file) do + local buf = vim.fn.bufadd(file) + vim.fn.bufload(buf) + -- What the compiler and the verifier say, the language server has usually + -- said already. + if require("litgo.lsp").reporting(buf) then + list = vim.tbl_filter(function(d) + return d.source ~= "go build" and d.source ~= "vego" + end, list) + end + vim.diagnostic.set(run_ns, buf, require("litgo.render").to_diagnostics(list, buf)) + end + vim.fn.setqflist({}, " ", { title = "litgo", items = qf }) +end + +function M.clear(buf) + vim.diagnostic.reset(run_ns) + if buf then + vim.diagnostic.reset(run_ns, buf) + end +end + +--- Keep run diagnostics accurate while editing: fixing a line dismisses its +--- error, and errors below an edit move with their text. +function M.attach(buf) + vim.diagnostic.config({ + virtual_lines = true, -- the message sits under the offending line + virtual_text = false, + signs = true, + underline = true, + severity_sort = true, + [require("litgo.proof").handler] = true, + }, run_ns) + -- Several edits can arrive before the scheduled update runs (a macro, a + -- :global command), so each works on the result of the one before. + local pending + vim.api.nvim_buf_attach(buf, false, { + on_lines = function(_, _, _, first, last_old, last_new) + local diags = pending or vim.diagnostic.get(buf, { namespace = run_ns }) + if #diags == 0 then + return + end + local kept, changed = {}, false + for _, d in ipairs(diags) do + if d.lnum >= first and d.lnum < math.max(last_old, first + 1) then + changed = true + else + if d.lnum >= last_old and last_new ~= last_old then + d.lnum = d.lnum + last_new - last_old + d.end_lnum = d.end_lnum and d.end_lnum + last_new - last_old + changed = true + end + kept[#kept + 1] = d + end + end + if changed and not pending then + vim.schedule(function() + if vim.api.nvim_buf_is_valid(buf) then + vim.diagnostic.set(run_ns, buf, pending) + end + pending = nil + end) + end + if changed then + pending = kept + end + end, + }) +end +``` + +```lua +local function save(buf) + if vim.bo[buf].modified then + vim.api.nvim_buf_call(buf, function() + vim.cmd("silent update") + end) + end +end + +function M.tangle(buf, force) + local bin = litgo().bin() + if not bin then + return + end + save(buf) + local cmd = { bin, "tangle" } + if force then + cmd[#cmd + 1] = "--force" + end + cmd[#cmd + 1] = vim.api.nvim_buf_get_name(buf) + vim.system(cmd, { text = true }, function(res) + vim.schedule(function() + local msg = vim.trim(res.stderr or "") + vim.notify(msg ~= "" and msg or "litgo: tangled", res.code == 0 and vim.log.levels.INFO or vim.log.levels.ERROR) + end) + end) +end + +function M.weave(buf, format) + local bin = litgo().bin() + if not bin then + return + end + save(buf) + local cmd = { bin, "weave" } + if format and format ~= "" then + vim.list_extend(cmd, { "--to", format }) + end + cmd[#cmd + 1] = vim.api.nvim_buf_get_name(buf) + vim.system(cmd, { text = true }, function(res) + vim.schedule(function() + if res.code ~= 0 then + vim.notify(vim.trim(res.stderr or "litgo: weave failed"), vim.log.levels.ERROR) + return + end + local out = vim.trim(res.stdout) + vim.notify("litgo: wove " .. vim.fn.fnamemodify(out, ":~:.")) + if litgo().config.weave.open then + vim.ui.open(out) + end + end) + end) +end + +--- Check the //@ annotations without compiling or running anything. +function M.prove(buf) + local bin = litgo().bin() + if not bin then + return + end + save(buf) + M.clear(buf) + vim.system({ bin, "prove", "--json", vim.api.nvim_buf_get_name(buf) }, { text = true }, function(res) + vim.schedule(function() + local ok, decoded = pcall(vim.json.decode, res.stdout or "") + if not ok or type(decoded) ~= "table" then + vim.notify("litgo: prove failed\n" .. (res.stderr or ""), vim.log.levels.ERROR) + return + end + publish(decoded.diagnostics or {}, buf) + vim.g.litgo_last_proof = ("%d/%d"):format(decoded.proved, decoded.functions) + local level = decoded.proved == decoded.functions and vim.log.levels.INFO or vim.log.levels.WARN + if decoded.functions == 0 then + vim.notify("litgo: no //@ annotations, so nothing to prove") + else + vim.notify(("litgo: ∴ proved %d of %d annotated functions (%d obligations)"):format(decoded.proved, decoded.functions, decoded.obligations), level) + end + end) + end) end -require("litgo").setup() +function M.stop() + if job then + job:kill("sigterm") + job = nil + panel_add("β–  stopped", "LitgoError") + end +end + +function M.run(buf, args) + local bin = litgo().bin() + if not bin then + return + end + if job then + job:kill("sigterm") + job = nil + end + save(buf) + M.clear(buf) + panel_clear() + panel.dir = vim.fn.fnamemodify(vim.api.nvim_buf_get_name(buf), ":p:h") + + local cfg = litgo().config.run + local cmd = { bin, "run", "--json" } + if cfg.vet then + cmd[#cmd + 1] = "--vet" + end + if cfg.timeout and cfg.timeout ~= "" then + vim.list_extend(cmd, { "--timeout", cfg.timeout }) + end + cmd[#cmd + 1] = vim.api.nvim_buf_get_name(buf) + if args and #args > 0 then + cmd[#cmd + 1] = "--" + vim.list_extend(cmd, args) + end + + local diags, pending = {}, "" + local function handle(ev) + if ev.event == "tangle" then + panel_add("⟫ tangled β†’ " .. ev.out, "LitgoInfo") + elseif ev.event == "diagnostic" then + diags[#diags + 1] = ev + local where = ev.file and (ev.file .. ":" .. (ev.line + 1) .. ":" .. (ev.col + 1) .. ": ") or "" + local hl = ev.severity == 1 and "LitgoError" or "DiagnosticWarn" + if require("litgo.proof").is_proof(ev) then + hl = ev.severity == 1 and "LitgoProofError" or "LitgoProofWarn" + end + panel_add(where .. ev.message, hl) + elseif ev.event == "prove" then + local all = ev.proved == ev.functions + panel_add(("∴ proved %d of %d annotated functions (%d obligations, %d ms)"):format(ev.proved, ev.functions, ev.obligations, ev.ms), all and "LitgoProofKeyword" or "LitgoProofError") + elseif ev.event == "build" then + if ev.ok then + panel_add(("βœ“ built in %d ms"):format(ev.ms), "LitgoOk") + else + local what = { tangle = "tangle failed", prove = "not proved, so not run" } + panel_add("βœ— " .. (what[ev.stage] or "build failed"), ev.stage == "prove" and "LitgoProofError" or "LitgoError") + end + elseif ev.event == "output" then + panel_add(ev.text, ev.stream == "stderr" and "DiagnosticWarn" or nil) + elseif ev.event == "exit" and ev.ran then + local note = ev.timed_out and " (timed out)" or "" + panel_add(("%s exit %d in %d ms%s"):format(ev.code == 0 and "βœ“" or "βœ—", ev.code, ev.ms, note), ev.code == 0 and "LitgoOk" or "LitgoError") + end + end + + local this + this = vim.system(cmd, { + text = true, + stdout = function(_, data) + if not data then + return + end + vim.schedule(function() + if job ~= this then + return -- a newer run replaced this one + end + pending = pending .. data + while true do + local nl = pending:find("\n", 1, true) + if not nl then + break + end + local line = pending:sub(1, nl - 1) + pending = pending:sub(nl + 1) + local ok, ev = pcall(vim.json.decode, line) + if ok and type(ev) == "table" then + handle(ev) + end + end + end) + end, + }, function() + vim.schedule(function() + if job ~= this then + return + end + job = nil + publish(diags, buf) + local jumped = false + if cfg.jump and vim.api.nvim_get_current_buf() == buf then + local name = vim.api.nvim_buf_get_name(buf) + for _, d in ipairs(diags) do + if d.severity == 1 and (not d.file or d.file == name) then + local row = math.min(d.line + 1, vim.api.nvim_buf_line_count(buf)) + vim.api.nvim_win_set_cursor(0, { row, d.col }) + vim.cmd("normal! zv") + jumped = true + break + end + end + end + if cfg.focus and not jumped and vim.api.nvim_get_current_buf() == buf then + panel_focus() + end + end) + end) + job = this +end + +return M ``` -## Setup +## Errors as you type - + + +The language server gets started for each `.lit.md` buffer, and buffers under +the same root share one server. Neovim shows what the server publishes like any +other diagnostics, except that the plugin asks for them to be shown in place, +as virtual lines under the code, the same way the diagnostics of a run are +shown. + +The server brings gopls with it, and the only thing it needs from the plugin is +help finding gopls. Most Neovim users get gopls from Mason, and Mason installs +it in a place where only Neovim looks. After that, everything Neovim does for a +buffer with a language server works here too: `K`, `grr`, `grn`, ``, +signature help on ``, and whatever a completion plugin or a distribution +adds. + +Completion is the exception, because nothing switches it on by itself. +nvim-cmp and blink.cmp find the server through the buffer it is attached to, so +they need nothing from the plugin; plain Neovim completes only when asked, on +``. So the plugin turns Neovim's own popup on when no completion +plugin is loaded, and stays out of the way when one is. A completion plugin is +usually loaded on the first insert, which is later than the server attaches, so +the question is asked on the first insert too β€” and then answered once. ```lua --- litgo.nvim: a tight loop for literate Go in .lit.md files. --- --- The plugin is deliberately thin. Everything that knows about the format --- lives in the `litgo` binary; Lua asks it questions (document on stdin, JSON --- back) and turns the answers into extmarks, edits and diagnostics. local M = {} -M.config = { - bin = nil, -- path to the litgo binary; found on $PATH when nil - render = { - enabled = true, - math = true, - mermaid = true, - tables = true, -- line up the columns of Markdown tables - chunks = true, -- show `// <>` as ⟨name⟩ and chunk directives as ⟨name⟩ ≑ - conceal_source = true, -- hide diagram/display-math source while the cursor is elsewhere - upright = false, -- true: keep math letters upright instead of π‘–π‘‘π‘Žπ‘™π‘–π‘ - indent = 4, - debounce_ms = 150, - }, - run = { - vet = false, -- also report `go vet` findings as warnings - timeout = "60s", -- kill runaway programs; "" to disable - jump = true, -- move the cursor to the first error - focus = true, -- put the cursor in the output panel when the run ends, so q closes it - on_save = false, -- :TangleCompileAndRun after every write (toggle with :LitWatch) - panel_height = 12, - }, - lsp = { - enabled = true, -- run `litgo lsp`: chunk, syntax and type errors as you type - virtual_lines = true, -- show them under the offending line rather than beside it - gopls = true, -- completion, hover, definitions, references, rename in Go blocks; or a path, or false - settings = nil, -- gopls settings, e.g. { buildFlags = { "-tags=integration" } } - completion = "auto", -- "auto": Neovim's own completion popup as you type, unless a - -- completion plugin is loaded. true forces it on, false off. - }, - weave = { open = true }, - untangle = { jump = false }, -- true: go to the new chunk to document it right away - keymaps = { - run = "r", - stop = "s", - tangle = "t", - weave = "w", - untangle = "u", -- visual mode - rename = "n", - toggle_render = "v", - goto_chunk = "gd", - hover = "K", - }, -} +--- The completion plugins that drive completion themselves. +local engines = { "blink.cmp", "cmp", "coq" } + +local function engine_loaded() + for _, name in ipairs(engines) do + if package.loaded[name] then + return true + end + end + return false +end + +--- Turn Neovim's own completion on for this buffer, unless something else +--- is doing the job. Deferred to the first insert, because that is when a +--- lazily loaded completion plugin arrives. +local function complete_with(id, buf) + vim.api.nvim_create_autocmd("InsertEnter", { + buffer = buf, + once = true, + callback = function() + vim.schedule(function() + if not engine_loaded() and vim.api.nvim_buf_is_valid(buf) then + vim.lsp.completion.enable(true, id, buf, { autotrigger = true }) + end + end) + end, + }) +end + +local function litgo() + return require("litgo") +end + +--- What to tell the server about gopls: where it is, to do without ("off"), +--- or to look for itself (""). +local function gopls(want) + if want == false then + return "off" + elseif type(want) == "string" then + return vim.fn.expand(want) + end + local mason = vim.fn.stdpath("data") .. "/mason/bin/gopls" + if vim.fn.executable("gopls") == 0 and vim.fn.executable(mason) == 1 then + return mason + end + return "" +end + +function M.start(buf) + local cfg = litgo().config.lsp + local bin = cfg.enabled and litgo().bin() + if not bin then + return + end + local name = vim.api.nvim_buf_get_name(buf) + vim.lsp.start({ + name = "litgo", + cmd = { bin, "lsp", "--gopls", gopls(cfg.gopls) }, + init_options = cfg.settings, + root_dir = vim.fs.root(name, { "go.mod", ".git" }) or vim.fs.dirname(name), + on_attach = function(client, bufnr) + vim.diagnostic.config({ + virtual_lines = cfg.virtual_lines, + virtual_text = not cfg.virtual_lines, + severity_sort = true, + [require("litgo.proof").handler] = true, + }, vim.lsp.diagnostic.get_namespace(client.id)) + if cfg.completion == "auto" then + complete_with(client.id, bufnr) + elseif cfg.completion then + vim.lsp.completion.enable(true, client.id, bufnr, { autotrigger = true }) + end + end, + }, { bufnr = buf }) +end + +local function client(buf) + return vim.lsp.get_clients({ bufnr = buf, name = "litgo" })[1] +end + +--- Is the server attached to this buffer? +function M.active(buf) + return client(buf) ~= nil +end + +--- Is the server reporting errors in this buffer right now? If it is, a +--- run's compile errors would only repeat them. +function M.reporting(buf) + local c = client(buf) + return c ~= nil + and #vim.diagnostic.get(buf, { + namespace = vim.lsp.diagnostic.get_namespace(c.id), + severity = vim.diagnostic.severity.ERROR, + }) + > 0 +end + +return M ``` -The plugin looks for the binary in its own checkout first. That is where a -plugin manager's build step puts it, and a binary from there is sure to match -the Lua. +## Proofs in their own colour + + + +Neovim colours a diagnostic by its severity and by nothing else, so a failed +proof would look like a compile error. It should not, because it is a +different kind of news and calls for a different kind of fix: not the code +the compiler choked on, but an invariant that says too little. + +Diagnostics are drawn by handlers, and handlers can be replaced. The ones that +draw (signs, underlines, virtual text, virtual lines) are wrapped so that in a +`.lit.md` buffer they never see a diagnostic whose source is `vego`, and one +more handler draws exactly those: a violet undercurl, a `∴` in the sign +column, and the message in violet under the line. They are still ordinary +diagnostics in every other way, so `]d`, the quickfix list and the location +list all know about them. ```lua -local function plugin_root() - local src = debug.getinfo(1, "S").source:sub(2) - return vim.fn.fnamemodify(src, ":p:h:h:h") +-- Diagnostics from the verifier, drawn apart from the compiler's. +local M = {} + +M.handler = "litgo/proof" + +local drawn = {} -- diagnostic namespace -> extmark namespace + +function M.is_proof(d) + return type(d.source) == "string" and d.source:find("vego", 1, true) ~= nil end -local warned = false +local function draw(buf, ns, d) + local line = vim.api.nvim_buf_get_lines(buf, d.lnum, d.lnum + 1, false)[1] or "" + local warn = d.severity ~= vim.diagnostic.severity.ERROR + local hl = warn and "LitgoProofWarn" or "LitgoProofError" + local pad = string.rep(" ", vim.fn.strdisplaywidth(line:match("^%s*"))) + local lines = {} + for i, text in ipairs(vim.split(d.message, "\n")) do + lines[i] = { { pad .. (i == 1 and "└─ ∴ " or " ") .. text, hl } } + end + local end_col = math.min(d.end_col or #line, #line) + pcall(vim.api.nvim_buf_set_extmark, buf, ns, d.lnum, math.min(d.col, #line), { + end_col = math.max(end_col, math.min(d.col, #line)), + hl_group = not warn and "LitgoProofUnderline" or nil, + sign_text = "∴", + sign_hl_group = hl, + virt_lines = lines, + priority = 220, + strict = false, + }) +end ---- Locate the litgo binary: explicit config, a build inside the plugin's own ---- checkout (what a plugin manager's build step produces), then $PATH. -function M.bin() - if M._bin then - return M._bin +--- Install the handler, and keep the built-in ones away from proofs. +function M.setup() + if vim.diagnostic.handlers[M.handler] then + return end - local candidates = { plugin_root() .. "/bin/litgo", "litgo" } - if M.config.bin then - table.insert(candidates, 1, vim.fn.expand(M.config.bin)) + local function is_lit(buf) + return require("litgo").is_lit(buf) end - for _, c in ipairs(candidates) do - if c and vim.fn.executable(c) == 1 then - M._bin = c - return c + vim.diagnostic.handlers[M.handler] = { + show = function(namespace, buf, diagnostics) + drawn[namespace] = drawn[namespace] or vim.api.nvim_create_namespace("litgo-proof-" .. namespace) + vim.api.nvim_buf_clear_namespace(buf, drawn[namespace], 0, -1) + if not is_lit(buf) then + return + end + for _, d in ipairs(diagnostics) do + if M.is_proof(d) then + draw(buf, drawn[namespace], d) + end + end + end, + hide = function(namespace, buf) + if drawn[namespace] and vim.api.nvim_buf_is_valid(buf) then + vim.api.nvim_buf_clear_namespace(buf, drawn[namespace], 0, -1) + end + end, + } + for _, name in ipairs({ "signs", "underline", "virtual_text", "virtual_lines" }) do + local builtin = vim.diagnostic.handlers[name] + if builtin then + vim.diagnostic.handlers[name] = { + show = function(namespace, buf, diagnostics, opts) + if is_lit(buf) then + diagnostics = vim.tbl_filter(function(d) + return not M.is_proof(d) + end, diagnostics) + end + builtin.show(namespace, buf, diagnostics, opts) + end, + hide = function(namespace, buf) + if builtin.hide then + builtin.hide(namespace, buf) + end + end, + } end end - if not warned then - warned = true - vim.notify( - "litgo: binary not found. Run `go build -o bin/litgo .` in " .. plugin_root() .. ", or set `bin` in setup().", - vim.log.levels.ERROR - ) - end end + +return M ``` -The plugin sees the protocol as two halves: ask a question about the buffer, -then apply the edits that come back. Several `nvim_buf_set_lines` calls made -from one command count as a single undo step. +## Chunks + + ```lua ---- Run litgo synchronously with the buffer on stdin; returns decoded JSON. -function M.query(buf, args) - local bin = M.bin() - if not bin then - return nil, "litgo binary not found" +-- Chunk editing and navigation: :Untangle, :RenameChunk, gd and K. +local M = {} + +local function litgo() + return require("litgo") +end + +local function refresh(buf) + require("litgo.render").refresh(buf) +end + +--- The chunk the cursor is on: a reference, a chunk directive, or the body of +--- a chunk block. Asks litgo afresh so it is right even mid-edit. +local function chunk_at_cursor(buf) + local analysis, err = litgo().query(buf, { "analyze", "--path", vim.api.nvim_buf_get_name(buf) }) + if not analysis then + return nil, err end - local cmd = vim.list_extend({ bin }, args) - local text = table.concat(vim.api.nvim_buf_get_lines(buf, 0, -1, false), "\n") .. "\n" - local res = vim.system(cmd, { stdin = text, text = true }):wait(5000) - local ok, decoded = pcall(vim.json.decode, res.stdout or "") - if not ok or type(decoded) ~= "table" then - return nil, (res.stderr ~= "" and res.stderr) or "litgo returned no JSON" + local pos = vim.api.nvim_win_get_cursor(0) + local row, col = pos[1] - 1, pos[2] + local on_line + for _, c in ipairs(analysis.chunks or {}) do + for _, r in ipairs(c.refs) do + if r.line == row then + if col >= r.col and col < r.end_col then + return c, "ref" + end + on_line = on_line or c + end + end end - if decoded.error then - return nil, decoded.error + if on_line then + return on_line, "ref" end - return decoded + for _, c in ipairs(analysis.chunks or {}) do + for _, d in ipairs(c.defs) do + if row == d.directive or (row >= d.first - 1 and row <= d.last + 1) then + return c, "def" + end + end + end + return nil, "the cursor is not on a chunk or a chunk reference" end ---- Apply bottom-up line edits as a single undo step. -function M.apply_edits(buf, edits) - for _, e in ipairs(edits) do - vim.api.nvim_buf_set_lines(buf, e.start, e["end"], false, e.lines) +--- :[range]Untangle [name] +--- A charwise visual selection that stops short of whole lines becomes an +--- inline reference (a block comment); anything else a whole-line one. +function M.untangle(buf, o) + local args = { + "untangle", + "--path", vim.api.nvim_buf_get_name(buf), + "--start", tostring(o.line1 - 1), + "--end", tostring(o.line2 - 1), + } + local s, e = vim.api.nvim_buf_get_mark(buf, "<"), vim.api.nvim_buf_get_mark(buf, ">") + local from_visual = o.range == 2 and s[1] == o.line1 and e[1] == o.line2 + if from_visual and vim.fn.visualmode() == "v" then + local last = vim.api.nvim_buf_get_lines(buf, e[1] - 1, e[1], false)[1] or "" + local end_col = math.min(e[2], math.max(#last - 1, 0)) + -- '> is the first byte of the last selected character; step past it. + end_col = end_col + #vim.fn.strcharpart(last:sub(end_col + 1), 0, 1) + if vim.o.selection == "exclusive" then + end_col = math.min(e[2], #last) + end + vim.list_extend(args, { "--start-col", tostring(s[2]), "--end-col", tostring(end_col) }) + end + if o.args and o.args ~= "" then + vim.list_extend(args, { "--name", o.args }) + end + + local res, err = litgo().query(buf, args) + if not res then + vim.notify("litgo: " .. err, vim.log.levels.ERROR) + return end + litgo().apply_edits(buf, res.edits) + local target = litgo().config.untangle.jump and res.def_line or res.ref_line + vim.api.nvim_win_set_cursor(0, { target + 1, 0 }) + vim.cmd("normal! ^") + refresh(buf) + vim.notify(("litgo: ⟨%s⟩ β†’ line %d (:RenameChunk to rename)"):format(res.name, res.def_line + 1)) end -function M.is_lit(buf) - return vim.api.nvim_buf_get_name(buf):match("%.lit%.md$") ~= nil +--- :RenameChunk [new name] β€” renames every definition and every reference. +function M.rename(buf, new_name) + local chunk, err = chunk_at_cursor(buf) + if not chunk then + vim.notify("litgo: " .. err, vim.log.levels.WARN) + return + end + local function apply(name) + if not name or vim.trim(name) == "" or name == chunk.name then + return + end + local res, rename_err = litgo().query(buf, { + "rename", "--path", vim.api.nvim_buf_get_name(buf), "--from", chunk.name, "--to", name, + }) + if not res then + vim.notify("litgo: " .. rename_err, vim.log.levels.ERROR) + return + end + litgo().apply_edits(buf, res.edits) + refresh(buf) + vim.notify(("litgo: ⟨%s⟩ β†’ ⟨%s⟩ (%d place%s)"):format(chunk.name, res.name, #res.edits, #res.edits == 1 and "" or "s")) + end + if new_name and new_name ~= "" then + apply(new_name) + else + vim.ui.input({ prompt = "Rename ⟨" .. chunk.name .. "⟩ to: ", default = chunk.name }, apply) + end end -local function define_highlights() - local links = { - LitgoMath = "Special", - LitgoDiagram = "Function", - LitgoTableRule = "Comment", - LitgoChunk = "Title", - LitgoFile = "Directory", - LitgoInfo = "Comment", - LitgoError = "DiagnosticError", - LitgoOk = "DiagnosticOk", - } - for name, target in pairs(links) do - vim.api.nvim_set_hl(0, name, { link = target, default = true }) +--- gd: from a reference to its definition; from a definition to its uses. +--- The language server does this better, because it does it for Go as well: +--- on a chunk it answers for itself, and anywhere else in a block, for gopls. +function M.goto_chunk() + local buf = vim.api.nvim_get_current_buf() + if require("litgo.lsp").active(buf) then + vim.lsp.buf.definition() + return + end + local chunk, where = chunk_at_cursor(buf) + if not chunk then + pcall(vim.cmd, "normal! gd") + return + end + vim.cmd("normal! m'") + if where == "ref" then + if #chunk.defs == 0 then + vim.notify("litgo: ⟨" .. chunk.name .. "⟩ is not defined", vim.log.levels.WARN) + return + end + vim.api.nvim_win_set_cursor(0, { chunk.defs[1].first + 1, 0 }) + elseif #chunk.refs == 1 then + vim.api.nvim_win_set_cursor(0, { chunk.refs[1].line + 1, chunk.refs[1].col }) + elseif #chunk.refs == 0 then + vim.notify("litgo: ⟨" .. chunk.name .. "⟩ is never used", vim.log.levels.WARN) + else + local items = {} + for _, r in ipairs(chunk.refs) do + items[#items + 1] = { + bufnr = buf, + lnum = r.line + 1, + col = r.col + 1, + text = vim.trim(vim.api.nvim_buf_get_lines(buf, r.line, r.line + 1, false)[1] or ""), + } + end + vim.fn.setloclist(0, {}, " ", { title = "uses of ⟨" .. chunk.name .. "⟩", items = items }) + vim.cmd("lopen") end + vim.cmd("normal! zvzz") end -``` - -A completion plugin offers the same sources everywhere in the buffer, which in -a Go block means Markdown snippets and words from the prose alongside what -gopls has to say. The plugin can't reach into someone else's configuration, so -it answers the question instead: is this line inside a Go block? Counting -fences from the top is enough, and cheap, because a completion menu only asks -about the line the cursor is on. -```lua ---- Is a line (0-based; the cursor's by default) inside a Go block? -function M.in_go_block(buf, line) - buf = buf or 0 - if not M.is_lit(buf) then - return false +--- K: preview the chunk behind a reference without leaving the host block. +--- This too is the language server's if it is there: a chunk on a reference, +--- gopls's documentation on a name. +function M.hover() + local buf = vim.api.nvim_get_current_buf() + if require("litgo.lsp").active(buf) then + vim.lsp.buf.hover() + return end - line = line or (vim.api.nvim_win_get_cursor(0)[1] - 1) - local lang, fence = nil, nil - for _, l in ipairs(vim.api.nvim_buf_get_lines(buf, 0, line, false)) do - local marks, rest = l:match("^%s*(``+`)%s*(.*)$") - if marks then - if not fence then - fence, lang = marks, rest:match("^[%w_+-]*") - elseif #marks >= #fence then - fence, lang = nil, nil - end - end + local chunk, where = chunk_at_cursor(buf) + if not chunk or where ~= "ref" or #chunk.defs == 0 then + pcall(vim.cmd, "normal! K") + return end - return lang == "go" + local lines = { "⟨" .. chunk.name .. "⟩ ≑", "```go" } + for _, d in ipairs(chunk.defs) do + vim.list_extend(lines, vim.api.nvim_buf_get_lines(buf, d.first, d.last + 1, false)) + end + lines[#lines + 1] = "```" + vim.lsp.util.open_floating_preview(lines, "markdown", { border = "rounded", focus_id = "litgo-chunk" }) end + +return M ``` -The filetype stays `markdown`, so treesitter highlighting and everything else -that works for Markdown keeps working. The commands are local to the buffer. +## Testing the plugin -```lua ---- Set up one .lit.md buffer: commands, keymaps, rendering. -function M.attach(buf) - if vim.b[buf].litgo_attached or not M.is_lit(buf) then - return - end - vim.b[buf].litgo_attached = true +Headless Neovim drives the real plugin against the real binary. The test covers +rendering, both kinds of untangling, renaming, a run that succeeds, a run with +an error inside a chunk, diagnostics that follow edits, a table whose columns +get lined up, and a proof: annotations in their own colour, an invariant that +is broken and reported in violet while the compiler stays quiet, a run that +refuses to go past it, and `:Prove`. - local run = require("litgo.run") - local chunks = require("litgo.chunks") - local render = require("litgo.render") +```sh +go build -o bin/litgo . && nvim --headless -u NONE -l test/plugin.lua +``` - local function command(name, fn, opts) - vim.api.nvim_buf_create_user_command(buf, name, fn, opts or {}) - end - command("Tangle", function(o) - run.tangle(buf, o.bang) - end, { bang = true, desc = "Write the Go source (! overwrites files litgo did not generate)" }) - command("Weave", function(o) - run.weave(buf, o.args) - end, { nargs = "?", desc = "Write and open the woven PDF (or :Weave html)" }) - command("TangleCompileAndRun", function(o) - run.run(buf, o.fargs) - end, { nargs = "*", desc = "Tangle, compile and run; errors appear inline" }) - command("LitStop", function() - run.stop() - end, { desc = "Kill the running program" }) - command("LitWatch", function() - M.config.run.on_save = not M.config.run.on_save - vim.notify("litgo: run on save " .. (M.config.run.on_save and "on" or "off")) - end, { desc = "Toggle :TangleCompileAndRun on every write" }) - command("LitClear", function() - run.clear(buf) - end, { desc = "Clear compile and run diagnostics" }) - command("Untangle", function(o) - chunks.untangle(buf, o) - end, { range = true, nargs = "*", desc = "Move the selection into a named chunk at the bottom" }) - command("RenameChunk", function(o) - chunks.rename(buf, o.args) - end, { nargs = "*", desc = "Rename the chunk under the cursor everywhere" }) - command("LitRender", function() - render.toggle(buf) - end, { desc = "Toggle math, diagram and chunk rendering" }) + + +```lua +-- Headless test of the plugin: nvim --headless -u NONE -l test/plugin.lua +local root = vim.fn.fnamemodify(debug.getinfo(1, "S").source:sub(2), ":p:h:h") +vim.opt.rtp:prepend(root) +local litgo = root .. "/bin/litgo" +assert(vim.fn.executable(litgo) == 1, "build bin/litgo first: go build -o bin/litgo .") +vim.o.swapfile = false - local keys = M.config.keymaps or {} - local function map(mode, lhs, rhs, desc) - if lhs and lhs ~= "" then - vim.keymap.set(mode, lhs, rhs, { buffer = buf, silent = true, desc = "litgo: " .. desc }) - end +local failures = 0 +local function check(cond, msg) + if cond then + print("ok " .. msg) + else + failures = failures + 1 + print("FAIL " .. msg) end - map("n", keys.run, "TangleCompileAndRun", "tangle, compile and run") - map("n", keys.stop, "LitStop", "stop the program") - map("n", keys.tangle, "Tangle", "tangle") - map("n", keys.weave, "Weave", "weave") - map("x", keys.untangle, ":Untangle", "untangle selection into a chunk") - map("n", keys.rename, "RenameChunk", "rename chunk") - map("n", keys.toggle_render, "LitRender", "toggle rendering") - map("n", keys.goto_chunk, chunks.goto_chunk, "go to chunk definition / uses") - map("n", keys.hover, chunks.hover, "preview chunk") - - vim.api.nvim_create_autocmd("BufWritePost", { - buffer = buf, - callback = function() - if M.config.run.on_save then - run.run(buf, {}) - end - end, - }) - - run.attach(buf) - render.attach(buf) - require("litgo.lsp").start(buf) end - -function M.setup(opts) - M.config = vim.tbl_deep_extend("force", M.config, opts or {}) - M._bin = nil - define_highlights() - local group = vim.api.nvim_create_augroup("litgo", { clear = true }) - vim.api.nvim_create_autocmd({ "BufReadPost", "BufNewFile", "BufWinEnter" }, { - group = group, - pattern = "*.lit.md", - callback = function(ev) - M.attach(ev.buf) - end, - }) - vim.api.nvim_create_autocmd("ColorScheme", { group = group, callback = define_highlights }) - for _, buf in ipairs(vim.api.nvim_list_bufs()) do - if vim.api.nvim_buf_is_loaded(buf) then - M.attach(buf) +local function wait(ms, cond, what) + if not vim.wait(ms, cond, 20) then + check(false, "timed out waiting for " .. what) + end +end +local function find(buf, pattern) + for i, l in ipairs(vim.api.nvim_buf_get_lines(buf, 0, -1, false)) do + if l:find(pattern, 1, true) then + return i end end end -return M -``` +local dir = vim.fn.tempname() +vim.fn.mkdir(dir, "p") +local file = dir .. "/pool.lit.md" +vim.fn.writefile(vim.fn.readfile(root .. "/examples/worker_pool.lit.md"), file) -## Rendering +require("litgo").setup({ bin = litgo, weave = { open = false }, run = { jump = true } }) +vim.cmd.edit(file) +local buf = vim.api.nvim_get_current_buf() +local render = require("litgo.render") +local run = require("litgo.run") - +-- Rendering ------------------------------------------------------------------ +wait(3000, function() + local st = render.get(buf) + return st and st.items ~= nil +end, "analysis") +local ns = vim.api.nvim_get_namespaces()["litgo-render"] +local marks = vim.api.nvim_buf_get_extmarks(buf, ns, 0, -1, { details = true }) +local inline, blocks, concealed = 0, 0, 0 +for _, m in ipairs(marks) do + local d = m[4] + if d.virt_text_pos == "inline" then inline = inline + 1 end + if d.virt_lines then blocks = blocks + 1 end + if d.conceal_lines then concealed = concealed + 1 end +end +check(inline >= 12, "inline math and chunk names are rendered (" .. inline .. ")") +check(blocks == 3, "two diagrams and one display formula are drawn (" .. blocks .. ")") +check(concealed == 3, "their source is concealed while the cursor is elsewhere") +check(#vim.diagnostic.get(buf) == 0, "a correct document has no diagnostics") -```lua --- In-buffer rendering: LaTeX as Unicode, Mermaid as box drawings, and chunk --- syntax as ⟨names⟩. Whatever the cursor touches falls back to its source, --- with block renderings kept underneath as a live preview. -local M = {} +-- Moving onto a diagram reveals its source. +vim.api.nvim_win_set_cursor(0, { find(buf, "```mermaid") + 1, 0 }) +vim.cmd("doautocmd CursorMoved") +concealed = 0 +for _, m in ipairs(vim.api.nvim_buf_get_extmarks(buf, ns, 0, -1, { details = true })) do + if m[4].conceal_lines then concealed = concealed + 1 end +end +check(concealed == 2, "the block under the cursor shows its source") -local ns = vim.api.nvim_create_namespace("litgo-render") +-- :Untangle, linewise ---------------------------------------------------------- +local before = vim.fn.system({ litgo, "tangle", "--stdout", file }) +local l1 = find(buf, "for job := range jobs") +vim.cmd(("%d,%dUntangle"):format(l1, l1 + 9)) +check(vim.api.nvim_buf_get_lines(buf, l1 - 1, l1, false)[1] == "\t// <>", "the selection became a reference") +check(find(buf, "") ~= nil, "the chunk landed at the bottom") +vim.cmd("silent write") +check(vim.fn.system({ litgo, "tangle", "--stdout", file }) == before, "untangling leaves the program unchanged") -local check_ns = vim.api.nvim_create_namespace("litgo-check") +-- :Untangle, charwise (an expression) ------------------------------------------ +local l2 = find(buf, "jobs := make(chan int, m)") +local line = vim.api.nvim_buf_get_lines(buf, l2 - 1, l2, false)[1] +local s = line:find("make", 1, true) +vim.api.nvim_win_set_cursor(0, { l2, s - 1 }) +vim.cmd("normal! v$h") +vim.cmd("normal! \27") +vim.cmd("'<,'>Untangle") +check(vim.api.nvim_buf_get_lines(buf, l2 - 1, l2, false)[1] == "\tjobs := /*<>*/", "a charwise selection became an inline reference") +vim.cmd("silent write") +check(vim.fn.system({ litgo, "tangle", "--stdout", file }) == before, "inline untangling leaves the program unchanged") -M.check_ns = check_ns +-- :RenameChunk ------------------------------------------------------------------- +vim.api.nvim_win_set_cursor(0, { l2, s + 5 }) +vim.cmd("let &undolevels = &undolevels") -- scripts never pause for input, so close the undo block by hand +vim.cmd("RenameChunk the jobs channel") +check(find(buf, "/*<>*/") ~= nil and find(buf, "") ~= nil, "rename updates reference and definition") +check(find(buf, "make call") == nil, "no trace of the old name") +vim.cmd("normal u") +check(find(buf, "/*<>*/") ~= nil and find(buf, "") ~= nil, "rename is a single undo step") +vim.cmd("normal! \18") -- redo + +-- gd --------------------------------------------------------------------------- +wait(30000, function() return require("litgo.lsp").active(buf) end, "the language server") +vim.api.nvim_win_set_cursor(0, { find(buf, "// <>"), 3 }) +require("litgo.chunks").goto_chunk() +wait(10000, function() return vim.api.nvim_get_current_line() == "for i := 0; i < m; i++ {" end, "gd") +check(vim.api.nvim_get_current_line() == "for i := 0; i < m; i++ {", "gd jumps from a reference to the chunk") + +-- Go in a Markdown buffer: gopls, behind the language server -------------------- +local server = vim.lsp.get_clients({ bufnr = buf, name = "litgo" })[1] +if server and server.server_capabilities.completionProvider then + local function ask(method, row, col) + local params = { textDocument = vim.lsp.util.make_text_document_params(buf), position = { line = row - 1, character = col } } + local answer = server:request_sync("textDocument/" .. method, params, 30000, buf) + return answer and answer.result + end + -- wg is declared in main and used in a chunk: only the tangled program knows what it is. + local l5 = find(buf, "wg.Add(1)") + local hover + wait(60000, function() + hover = ask("hover", l5, 1) + return hover ~= nil + end, "gopls to load the program") + check(hover and hover.contents.value:find("var wg sync.WaitGroup", 1, true) ~= nil, "hover in a chunk knows a variable of the block it is spliced into") + check(hover and hover.range.start.line == l5 - 1 and hover.range.start.character == 1, "and answers about the Markdown") + check(ask("hover", find(buf, "## The worker"), 4) == nil, "prose is not Go") --- bufnr -> { items, chunks, enabled, dirty, timer, cursor_row } -local state = {} + vim.api.nvim_buf_set_lines(buf, l5, l5, false, { "\twg." }) + local items = ask("completion", l5 + 1, 4) + local labels = {} + for _, item in ipairs(items and (items.items or items) or {}) do + labels[item.label] = item + end + check(labels.Wait ~= nil and labels.Done ~= nil, "completion in a chunk offers the methods of wg") + local edit = labels.Wait and labels.Wait.textEdit or {} + check((edit.insert or edit.range or { start = {} }).start.line == l5, "with edits that are edits of the Markdown") + vim.api.nvim_buf_set_lines(buf, l5, l5 + 1, false, {}) -local function litgo() - return require("litgo") + local l6 = find(buf, "go worker(ctx") + vim.api.nvim_win_set_cursor(0, { l6, 4 }) + require("litgo.chunks").goto_chunk() + wait(10000, function() return vim.api.nvim_get_current_line():find("^func worker%(") ~= nil end, "gd on a Go name") + check(vim.api.nvim_get_current_line():find("^func worker%(") ~= nil, "gd on a Go name jumps to its definition, in another chunk") +else + print("skip gopls is not installed") end -function M.get(buf) - return state[buf] +-- :TangleCompileAndRun, success ------------------------------------------------ +vim.cmd("silent write") +vim.cmd("TangleCompileAndRun") +local function panel_text() + local pb = vim.fn.bufnr("litgo://output") + return pb > 0 and table.concat(vim.api.nvim_buf_get_lines(pb, 0, -1, false), "\n") or "" end +wait(60000, function() return panel_text():find("exit %d+ in") ~= nil end, "the program to finish") +check(panel_text():find("jobs finished") ~= nil, "program output reaches the panel") +check(panel_text():find("βœ“ built") ~= nil, "the build is reported") +check(#vim.diagnostic.get(buf, { namespace = run.ns }) == 0, "a clean run leaves no diagnostics") -local function virt_lines(lines, hl, indent) - local pad = string.rep(" ", indent) - local out = {} - for _, l in ipairs(lines) do - out[#out + 1] = { { pad .. l, hl } } +-- The panel has the cursor when the run is over, and q gives it back. +local function panel_win() + local pb = vim.fn.bufnr("litgo://output") + for _, w in ipairs(vim.api.nvim_list_wins()) do + if vim.api.nvim_win_get_buf(w) == pb then + return w + end end - return out -end - -local function mark(buf, row, col, opts) - opts.strict = false - pcall(vim.api.nvim_buf_set_extmark, buf, ns, row, col, opts) end +wait(10000, function() return vim.api.nvim_get_current_win() == panel_win() end, "the panel to take the cursor") +check(vim.api.nvim_get_current_win() == panel_win(), "the output panel takes the cursor when the run ends") +vim.api.nvim_feedkeys("q", "x", false) +check(panel_win() == nil, "q closes it") +check(vim.api.nvim_get_current_buf() == buf, "and the cursor is back in the document") -local function wanted(item, cfg) - if item.kind == "math" or item.kind == "math-block" then - return cfg.math - elseif item.kind == "mermaid" then - return cfg.mermaid - elseif item.kind == "table" then - return cfg.tables +-- Errors as you type: the language server, nothing run ------------------------- +local function diag(pattern, ns) + for _, x in ipairs(vim.diagnostic.get(buf, { namespace = ns })) do + if x.message:find(pattern, 1, true) then + return x + end end - return cfg.chunks end -``` - -Everything is drawn with extmarks, and whatever the cursor is on doesn't get -drawn, so the source is always one keystroke away from its rendering. +check(require("litgo.lsp").active(buf), "the language server is attached") -```lua ---- Redraw every extmark. Cheap enough (one pass over the items) to run on ---- each cursor-line change, which keeps the reveal logic trivial. -function M.draw(buf) - local st = state[buf] - if not st or not vim.api.nvim_buf_is_valid(buf) then - return - end - vim.api.nvim_buf_clear_namespace(buf, ns, 0, -1) - if not st.enabled or not st.items then - return - end - local cfg = litgo().config.render - local cursor = -1 - if vim.api.nvim_get_current_buf() == buf then - cursor = vim.api.nvim_win_get_cursor(0)[1] - 1 - end - st.cursor_row = cursor - local last = vim.api.nvim_buf_line_count(buf) - 1 +-- What a completion plugin needs to keep Markdown snippets out of Go blocks. +local blocks = require("litgo") +check(blocks.in_go_block(buf, find(buf, "fmt.Println(r)") - 1), "a line of Go is in a Go block") +check(not blocks.in_go_block(buf, find(buf, "# Worker pool") - 1), "a heading is not") +check(not blocks.in_go_block(buf, find(buf, "flowchart LR") - 1), "and neither is a diagram") - for _, it in ipairs(st.items) do - if wanted(it, cfg) then - local under_cursor = cursor >= it.line and cursor <= it.end_line - if it.lines then - -- <> - elseif it.table then - -- <> - else - -- <> - end - end - end -end -``` +local l3 = find(buf, "fmt.Println(r)") +vim.api.nvim_buf_set_lines(buf, l3 - 1, l3, false, { "\tfmt.Println(rr)" }) +wait(60000, function() return diag("undefined: rr") ~= nil end, "the language server to find the type error") +local d = diag("undefined: rr") +check(d and d.lnum == l3 - 1 and d.col == 13 and d.end_col == 15, "a type error inside a chunk is underlined where it is, unsaved and unrun") +local first = diag("declared and not used: r") +check(first ~= nil, "and so is the variable that is now unused") -A block rendering replaces its source. The source lines get concealed, and the -drawing hangs off a neighbouring line as virtual lines. While the cursor is -inside the block, the source shows and the drawing stays underneath as a live -preview. +-- A run adds nothing the server has said, but still jumps and fills quickfix. +vim.api.nvim_win_set_cursor(0, { 1, 0 }) +vim.cmd("TangleCompileAndRun") +wait(60000, function() return panel_text():find("build failed") ~= nil end, "the build to fail") +wait(2000, function() return #vim.fn.getqflist() == 2 end, "the quickfix list") +check(vim.api.nvim_get_current_win() ~= panel_win(), "a run that fails leaves the cursor in the document") +check(first and vim.api.nvim_win_get_cursor(0)[1] == first.lnum + 1, "the cursor jumped to the first error") +check(#vim.fn.getqflist() == 2, "the quickfix list mirrors the compiler") +check(diag("undefined: rr", run.ns) == nil, "the run does not repeat what the server reports") - -```lua -local hl = it.kind == "mermaid" and "LitgoDiagram" or "LitgoMath" -if it.error then - hl = "LitgoError" -end -local vl = virt_lines(it.lines, hl, cfg.indent) -if under_cursor or not cfg.conceal_source then - -- Editing: keep the source visible and preview underneath. Anchor to - -- the line after the block: treesitter conceals fence lines, and - -- virtual lines hanging off a concealed line disappear with it. - if it.end_line < last then - mark(buf, it.end_line + 1, 0, { virt_lines = vl, virt_lines_above = true }) - else - mark(buf, it.end_line, 0, { virt_lines = vl }) - end -elseif it.end_line < last then - mark(buf, it.line, 0, { end_row = it.end_line, conceal_lines = "" }) - mark(buf, it.end_line + 1, 0, { virt_lines = vl, virt_lines_above = true }) -elseif it.line > 0 then - mark(buf, it.line, 0, { end_row = it.end_line, conceal_lines = "" }) - mark(buf, it.line - 1, 0, { virt_lines = vl }) -else - mark(buf, it.end_line, 0, { virt_lines = vl }) -end -``` +vim.api.nvim_buf_set_lines(buf, l3 - 1, l3, false, { "\tfmt.Println(r)" }) +wait(20000, function() return diag("undefined: rr") == nil and diag("declared and not used") == nil end, "the errors to clear") +check(#vim.diagnostic.get(buf) == 0, "fixing the line clears the errors") -A table stays where it is, and its lines stay lines of the buffer. The only -things added are spaces, as inline virtual text, so that the pipes of every row -end up under the pipes of the header. A column is as wide as its widest cell -looks. A cell that looks narrower gets the difference as spaces: after its text -if the column is aligned left, before its text if it is aligned right, and half -on each side if it is centred. The rule under the header is the one thing that -gets replaced, by dashes that are as wide as the column. The row the cursor is -on is left exactly as it was typed, like everything else the cursor is on. +-- Diagnostics from a run (a panic, a failing test) stay accurate between runs. +vim.diagnostic.set(run.ns, buf, { + { lnum = l3 - 1, col = 0, message = "panic: one", severity = 1 }, + { lnum = l3 + 1, col = 0, message = "panic: two", severity = 1 }, +}) +vim.api.nvim_buf_set_lines(buf, l3 - 1, l3, false, { "\tfmt.Println(r) " }) +vim.api.nvim_buf_set_lines(buf, 0, 0, false, { "", "" }) +wait(2000, function() local x = diag("panic: two", run.ns) return x and x.lnum == l3 + 3 end, "run diagnostics to move") +check(diag("panic: one", run.ns) == nil, "editing a line dismisses the run diagnostic on it") +check(diag("panic: two", run.ns).lnum == l3 + 3, "and the ones below follow their lines") +vim.api.nvim_buf_set_lines(buf, 0, 2, false, {}) +run.clear(buf) -The widths get measured with `strdisplaywidth`, because that is the editor's -own answer to how many screen cells a string takes up. They are kept with the -item, so moving the cursor doesn't measure every table again. +-- Structure is the server's business too. +local l4 = find(buf, "// <>") +vim.api.nvim_buf_set_lines(buf, l4 - 1, l4, false, { "\t// <>" }) +wait(20000, function() return diag("is not defined") ~= nil and diag("is never used") ~= nil end, "the structural diagnostics") +check(diag("spawn teh workers⟩ is not defined") and diag("spawn the workers⟩ is never used"), "a misspelt reference is flagged at both ends") +vim.cmd("normal u") - -```lua -local t = it.table -local function looks(cell) - return (cell.col - cell.from) + vim.fn.strdisplaywidth(cell.text) + (cell.to - cell.end_col) -end -if not it.widths then - it.widths = {} - for _, row in ipairs(t.rows) do - for c, cell in ipairs(row.cells) do - it.widths[c] = math.max(it.widths[c] or 3, looks(cell)) - end - end -end -local function spaces(row, col, n) - if n > 0 then - mark(buf, row, col, { virt_text = { { string.rep(" ", n) } }, virt_text_pos = "inline" }) +-- :Weave ------------------------------------------------------------------------ +if vim.fn.executable("pandoc") == 1 then + vim.cmd("silent write") + vim.cmd("Weave html") + wait(20000, function() return vim.uv.fs_stat(dir .. "/pool.html") ~= nil end, "the woven page") + check(vim.uv.fs_stat(dir .. "/pool.html") ~= nil, ":Weave html writes pool.html") + if vim.fn.executable("typst") == 1 and vim.system({ "typst", "--version" }):wait().code == 0 then + vim.cmd("Weave") + wait(30000, function() return vim.uv.fs_stat(dir .. "/pool.pdf") ~= nil end, "the woven PDF") + check(vim.uv.fs_stat(dir .. "/pool.pdf") ~= nil, ":Weave writes pool.pdf") end end -for _, row in ipairs(t.rows) do - if row.line ~= cursor then - for _, h in ipairs(row.hide or {}) do - mark(buf, row.line, h[1], { end_col = h[2], conceal = "" }) - end - for c, cell in ipairs(row.cells) do - local gap = (it.widths[c] or 0) - looks(cell) - local before = ({ right = gap, center = math.floor(gap / 2) })[t.align[c]] or 0 - spaces(row.line, cell.from, before) - spaces(row.line, cell.to, gap - before) + +-- Tables --------------------------------------------------------------------------- +vim.cmd("enew!") +local tbuf = vim.api.nvim_get_current_buf() +vim.api.nvim_buf_set_name(tbuf, dir .. "/table.lit.md") +vim.api.nvim_buf_set_lines(tbuf, 0, -1, false, { + "# Keys", + "", + "| Key | What it does | Cost |", + "|:-:|---|--:|", + "| `K` | shows the *documentation* | $x^2$ |", + "| `` | completes | 10 |", + "", + "The end.", +}) +require("litgo").attach(tbuf) +wait(3000, function() + local st = render.get(tbuf) + return st and st.items ~= nil +end, "the analysis of the table") + +-- What a row looks like: the text that is not concealed, and the virtual +-- text where it is put in. +local function shown(row) + local text = vim.api.nvim_buf_get_lines(tbuf, row, row + 1, false)[1] + local hidden, put = {}, {} + for _, m in ipairs(vim.api.nvim_buf_get_extmarks(tbuf, ns, { row, 0 }, { row, -1 }, { details = true })) do + local d = m[4] + if d.conceal then + for i = m[3] + 1, d.end_col do hidden[i] = true end + elseif d.virt_text_pos == "inline" then + put[m[3]] = (put[m[3]] or "") .. d.virt_text[1][1] end end -end -if it.line + 1 ~= cursor then - for c, cell in ipairs(t.rule) do - local w, a = it.widths[c] or 3, t.align[c] - local dashes = (a == "center" and ":" or "-") .. string.rep("-", w - 2) .. (a == "left" and "-" or ":") - mark(buf, it.line + 1, cell.from, { end_col = cell.to, conceal = "" }) - mark(buf, it.line + 1, cell.from, { virt_text = { { dashes, "LitgoTableRule" } }, virt_text_pos = "inline" }) + local out = "" + for i = 1, #text + 1 do + out = out .. (put[i - 1] or "") + if i <= #text and not hidden[i] then out = out .. text:sub(i, i) end end + return out end -``` +check(vim.api.nvim_buf_line_count(tbuf) == 8 and #vim.api.nvim_buf_get_extmarks(tbuf, ns, 0, -1, { details = true }) > 0, + "a table stays where it is") +check(shown(2) == "| Key | What it does | Cost |", "the header is padded, and its first column is centred: " .. shown(2)) +check(shown(3) == "|:----------:|-------------------------|-----:|", "the rule is as wide as the columns: " .. shown(3)) +check(shown(4) == "| K | shows the documentation | π‘₯Β² |", "cells are as wide as they look, not as their source: " .. shown(4)) +check(shown(5) == "| | completes | 10 |", "the widest cell decides: " .. shown(5)) - -```lua -local hl = ({ math = "LitgoMath", file = "LitgoFile" })[it.kind] or "LitgoChunk" -if it.info == "undefined" then - hl = "LitgoError" +-- The same thing as the screen shows it, which is what counts. +vim.cmd("redraw") +local screen = {} +for r = 3, 6 do + local s = "" + for c = 1, 60 do s = s .. vim.fn.screenstring(r, c) end + screen[#screen + 1] = vim.trim(s) end -if not under_cursor and it.text then - mark(buf, it.line, it.col, { end_col = it.end_col, conceal = "" }) - mark(buf, it.line, it.col, { virt_text = { { it.text, hl } }, virt_text_pos = "inline" }) +check(screen[3] == shown(4) and screen[4] == shown(5), "and that is what is on the screen: " .. screen[3]) +vim.api.nvim_win_set_cursor(0, { 5, 0 }) +vim.cmd("doautocmd CursorMoved") +check(shown(4) == "| `K` | shows the *documentation* | $x^2$ |", "the row under the cursor is left as it was typed") +check(shown(5):find("completes |", 1, true) ~= nil, "and the other rows stay lined up") +vim.cmd("bwipeout!") + +vim.fn.delete(dir, "rf") +-- Proofs --------------------------------------------------------------------------- +local pdir = vim.fn.tempname() +vim.fn.mkdir(pdir, "p") +local pfile = pdir .. "/isqrt.lit.md" +vim.fn.writefile(vim.fn.readfile(root .. "/examples/isqrt.lit.md"), pfile) +vim.cmd.edit(pfile) +local pbuf = vim.api.nvim_get_current_buf() +wait(3000, function() + local st = render.get(pbuf) + return st and st.items ~= nil +end, "analysis of the proved example") +local keywords, formulas = 0, 0 +for _, m in ipairs(vim.api.nvim_buf_get_extmarks(pbuf, ns, 0, -1, { details = true })) do + if m[4].hl_group == "LitgoProofKeyword" then keywords = keywords + 1 end + if m[4].hl_group == "LitgoProof" then formulas = formulas + 1 end end -if it.info and it.info ~= "" then - mark(buf, it.line, 0, { virt_text = { { " " .. it.info, "LitgoInfo" } }, virt_text_pos = "eol" }) +check(keywords == 5 and formulas == 5, "the five annotations have a colour of their own (" .. keywords .. ")") +local inv = find(pbuf, "//@ Invariant lo*lo") +local km = vim.api.nvim_buf_get_extmarks(pbuf, ns, { inv - 1, 0 }, { inv - 1, -1 }, { details = true })[1] +check(km and km[3] == 0 and km[4].end_col == #"//@ Invariant", "the keyword is set apart from the formula") + +-- The formula is typeset like the math in the prose, except under the cursor. +local function typeset(row) + local text, hidden + for _, m in ipairs(vim.api.nvim_buf_get_extmarks(pbuf, ns, { row, 0 }, { row, -1 }, { details = true })) do + if m[4].virt_text_pos == "inline" then text = m[4].virt_text[1][1] end + if m[4].conceal then hidden = { m[3], m[4].end_col } end + end + return text, hidden end -``` - -Analysis is asynchronous and debounced. An answer only gets used if the buffer -hasn't changed since the question was asked. +vim.api.nvim_win_set_cursor(0, { 1, 0 }) +render.draw(pbuf) +local text, hidden = typeset(inv - 1) +check(text == "π‘™π‘œΒ² ≀ 𝑛 ∧ 𝑛 < β„Žπ‘–Β²", "a formula is typeset: " .. tostring(text)) +check(hidden and hidden[1] == #"//@ Invariant " and hidden[2] == #"//@ Invariant lo*lo <= n ^ n < hi*hi", "in place of its source, and the keyword stays") +vim.api.nvim_win_set_cursor(0, { inv, 0 }) +render.draw(pbuf) +check(typeset(inv - 1) == nil, "and the source comes back under the cursor") +vim.api.nvim_win_set_cursor(0, { 1, 0 }) -```lua -local function to_diagnostics(list, buf) - local name = vim.api.nvim_buf_get_name(buf) - local count = vim.api.nvim_buf_line_count(buf) +local function proof_marks() local out = {} - for _, d in ipairs(list or {}) do - if not d.file or d.file == "" or d.file == name then - out[#out + 1] = { - lnum = math.min(d.line, count - 1), - col = d.col or 0, - end_col = (d.end_col and d.end_col > 0) and d.end_col or nil, - severity = d.severity, - message = d.message, - source = d.source, - } + for name, id in pairs(vim.api.nvim_get_namespaces()) do + if name:find("litgo-proof-", 1, true) then + vim.list_extend(out, vim.api.nvim_buf_get_extmarks(pbuf, id, 0, -1, { details = true })) end end return out end - -M.to_diagnostics = to_diagnostics - ---- Ask litgo about the buffer and redraw when the answer is still current. -function M.refresh(buf) - local st = state[buf] - local bin = litgo().bin() - if not st or not bin or not vim.api.nvim_buf_is_valid(buf) then - return - end - local tick = vim.api.nvim_buf_get_changedtick(buf) - local cmd = { bin, "analyze", "--path", vim.api.nvim_buf_get_name(buf) } - if litgo().config.render.upright then - cmd[#cmd + 1] = "--upright" - end - local text = table.concat(vim.api.nvim_buf_get_lines(buf, 0, -1, false), "\n") .. "\n" - vim.system(cmd, { stdin = text, text = true }, function(res) - vim.schedule(function() - if not vim.api.nvim_buf_is_valid(buf) then - return - end - if vim.api.nvim_buf_get_changedtick(buf) ~= tick then - -- The buffer moved on. Usually a debounced request is already queued; - -- if not (the tick also bumps while a file finishes loading), ask again. - if not st.dirty then - M.refresh(buf) - end - return - end - local ok, decoded = pcall(vim.json.decode, res.stdout or "") - if not ok or type(decoded) ~= "table" then - return - end - st.items, st.chunks, st.dirty = decoded.items, decoded.chunks, false - M.draw(buf) - -- Reporting problems is the language server's job. Without it, the - -- ones this analysis found are better than none; syntax errors while - -- typing are noise, so they wait until insert mode ends. - if require("litgo.lsp").active(buf) then - vim.diagnostic.reset(check_ns, buf) - elseif vim.api.nvim_get_mode().mode:sub(1, 1) ~= "i" then - vim.diagnostic.set(check_ns, buf, to_diagnostics(decoded.diagnostics, buf)) - end - end) - end) +local function proofs() + return vim.tbl_filter(require("litgo.proof").is_proof, vim.diagnostic.get(pbuf)) end - -function M.schedule(buf) - local st = state[buf] - if not st then - return - end - st.dirty = true - if st.timer then - st.timer:stop() - else - st.timer = vim.uv.new_timer() +wait(10000, function() return require("litgo.lsp").active(pbuf) end, "the server on the proved example") +check(#proofs() == 0 and #proof_marks() == 0, "a proved document has nothing violet under it") + +-- Break the bisection. It still compiles, so only the verifier can object. +local lo = find(pbuf, "\tlo = mid") +vim.api.nvim_buf_set_lines(pbuf, lo - 1, lo, false, { "\tlo = mid + 1" }) +wait(15000, function() return #proofs() == 2 end, "the verifier's objections") +local failed = proofs() +check(#failed == 2 and failed[1].lnum == inv - 2 and failed[1].message:find("Invariant is not maintained", 1, true) ~= nil, + "a broken invariant is reported at the invariant, as you type") +check(#vim.diagnostic.get(pbuf) == 2, "and the compiler has nothing to add") +local violet = proof_marks() +check(#violet == 2 and violet[1][4].hl_group == "LitgoProofUnderline" and violet[1][4].virt_lines[1][1][2] == "LitgoProofError" + and vim.trim(violet[1][4].sign_text) == "∴", "it is drawn in the proof colour, not the compiler's") +-- Neovim keeps marks of its own to follow a diagnostic through edits; what +-- counts is a mark that draws something. +local builtin = 0 +for name, id in pairs(vim.api.nvim_get_namespaces()) do + if not name:find("litgo-", 1, true) then + for _, m in ipairs(vim.api.nvim_buf_get_extmarks(pbuf, id, 0, -1, { details = true })) do + local d = m[4] + if d.hl_group or d.virt_text or d.virt_lines or d.sign_text then + builtin = builtin + 1 + end + end end - st.timer:start(litgo().config.render.debounce_ms, 0, function() - vim.schedule(function() - M.refresh(buf) - end) - end) end +check(builtin == 0, "and the built-in handlers leave it alone") + +-- A run refuses to go further than the proof. +vim.cmd("TangleCompileAndRun") +wait(20000, function() return panel_text():find("not proved, so not run", 1, true) ~= nil end, "the run that is not proved") +check(panel_text():find("proved 0 of 1", 1, true) ~= nil, "a run stops at a failed proof") +check(panel_text():find("isqrt(", 1, true) == nil, "and the program was not run") + +vim.api.nvim_set_current_buf(pbuf) +vim.api.nvim_buf_set_lines(pbuf, lo - 1, lo, false, { "\tlo = mid" }) +wait(15000, function() return #proofs() == 0 and #proof_marks() == 0 end, "the proof to come back") +check(#proofs() == 0, "fixing the code takes the violet away") +vim.cmd("Prove") +wait(10000, function() return vim.g.litgo_last_proof ~= nil end, ":Prove") +check(vim.g.litgo_last_proof == "1/1", ":Prove proves it without running it") + +print(failures == 0 and "\nall passed" or ("\n" .. failures .. " failed")) +os.exit(failures == 0 and 0 or 1) +``` + +# An example + +This is a complete literate program. The tests above use it, and it is a good +first file to open in the editor. It is a document inside a document, so its +fence is one backtick longer than the fences it contains. It is also +`verbatim`, because its references belong to it and not to this document. + + + +````markdown + + + + + +# Worker pool in Go + +A pool of $n$ workers processes $m$ jobs concurrently. The jobs travel through +a channel, and the context is cancellable, so we can stop early and still keep +the partial results. + +If every job costs $t$ seconds, the pool finishes in about + +$$ +T(n, m) = \left\lceil \frac{m}{n} \right\rceil \cdot t +$$ + +instead of the $m \cdot t$ a single goroutine would need: a speed-up of +$\frac{m t}{T} \approx n$ for as long as $n \le m$. + +```mermaid +flowchart LR + main -->|jobs| w1[worker 1] & w2[worker 2] & w3[worker 3] + w1 & w2 & w3 -->|results| collect +``` + +## The shape of the program + +The whole program fits on one screen, because each step is a named chunk that +is explained further down. Chunks are spliced in *textually*, so they see +`ctx`, `jobs`, `results` and `wg` exactly as if they had been written here. + +```go +// <> + +func main() { + // <> + + n, m := 3, 10 + jobs := make(chan int, m) + results := make(chan int, m) + var wg sync.WaitGroup + + // <> + // <> + // <> + // <> +} +``` + +## The worker + +A worker drains `jobs` until the channel closes or the context is cancelled, +whichever comes first. Doubling stands in for real work. + + +```go +func worker(ctx context.Context, id int, jobs <-chan int, results chan<- int, wg *sync.WaitGroup) { + defer wg.Done() + for job := range jobs { + select { + case <-ctx.Done(): + return + default: + // NOTE: this line is dropped by the tangler-exclude directive. + time.Sleep(20 * time.Millisecond) + results <- job * 2 + } + } +} +``` + +```mermaid +sequenceDiagram + participant M as main + participant W as worker + M->>W: job + W-->>M: job * 2 + M-xW: cancel +``` + +## Cancellation + +The deadline is a tiny chunk of its own: an expression, interpolated inline. + + +```go +ctx, cancel := context.WithCancel(context.Background()) +defer cancel() +time.AfterFunc(/*<>*/, cancel) +``` + +Long enough for roughly half of the jobs. + + +```go +50 * time.Millisecond +``` -function M.toggle(buf) - local st = state[buf] - if st then - st.enabled = not st.enabled - M.draw(buf) - end -end +## Spawn, feed, collect -local function window_options(buf) - for _, win in ipairs(vim.fn.win_findbuf(buf)) do - vim.wo[win][0].conceallevel = 2 - vim.wo[win][0].concealcursor = "" - end -end + +```go +for i := 0; i < n; i++ { + wg.Add(1) + go worker(ctx, i, jobs, results, &wg) +} +``` -function M.attach(buf) - state[buf] = { enabled = litgo().config.render.enabled } - local group = vim.api.nvim_create_augroup("litgo-render-" .. buf, { clear = true }) - vim.api.nvim_create_autocmd({ "TextChanged", "TextChangedI", "TextChangedP" }, { - group = group, - buffer = buf, - callback = function() - M.schedule(buf) - end, - }) - vim.api.nvim_create_autocmd("InsertLeave", { - group = group, - buffer = buf, - callback = function() - M.refresh(buf) - end, - }) - vim.api.nvim_create_autocmd({ "CursorMoved", "CursorMovedI", "BufEnter", "WinEnter" }, { - group = group, - buffer = buf, - callback = function() - local st = state[buf] - -- While an analysis is pending the item positions are stale; the - -- refresh that follows will redraw. - if st and not st.dirty and st.cursor_row ~= vim.api.nvim_win_get_cursor(0)[1] - 1 then - M.draw(buf) - end - end, - }) - vim.api.nvim_create_autocmd("BufWinEnter", { - group = group, - buffer = buf, - callback = function() - window_options(buf) - end, - }) - vim.api.nvim_create_autocmd("BufWipeout", { - group = group, - buffer = buf, - callback = function() - local st = state[buf] - if st and st.timer then - st.timer:stop() - st.timer:close() - end - state[buf] = nil - end, - }) - vim.diagnostic.config({ virtual_text = true, signs = true, underline = true }, check_ns) - window_options(buf) - M.refresh(buf) -end +The channel is buffered with room for all $m$ jobs, so feeding never blocks. -return M + +```go +for i := 0; i < m; i++ { + jobs <- i +} +close(jobs) ``` -## Tangle, weave, run + +```go +go func() { + wg.Wait() + close(results) +}() +``` - +Ranging over `results` ends when the channel closes, which happens after the +last worker returns β€” whether it ran out of jobs or was cancelled. -```lua --- :Tangle, :Weave and :TangleCompileAndRun. The run loop streams JSON events --- from `litgo run --json`; diagnostics arrive already translated to .lit.md --- positions, so they can be placed in the buffer the programmer is editing. -local M = {} + +```go +done := 0 +for r := range results { + fmt.Println(r) + done++ +} +fmt.Printf("%d of %d jobs finished\n", done, m) +``` +```` -local run_ns = vim.api.nvim_create_namespace("litgo-run") -M.ns = run_ns +# An example with a proof -local job -- the running vim.system object +The second example is as small as a proved program gets and still says +something: an integer square root by bisection, with a contract, an invariant +and a variant. It is the file to open to see what [Proofs](#proofs) looks like +from the outside. The tests prove it, and the plugin's test breaks it to see +the editor object. -local panel = { buf = nil } + + +````markdown + + + -local function litgo() - return require("litgo") -end -``` +# A square root you can trust -Program output streams into a panel at the bottom. The positions in it are -already in terms of the document, and pressing `` on one jumps there, as -`q` closes it again. +The integer square root of $n$ is the largest integer whose square does not +exceed $n$: -While the program is running the cursor stays where the typing was, because a -panel that steals the cursor mid-run is a panel that eats keystrokes. So the -panel opens beside the work and gives the window back, and takes the cursor -only once the run is over β€” unless the run ended at an error, where the cursor -has somewhere better to be. +$$ +r = \lfloor \sqrt{n} \rfloor \quad\Longleftrightarrow\quad r^2 \le n < (r+1)^2 +$$ -```lua -local function panel_buf() - if panel.buf and vim.api.nvim_buf_is_valid(panel.buf) then - return panel.buf - end - local buf = vim.api.nvim_create_buf(false, true) - vim.api.nvim_buf_set_name(buf, "litgo://output") - vim.bo[buf].buftype = "nofile" - vim.bo[buf].bufhidden = "hide" - vim.bo[buf].swapfile = false - vim.bo[buf].filetype = "litgo-output" - vim.keymap.set("n", "q", "close", { buffer = buf, silent = true }) - -- on a `file.lit.md:12:3` reference jumps there. - vim.keymap.set("n", "", function() - local file, lnum, col = vim.api.nvim_get_current_line():match("([^%s:]+%.lit%.md):(%d+):?(%d*)") - if not file then - return - end - if vim.fn.filereadable(file) == 0 and panel.dir then - file = panel.dir .. "/" .. file - end - local target = vim.fn.bufnr(file) - for _, win in ipairs(vim.api.nvim_list_wins()) do - if vim.api.nvim_win_get_buf(win) == target then - vim.api.nvim_set_current_win(win) - vim.api.nvim_win_set_cursor(win, { tonumber(lnum), math.max((tonumber(col) or 1) - 1, 0) }) - return - end - end - vim.cmd("wincmd p | edit +" .. lnum .. " " .. vim.fn.fnameescape(file)) - end, { buffer = buf, silent = true }) - panel.buf = buf - return buf -end +This program finds it by bisection, and it comes with a proof. Not a test that +it gets $\sqrt{17}$ right, but a proof that the inequality on the right holds +for the result of *every* call with $n \ge 0$, that the loop always stops, and +that the division in it can never divide by zero. The proof is written in the +`//@` comments, in the notation of +[VeGo](https://arxiv.org/abs/2608.22630), and `litgo run` checks it before it +compiles anything: -local function panel_win() - local buf = panel_buf() - for _, win in ipairs(vim.api.nvim_list_wins()) do - if vim.api.nvim_win_get_buf(win) == buf then - return win - end - end - local current = vim.api.nvim_get_current_win() - vim.cmd("botright " .. litgo().config.run.panel_height .. "split") - local win = vim.api.nvim_get_current_win() - vim.api.nvim_win_set_buf(win, buf) - vim.wo[win].number = false - vim.wo[win].relativenumber = false - vim.wo[win].signcolumn = "no" - vim.wo[win].winfixheight = true - vim.wo[win].wrap = true - vim.api.nvim_set_current_win(current) - return win -end +```sh +litgo run examples/isqrt.lit.md # prove, then compile, then run +litgo prove examples/isqrt.lit.md # only prove +``` ---- Put the cursor in the panel, if it is on screen. -local function panel_focus() - local buf = panel.buf - if not (buf and vim.api.nvim_buf_is_valid(buf)) then - return - end - for _, win in ipairs(vim.api.nvim_list_wins()) do - if vim.api.nvim_win_get_buf(win) == buf then - vim.api.nvim_set_current_win(win) - return - end - end -end +```mermaid +flowchart LR + doc[isqrt.lit.md] -->|tangle| go[isqrt.go] + go -->|prove| ok{proved?} + ok -->|yes| build[go build, run] + ok -->|no| line[a violet line under the annotation that failed] +``` -local function panel_clear() - local buf = panel_buf() - vim.api.nvim_buf_set_lines(buf, 0, -1, false, {}) - vim.api.nvim_buf_clear_namespace(buf, run_ns, 0, -1) -end +## The contract -local function panel_add(text, hl) - local buf = panel_buf() - if panel.dir then - text = text:gsub(vim.pesc(panel.dir .. "/"), "") -- positions read better without the directory - end - local lines = vim.split(text, "\n", { plain = true }) - local count = vim.api.nvim_buf_line_count(buf) - local empty = count == 1 and vim.api.nvim_buf_get_lines(buf, 0, 1, false)[1] == "" - local first = empty and 0 or count - vim.api.nvim_buf_set_lines(buf, first, -1, false, lines) - if hl then - for i = 0, #lines - 1 do - vim.api.nvim_buf_set_extmark(buf, run_ns, first + i, 0, { line_hl_group = hl }) - end - end - local win = panel_win() - vim.api.nvim_win_set_cursor(win, { vim.api.nvim_buf_line_count(buf), 0 }) -end +The function promises the defining inequality, and asks for one thing in +return. `Requires` is what the caller owes, `Ensures` is what the function +owes back, and the result has a name, `r`, so that the promise can mention it. + +```go +//@ Requires n >= 0 +func Isqrt(n int) (r int) { + // <> +} +//@ Ensures r*r <= n ^ n < (r+1)*(r+1) ``` -Diagnostics from a run are shown as virtual lines, right under the code that -caused them. They stay accurate while you edit. Changing a line dismisses the -error on it, and errors below an edit move with their text. +`^` is *and*. The `Ensures` goes under the closing brace, where a conclusion +belongs. -```lua ---- Place diagnostics in every file they mention and mirror them in quickfix. -local function publish(diags, origin) - local by_file, qf = {}, {} - for _, d in ipairs(diags) do - local file = (d.file and d.file ~= "") and d.file or vim.api.nvim_buf_get_name(origin) - by_file[file] = by_file[file] or {} - table.insert(by_file[file], d) - qf[#qf + 1] = { - filename = file, - lnum = d.line + 1, - col = d.col + 1, - text = d.message, - type = d.severity == 1 and "E" or "W", - } - end - for file, list in pairs(by_file) do - local buf = vim.fn.bufadd(file) - vim.fn.bufload(buf) - -- What the compiler says, the language server has usually said already. - if require("litgo.lsp").reporting(buf) then - list = vim.tbl_filter(function(d) - return d.source ~= "go build" - end, list) - end - vim.diagnostic.set(run_ns, buf, require("litgo.render").to_diagnostics(list, buf)) - end - vim.fn.setqflist({}, " ", { title = "litgo", items = qf }) -end +## The loop -function M.clear(buf) - vim.diagnostic.reset(run_ns) - if buf then - vim.diagnostic.reset(run_ns, buf) - end -end +Bisection keeps two numbers, one whose square is known to be small enough and +one whose square is known to be too big, and moves one of them to the middle +until they are neighbours. ---- Keep run diagnostics accurate while editing: fixing a line dismisses its ---- error, and errors below an edit move with their text. -function M.attach(buf) - vim.diagnostic.config({ - virtual_lines = true, -- the message sits under the offending line - virtual_text = false, - signs = true, - underline = true, - severity_sort = true, - }, run_ns) - -- Several edits can arrive before the scheduled update runs (a macro, a - -- :global command), so each works on the result of the one before. - local pending - vim.api.nvim_buf_attach(buf, false, { - on_lines = function(_, _, _, first, last_old, last_new) - local diags = pending or vim.diagnostic.get(buf, { namespace = run_ns }) - if #diags == 0 then - return - end - local kept, changed = {}, false - for _, d in ipairs(diags) do - if d.lnum >= first and d.lnum < math.max(last_old, first + 1) then - changed = true - else - if d.lnum >= last_old and last_new ~= last_old then - d.lnum = d.lnum + last_new - last_old - d.end_lnum = d.end_lnum and d.end_lnum + last_new - last_old - changed = true - end - kept[#kept + 1] = d - end - end - if changed and not pending then - vim.schedule(function() - if vim.api.nvim_buf_is_valid(buf) then - vim.diagnostic.set(run_ns, buf, pending) - end - pending = nil - end) - end - if changed then - pending = kept - end - end, - }) -end + +```go +lo, hi := 0, n+1 +//@ Invariant 0 <= lo < hi +//@ Invariant lo*lo <= n ^ n < hi*hi +//@ Variant hi - lo +for hi-lo > 1 { + mid := lo + (hi-lo)/2 + // <> +} +return lo ``` -```lua -local function save(buf) - if vim.bo[buf].modified then - vim.api.nvim_buf_call(buf, function() - vim.cmd("silent update") - end) - end -end +The sentence above *is* the invariant: `lo*lo <= n ^ n < hi*hi`. A loop +invariant is something true before the loop and true again after each time +round, and the verifier checks exactly those two things. It is true before, +because $0 \le n$ and $n < (n+1)^2$. Then the verifier forgets everything it +knew about `lo` and `hi` except the invariant, goes round once, and has to get +the invariant back. -function M.tangle(buf, force) - local bin = litgo().bin() - if not bin then - return - end - save(buf) - local cmd = { bin, "tangle" } - if force then - cmd[#cmd + 1] = "--force" - end - cmd[#cmd + 1] = vim.api.nvim_buf_get_name(buf) - vim.system(cmd, { text = true }, function(res) - vim.schedule(function() - local msg = vim.trim(res.stderr or "") - vim.notify(msg ~= "" and msg or "litgo: tangled", res.code == 0 and vim.log.levels.INFO or vim.log.levels.ERROR) - end) - end) -end + +```go +if mid*mid <= n { + lo = mid +} else { + hi = mid +} +``` -function M.weave(buf, format) - local bin = litgo().bin() - if not bin then - return - end - save(buf) - local cmd = { bin, "weave" } - if format and format ~= "" then - vim.list_extend(cmd, { "--to", format }) - end - cmd[#cmd + 1] = vim.api.nvim_buf_get_name(buf) - vim.system(cmd, { text = true }, function(res) - vim.schedule(function() - if res.code ~= 0 then - vim.notify(vim.trim(res.stderr or "litgo: weave failed"), vim.log.levels.ERROR) - return - end - local out = vim.trim(res.stdout) - vim.notify("litgo: wove " .. vim.fn.fnamemodify(out, ":~:.")) - if litgo().config.weave.open then - vim.ui.open(out) - end - end) - end) -end +Whichever branch runs, the test it just made is word for word the half of the +invariant it has to restore, so that part is immediate. The other line of the +invariant, `0 <= lo < hi`, needs the middle to be strictly between the ends, +and that is where the loop condition earns its keep: `hi - lo > 1` means +`(hi-lo)/2` is at least 1 and less than `hi - lo`. -function M.stop() - if job then - job:kill("sigterm") - job = nil - panel_add("β–  stopped", "LitgoError") - end -end +The same fact proves that the loop stops. The `Variant` is a quantity that is +never negative while the loop runs and gets smaller on every round. Here it is +the width of the interval, and both branches narrow it. -function M.run(buf, args) - local bin = litgo().bin() - if not bin then - return - end - if job then - job:kill("sigterm") - job = nil - end - save(buf) - M.clear(buf) - panel_clear() - panel.dir = vim.fn.fnamemodify(vim.api.nvim_buf_get_name(buf), ":p:h") +After the loop the verifier knows the invariant and that the condition is +false. From `lo < hi` and not `hi - lo > 1` it follows that `hi = lo + 1`, and +putting that into `n < hi*hi` gives the `Ensures`. Nobody had to say so: those +few lines of arithmetic are what the prover is for. - local cfg = litgo().config.run - local cmd = { bin, "run", "--json" } - if cfg.vet then - cmd[#cmd + 1] = "--vet" - end - if cfg.timeout and cfg.timeout ~= "" then - vim.list_extend(cmd, { "--timeout", cfg.timeout }) - end - cmd[#cmd + 1] = vim.api.nvim_buf_get_name(buf) - if args and #args > 0 then - cmd[#cmd + 1] = "--" - vim.list_extend(cmd, args) - end +## What the proof does not say - local diags, pending = {}, "" - local function handle(ev) - if ev.event == "tangle" then - panel_add("⟫ tangled β†’ " .. ev.out, "LitgoInfo") - elseif ev.event == "diagnostic" then - diags[#diags + 1] = ev - local where = ev.file and (ev.file .. ":" .. (ev.line + 1) .. ":" .. (ev.col + 1) .. ": ") or "" - panel_add(where .. ev.message, ev.severity == 1 and "LitgoError" or "DiagnosticWarn") - elseif ev.event == "build" then - if ev.ok then - panel_add(("βœ“ built in %d ms"):format(ev.ms), "LitgoOk") - else - panel_add("βœ— " .. (ev.stage == "tangle" and "tangle failed" or "build failed"), "LitgoError") - end - elseif ev.event == "output" then - panel_add(ev.text, ev.stream == "stderr" and "DiagnosticWarn" or nil) - elseif ev.event == "exit" and ev.ran then - local note = ev.timed_out and " (timed out)" or "" - panel_add(("%s exit %d in %d ms%s"):format(ev.code == 0 and "βœ“" or "βœ—", ev.code, ev.ms, note), ev.code == 0 and "LitgoOk" or "LitgoError") - end - end +The integers of the proof are the integers of mathematics. Go's `int` has 64 +bits, and `(n+1)*(n+1)` overflows long before `n` runs out of them. For +$n < 3 \cdot 10^9$ the two agree; beyond that, the theorem is about a program +this one only resembles. VeGo makes the same simplification, and it is the +kind of thing worth knowing about a proof before leaning on it. - local this - this = vim.system(cmd, { - text = true, - stdout = function(_, data) - if not data then - return - end - vim.schedule(function() - if job ~= this then - return -- a newer run replaced this one - end - pending = pending .. data - while true do - local nl = pending:find("\n", 1, true) - if not nl then - break - end - local line = pending:sub(1, nl - 1) - pending = pending:sub(nl + 1) - local ok, ev = pcall(vim.json.decode, line) - if ok and type(ev) == "table" then - handle(ev) - end - end - end) - end, - }, function() - vim.schedule(function() - if job ~= this then - return - end - job = nil - publish(diags, buf) - local jumped = false - if cfg.jump and vim.api.nvim_get_current_buf() == buf then - local name = vim.api.nvim_buf_get_name(buf) - for _, d in ipairs(diags) do - if d.severity == 1 and (not d.file or d.file == name) then - local row = math.min(d.line + 1, vim.api.nvim_buf_line_count(buf)) - vim.api.nvim_win_set_cursor(0, { row, d.col }) - vim.cmd("normal! zv") - jumped = true - break - end - end - end - if cfg.focus and not jumped and vim.api.nvim_get_current_buf() == buf then - panel_focus() - end - end) - end) - job = this -end +## Running it -return M +`main` has no annotations, so the verifier leaves it alone: a program can be +proved one function at a time. + +```go +func main() { + for _, n := range []int{0, 1, 2, 15, 16, 17, 1_000_000, 2_147_395_599} { + fmt.Printf("isqrt(%d) = %d\n", n, Isqrt(n)) + } +} ``` -## Errors as you type +## Breaking it - +The quickest way to believe a verifier is to lie to it. Change `lo = mid` to +`lo = mid + 1`, which looks like the usual bisection and is wrong here, and +run it again: -The language server gets started for each `.lit.md` buffer, and buffers under -the same root share one server. Neovim shows what the server publishes like any -other diagnostics, except that the plugin asks for them to be shown in place, -as virtual lines under the code, the same way the diagnostics of a run are -shown. +```text +isqrt.lit.md:61:1: error: Invariant is not maintained by the body of the loop at line 64: 0 <= lo < hi +isqrt.lit.md:62:1: error: Invariant is not maintained by the body of the loop at line 64: lo*lo <= n ^ n < hi*hi +litgo: proved 0 of 1 annotated functions in isqrt.lit.md (11 obligations, 79 ms) +litgo: not proved, so not run (--no-prove runs it anyway) +``` -The server brings gopls with it, and the only thing it needs from the plugin is -help finding gopls. Most Neovim users get gopls from Mason, and Mason installs -it in a place where only Neovim looks. After that, everything Neovim does for a -buffer with a language server works here too: `K`, `grr`, `grn`, ``, -signature help on ``, and whatever a completion plugin or a distribution -adds. +Both complaints are right. `(mid+1)*(mid+1) <= n` is not what the test +established, and `mid + 1` can be `hi` itself, which leaves the interval +empty. In the editor the two lines are violet rather than red, because the +program still compiles: it is the argument that is broken, not the syntax. -Completion is the exception, because nothing switches it on by itself. -nvim-cmp and blink.cmp find the server through the buffer it is attached to, so -they need nothing from the plugin; plain Neovim completes only when asked, on -``. So the plugin turns Neovim's own popup on when no completion -plugin is loaded, and stays out of the way when one is. A completion plugin is -usually loaded on the first insert, which is later than the server attaches, so -the question is asked on the first insert too β€” and then answered once. +Or weaken the loop condition to `hi-lo > 0`, and the verifier points at the +`Variant`: with an interval of width one the middle is `lo`, nothing moves, +and the loop never ends. A test would have hung. The proof just says no. +```` + +# A tutorial in proofs + +The third is a tour of every annotation the verifier checks, one small +function at a time, ending with binary search. Every function in it is proved +by the tests, so the tutorial cannot drift away from the verifier it +describes. + + + +````markdown + + + + +# VeGo in litgo: a tutorial + +[VeGo](https://arxiv.org/abs/2608.22630) (Verified Go, by Tina Massoudi and +Chris Dutchyn) is a way of writing down what a Go function promises, and why +its loops are right, in comments that start with `//@`. The program stays +ordinary Go. litgo reads the comments and proves them, or tells you which one +it could not prove and for which line of code. + +This document is a tour of every annotation litgo checks, one small function +at a time. Every function here is proved each time the document is run: + +```sh +litgo prove examples/vego-tutorial.lit.md # prove, and list what was proved +litgo run examples/vego-tutorial.lit.md # prove, then compile and run +litgo check examples/vego-tutorial.lit.md # chunks, syntax, types, proofs +``` + +In Neovim the annotations are violet and their formulas are typeset, so that +`^` reads ∧, `<=` reads ≀ and `Forall` reads βˆ€ until the cursor is on the line, +where you get back what you typed. Anything the verifier has to say about them +is violet too, as you type. A red line means the compiler will not take the +program. A violet line means the program runs and has not been shown to keep +its word. -```lua -local M = {} +The best way to read this is to break things. Change a `<` to a `<=` in any +block and see what the verifier says. ---- The completion plugins that drive completion themselves. -local engines = { "blink.cmp", "cmp", "coq" } +```mermaid +flowchart LR + req[Requires] --> body[the body, one path at a time] + inv[Invariant, Variant] --> body + body --> ens[Ensures at every return] + body --> safe[every index in range, every divisor not zero] + body --> calls[the Requires of every call] +``` -local function engine_loaded() - for _, name in ipairs(engines) do - if package.loaded[name] then - return true - end - end - return false -end +## A contract ---- Turn Neovim's own completion on for this buffer, unless something else ---- is doing the job. Deferred to the first insert, because that is when a ---- lazily loaded completion plugin arrives. -local function complete_with(id, buf) - vim.api.nvim_create_autocmd("InsertEnter", { - buffer = buf, - once = true, - callback = function() - vim.schedule(function() - if not engine_loaded() and vim.api.nvim_buf_is_valid(buf) then - vim.lsp.completion.enable(true, id, buf, { autotrigger = true }) - end - end) - end, - }) -end +`Requires` is what the caller owes. `Ensures` is what the function owes back. +`Requires` goes above the function and `Ensures` directly under its closing +brace, which is where a conclusion belongs. The result needs a name, so that +the `Ensures` has something to call it. -local function litgo() - return require("litgo") -end +```go +//@ Requires x > 0 ^ y > 0 +func AddPositives(x, y int) (sum int) { + return x + y +} +//@ Ensures sum > x ^ sum > y +``` ---- What to tell the server about gopls: where it is, to do without ("off"), ---- or to look for itself (""). -local function gopls(want) - if want == false then - return "off" - elseif type(want) == "string" then - return vim.fn.expand(want) - end - local mason = vim.fn.stdpath("data") .. "/mason/bin/gopls" - if vim.fn.executable("gopls") == 0 and vim.fn.executable(mason) == 1 then - return mason - end - return "" -end +Without the `Requires` this is false (try `0` and `0`), and the verifier says +so: *Ensures is not proved for the return at line 50*. -function M.start(buf) - local cfg = litgo().config.lsp - local bin = cfg.enabled and litgo().bin() - if not bin then - return - end - local name = vim.api.nvim_buf_get_name(buf) - vim.lsp.start({ - name = "litgo", - cmd = { bin, "lsp", "--gopls", gopls(cfg.gopls) }, - init_options = cfg.settings, - root_dir = vim.fs.root(name, { "go.mod", ".git" }) or vim.fs.dirname(name), - on_attach = function(client, bufnr) - vim.diagnostic.config({ - virtual_lines = cfg.virtual_lines, - virtual_text = not cfg.virtual_lines, - severity_sort = true, - }, vim.lsp.diagnostic.get_namespace(client.id)) - if cfg.completion == "auto" then - complete_with(client.id, bufnr) - elseif cfg.completion then - vim.lsp.completion.enable(true, client.id, bufnr, { autotrigger = true }) - end - end, - }, { bufnr = buf }) -end +A function without annotations is not looked at. So a program can be proved a +function at a time, and `main` below is left alone. -local function client(buf) - return vim.lsp.get_clients({ bufnr = buf, name = "litgo" })[1] -end +One thing to know before your editor surprises you: gofmt rewrites `//@` in a +doc comment as `// @`, and puts a blank line above an `Ensures`. litgo reads +both spellings and looks upward past the blank line, so formatting on save +changes how a contract looks and not what it means. ---- Is the server attached to this buffer? -function M.active(buf) - return client(buf) ~= nil -end +## The language of formulas ---- Is the server reporting errors in this buffer right now? If it is, a ---- run's compile errors would only repeat them. -function M.reporting(buf) - local c = client(buf) - return c ~= nil - and #vim.diagnostic.get(buf, { - namespace = vim.lsp.diagnostic.get_namespace(c.id), - severity = vim.diagnostic.severity.ERROR, - }) - > 0 -end +| | | +|:--|:--| +| `^` `v` `~` | and, or, not | +| `->` `<-` `<->` | implies, is implied by, if and only if | +| `=` `<>` `<` `<=` `>` `>=` | comparisons. They chain: `0 <= i < n`. Go's `==` and `!=` are accepted too. | +| `+` `-` `*` `/` `%` | integer arithmetic; `/` and `%` are Go's, which round towards zero | +| `len(A)`, `A[i]` | the length and the elements of a slice of integers | +| `s.Len` | an integer or boolean field of a struct | +| `Forall k in [a, b) : P` | for every integer `k` with `a <= k < b` | +| `Exists k in [a, b] : P` | for some `k` with `a <= k <= b` | +| `Unique k in (a, b) : P` | for exactly one `k` with `a < k < b` | +| `Z`, `[0, ...)` | all integers; an interval without an upper end | +| `A[a:b) <= m` | every element of that range of `A` is at most `m` | +| `m in A[a:b)`, `m = A[a:b)` | some element of the range is `m` | +| `x'` | `x` after an assignment, where `x` is before it | -return M -``` +The length of a slice is usually written the way mathematicians write it, +`|A|`, and that is how the rest of this tutorial writes it. Go's own `&&`, +`||` and `!` are accepted for *and*, *or* and *not*. -## Chunks +A formula that is too long for a line goes on in the next `//@` comment, if +its line ends in something that cannot end a formula: an operator, say. - +## Safety comes for free -```lua --- Chunk editing and navigation: :Untangle, :RenameChunk, gd and K. -local M = {} +Some obligations are not written by anybody, because the code implies them. +Every index has to be inside its slice, and every divisor has to be different +from zero. This function needs its `Requires` for no other reason: -local function litgo() - return require("litgo") -end +```go +//@ Requires 0 <= i ^ i < |A| ^ d <> 0 +func ElementOver(A []int, i, d int) (q int) { + return A[i] / d +} +//@ Ensures q * d + A[i] % d = A[i] +``` -local function refresh(buf) - require("litgo.render").refresh(buf) -end +Take away `i < |A|` and the complaint is at the `A[i]` itself: *the index may +be out of range*. Go stops evaluating `&&` and `||` early, and the verifier +knows that, so `i < len(A) && A[i] > 0` is fine with nothing said about `i`. ---- The chunk the cursor is on: a reference, a chunk directive, or the body of ---- a chunk block. Asks litgo afresh so it is right even mid-edit. -local function chunk_at_cursor(buf) - local analysis, err = litgo().query(buf, { "analyze", "--path", vim.api.nvim_buf_get_name(buf) }) - if not analysis then - return nil, err - end - local pos = vim.api.nvim_win_get_cursor(0) - local row, col = pos[1] - 1, pos[2] - local on_line - for _, c in ipairs(analysis.chunks or {}) do - for _, r in ipairs(c.refs) do - if r.line == row then - if col >= r.col and col < r.end_col then - return c, "ref" - end - on_line = on_line or c - end - end - end - if on_line then - return on_line, "ref" - end - for _, c in ipairs(analysis.chunks or {}) do - for _, d in ipairs(c.defs) do - if row == d.directive or (row >= d.first - 1 and row <= d.last + 1) then - return c, "def" - end - end - end - return nil, "the cursor is not on a chunk or a chunk reference" -end +The `Ensures` shows that the prover knows what division means: a quotient and +a remainder add up. ---- :[range]Untangle [name] ---- A charwise visual selection that stops short of whole lines becomes an ---- inline reference (a block comment); anything else a whole-line one. -function M.untangle(buf, o) - local args = { - "untangle", - "--path", vim.api.nvim_buf_get_name(buf), - "--start", tostring(o.line1 - 1), - "--end", tostring(o.line2 - 1), - } - local s, e = vim.api.nvim_buf_get_mark(buf, "<"), vim.api.nvim_buf_get_mark(buf, ">") - local from_visual = o.range == 2 and s[1] == o.line1 and e[1] == o.line2 - if from_visual and vim.fn.visualmode() == "v" then - local last = vim.api.nvim_buf_get_lines(buf, e[1] - 1, e[1], false)[1] or "" - local end_col = math.min(e[2], math.max(#last - 1, 0)) - -- '> is the first byte of the last selected character; step past it. - end_col = end_col + #vim.fn.strcharpart(last:sub(end_col + 1), 0, 1) - if vim.o.selection == "exclusive" then - end_col = math.min(e[2], #last) - end - vim.list_extend(args, { "--start-col", tostring(s[2]), "--end-col", tostring(end_col) }) - end - if o.args and o.args ~= "" then - vim.list_extend(args, { "--name", o.args }) - end +```go +//@ Requires n >= 0 +func Halve(n int) (h int) { + return n / 2 +} +//@ Ensures 2*h <= n ^ n < 2*h + 2 +``` - local res, err = litgo().query(buf, args) - if not res then - vim.notify("litgo: " .. err, vim.log.levels.ERROR) - return - end - litgo().apply_edits(buf, res.edits) - local target = litgo().config.untangle.jump and res.def_line or res.ref_line - vim.api.nvim_win_set_cursor(0, { target + 1, 0 }) - vim.cmd("normal! ^") - refresh(buf) - vim.notify(("litgo: ⟨%s⟩ β†’ line %d (:RenameChunk to rename)"):format(res.name, res.def_line + 1)) -end +## Loops ---- :RenameChunk [new name] β€” renames every definition and every reference. -function M.rename(buf, new_name) - local chunk, err = chunk_at_cursor(buf) - if not chunk then - vim.notify("litgo: " .. err, vim.log.levels.WARN) - return - end - local function apply(name) - if not name or vim.trim(name) == "" or name == chunk.name then - return - end - local res, rename_err = litgo().query(buf, { - "rename", "--path", vim.api.nvim_buf_get_name(buf), "--from", chunk.name, "--to", name, - }) - if not res then - vim.notify("litgo: " .. rename_err, vim.log.levels.ERROR) - return - end - litgo().apply_edits(buf, res.edits) - refresh(buf) - vim.notify(("litgo: ⟨%s⟩ β†’ ⟨%s⟩ (%d place%s)"):format(chunk.name, res.name, #res.edits, #res.edits == 1 and "" or "s")) - end - if new_name and new_name ~= "" then - apply(new_name) - else - vim.ui.input({ prompt = "Rename ⟨" .. chunk.name .. "⟩ to: ", default = chunk.name }, apply) - end -end +Nobody knows how often a loop goes round, so the verifier does not try to +follow it. It asks for an **invariant**: a formula that is true when the loop +is reached and true again after every round. It proves those two things, and +after the loop the invariant, together with the loop condition being false, is +everything it knows. ---- gd: from a reference to its definition; from a definition to its uses. ---- The language server does this better, because it does it for Go as well: ---- on a chunk it answers for itself, and anywhere else in a block, for gopls. -function M.goto_chunk() - local buf = vim.api.nvim_get_current_buf() - if require("litgo.lsp").active(buf) then - vim.lsp.buf.definition() - return - end - local chunk, where = chunk_at_cursor(buf) - if not chunk then - pcall(vim.cmd, "normal! gd") - return - end - vim.cmd("normal! m'") - if where == "ref" then - if #chunk.defs == 0 then - vim.notify("litgo: ⟨" .. chunk.name .. "⟩ is not defined", vim.log.levels.WARN) - return - end - vim.api.nvim_win_set_cursor(0, { chunk.defs[1].first + 1, 0 }) - elseif #chunk.refs == 1 then - vim.api.nvim_win_set_cursor(0, { chunk.refs[1].line + 1, chunk.refs[1].col }) - elseif #chunk.refs == 0 then - vim.notify("litgo: ⟨" .. chunk.name .. "⟩ is never used", vim.log.levels.WARN) - else - local items = {} - for _, r in ipairs(chunk.refs) do - items[#items + 1] = { - bufnr = buf, - lnum = r.line + 1, - col = r.col + 1, - text = vim.trim(vim.api.nvim_buf_get_lines(buf, r.line, r.line + 1, false)[1] or ""), - } - end - vim.fn.setloclist(0, {}, " ", { title = "uses of ⟨" .. chunk.name .. "⟩", items = items }) - vim.cmd("lopen") - end - vim.cmd("normal! zvzz") -end +A **variant** is for termination: an integer expression that is not negative +while the loop runs and that every round makes smaller. A loop without one is +still proved correct *if* it ends, and gets a warning that says so. + +```go +//@ Requires n >= 0 ^ d > 0 +func Divide(n, d int) (q, r int) { + r = n + //@ Invariant n = q*d + r ^ r >= 0 + //@ Variant r + for r >= d { + q, r = q+1, r-d + } + return q, r +} +//@ Ensures n = q*d + r ^ 0 <= r < d +``` ---- K: preview the chunk behind a reference without leaving the host block. ---- This too is the language server's if it is there: a chunk on a reference, ---- gopls's documentation on a name. -function M.hover() - local buf = vim.api.nvim_get_current_buf() - if require("litgo.lsp").active(buf) then - vim.lsp.buf.hover() - return - end - local chunk, where = chunk_at_cursor(buf) - if not chunk or where ~= "ref" or #chunk.defs == 0 then - pcall(vim.cmd, "normal! K") - return - end - local lines = { "⟨" .. chunk.name .. "⟩ ≑", "```go" } - for _, d in ipairs(chunk.defs) do - vim.list_extend(lines, vim.api.nvim_buf_get_lines(buf, d.first, d.last + 1, false)) - end - lines[#lines + 1] = "```" - vim.lsp.util.open_floating_preview(lines, "markdown", { border = "rounded", focus_id = "litgo-chunk" }) -end +Read the invariant against the `Ensures`. They differ only in `r < d`, and +that is exactly the loop condition being false. Finding an invariant is mostly +this: take the postcondition and weaken it until it is true before the loop +starts. -return M +`Requires d > 0` is there for the variant. With `d = 0` the loop would run +forever, and the verifier notices: *Variant is not shown to decrease*. + +The two annotations may also stand at the end of the loop's body, which is +where VeGo's own examples tend to put them. + +### Counted loops, `range`, `continue` + +The counter of a `for` loop is in scope for the invariant. + +```go +//@ Requires n >= 0 +func SumTo(n int) (s int) { + //@ Invariant 0 <= i <= n + 1 ^ 2*s = i*(i-1) + //@ Variant n + 1 - i + for i := 0; i <= n; i++ { + s += i + } + return s +} +//@ Ensures 2*s = n*(n+1) ``` -## Testing the plugin +That is Gauss's formula, with the division multiplied away. Notice that +nothing here is linear, and the proof goes through anyway, because +`(i+1)*i` and `i*i + i` are the same polynomial. -Headless Neovim drives the real plugin against the real binary. The test covers -rendering, both kinds of untangling, renaming, a run that succeeds, a run with -an error inside a chunk, diagnostics that follow edits, and a table whose -columns get lined up. +A `range` loop always ends, so it needs no variant. Its counter runs from `0` +to the length, and the verifier knows that without being told. `continue` +jumps to the end of the round, where the invariant is due as usual. The call +of `fmt.Println` is let through because all it is handed are numbers and a +string, so it cannot change anything the proof is about. -```sh -go build -o bin/litgo . && nvim --headless -u NONE -l test/plugin.lua +```go +func Count(A []int, x int) (n int) { + //@ Invariant 0 <= n <= i + for i, a := range A { + if a != x { + continue + } + n++ + fmt.Println(" found", x, "at", i) + } + return n +} +//@ Ensures 0 <= n <= |A| ``` - - -```lua --- Headless test of the plugin: nvim --headless -u NONE -l test/plugin.lua -local root = vim.fn.fnamemodify(debug.getinfo(1, "S").source:sub(2), ":p:h:h") -vim.opt.rtp:prepend(root) -local litgo = root .. "/bin/litgo" -assert(vim.fn.executable(litgo) == 1, "build bin/litgo first: go build -o bin/litgo .") -vim.o.swapfile = false +### `break`, and what cannot happen -local failures = 0 -local function check(cond, msg) - if cond then - print("ok " .. msg) - else - failures = failures + 1 - print("FAIL " .. msg) - end -end -local function wait(ms, cond, what) - if not vim.wait(ms, cond, 20) then - check(false, "timed out waiting for " .. what) - end -end -local function find(buf, pattern) - for i, l in ipairs(vim.api.nvim_buf_get_lines(buf, 0, -1, false)) do - if l:find(pattern, 1, true) then - return i - end - end -end +A `break` leaves the loop with whatever is known at that point. The invariant +is not assumed there, since the round was not finished. -local dir = vim.fn.tempname() -vim.fn.mkdir(dir, "p") -local file = dir .. "/pool.lit.md" -vim.fn.writefile(vim.fn.readfile(root .. "/examples/worker_pool.lit.md"), file) +`Exsures` is the negative of `Ensures`. It describes a state that no return +may be in, which is sometimes the more natural thing to say: *it never +happens that a position is returned and the element there is not `x`*. -require("litgo").setup({ bin = litgo, weave = { open = false }, run = { jump = true } }) -vim.cmd.edit(file) -local buf = vim.api.nvim_get_current_buf() -local render = require("litgo.render") -local run = require("litgo.run") +```go +func Find(A []int, x int) (at int) { + at = -1 + //@ Invariant 0 <= i <= |A| ^ at = -1 + //@ Invariant x <> A[0:i) + //@ Variant |A| - i + for i := 0; i < len(A); i++ { + if A[i] == x { + at = i + break + } + } + return at +} +//@ Ensures at = -1 v 0 <= at < |A| +//@ Ensures at = -1 -> Forall k in [0, |A|) : A[k] <> x +//@ Exsures at >= 0 ^ A[at] <> x +``` --- Rendering ------------------------------------------------------------------ -wait(3000, function() - local st = render.get(buf) - return st and st.items ~= nil -end, "analysis") -local ns = vim.api.nvim_get_namespaces()["litgo-render"] -local marks = vim.api.nvim_buf_get_extmarks(buf, ns, 0, -1, { details = true }) -local inline, blocks, concealed = 0, 0, 0 -for _, m in ipairs(marks) do - local d = m[4] - if d.virt_text_pos == "inline" then inline = inline + 1 end - if d.virt_lines then blocks = blocks + 1 end - if d.conceal_lines then concealed = concealed + 1 end -end -check(inline >= 12, "inline math and chunk names are rendered (" .. inline .. ")") -check(blocks == 3, "two diagrams and one display formula are drawn (" .. blocks .. ")") -check(concealed == 3, "their source is concealed while the cursor is elsewhere") -check(#vim.diagnostic.get(buf) == 0, "a correct document has no diagnostics") +## Quantifiers, ranges and predicates --- Moving onto a diagram reveals its source. -vim.api.nvim_win_set_cursor(0, { find(buf, "```mermaid") + 1, 0 }) -vim.cmd("doautocmd CursorMoved") -concealed = 0 -for _, m in ipairs(vim.api.nvim_buf_get_extmarks(buf, ns, 0, -1, { details = true })) do - if m[4].conceal_lines then concealed = concealed + 1 end -end -check(concealed == 2, "the block under the cursor shows its source") +`x <> A[0:i)` in the last example was a quantifier in disguise: *`x` differs +from every element of `A` from `0` up to but not including `i`*. The ends of a +range are written like the ends of an interval, `[` or `]` for an end that is +included and `(` or `)` for one that is not. An order comparison with a range +holds for all of its elements; `=` and `in` hold for some element. --- :Untangle, linewise ---------------------------------------------------------- -local before = vim.fn.system({ litgo, "tangle", "--stdout", file }) -local l1 = find(buf, "for job := range jobs") -vim.cmd(("%d,%dUntangle"):format(l1, l1 + 9)) -check(vim.api.nvim_buf_get_lines(buf, l1 - 1, l1, false)[1] == "\t// <>", "the selection became a reference") -check(find(buf, "") ~= nil, "the chunk landed at the bottom") -vim.cmd("silent write") -check(vim.fn.system({ litgo, "tangle", "--stdout", file }) == before, "untangling leaves the program unchanged") +```go +//@ Requires |A| > 0 +func Max(A []int) (m int) { + m = A[0] + //@ Invariant 1 <= i <= |A| + //@ Invariant A[0:i) <= m ^ m in A[0:i) + //@ Variant |A| - i + for i := 1; i < len(A); i++ { + if A[i] > m { + m = A[i] + } + } + return m +} +//@ Ensures A[:] <= m ^ m in A[:] +``` --- :Untangle, charwise (an expression) ------------------------------------------ -local l2 = find(buf, "jobs := make(chan int, m)") -local line = vim.api.nvim_buf_get_lines(buf, l2 - 1, l2, false)[1] -local s = line:find("make", 1, true) -vim.api.nvim_win_set_cursor(0, { l2, s - 1 }) -vim.cmd("normal! v$h") -vim.cmd("normal! \27") -vim.cmd("'<,'>Untangle") -check(vim.api.nvim_buf_get_lines(buf, l2 - 1, l2, false)[1] == "\tjobs := /*<>*/", "a charwise selection became an inline reference") -vim.cmd("silent write") -check(vim.fn.system({ litgo, "tangle", "--stdout", file }) == before, "inline untangling leaves the program unchanged") +Both halves of that `Ensures` matter. `A[:] <= m` by itself is satisfied by +returning a billion. A contract that is too weak is proved just as happily as +a good one, and it is the reader who has to notice. --- :RenameChunk ------------------------------------------------------------------- -vim.api.nvim_win_set_cursor(0, { l2, s + 5 }) -vim.cmd("let &undolevels = &undolevels") -- scripts never pause for input, so close the undo block by hand -vim.cmd("RenameChunk the jobs channel") -check(find(buf, "/*<>*/") ~= nil and find(buf, "") ~= nil, "rename updates reference and definition") -check(find(buf, "make call") == nil, "no trace of the old name") -vim.cmd("normal u") -check(find(buf, "/*<>*/") ~= nil and find(buf, "") ~= nil, "rename is a single undo step") -vim.cmd("normal! \18") -- redo +A `Predicate` gives a formula a name. It is not a function and is never run; +using it means its body. This one also shows a formula going on in the next +line, after the `:`. --- gd --------------------------------------------------------------------------- -wait(30000, function() return require("litgo.lsp").active(buf) end, "the language server") -vim.api.nvim_win_set_cursor(0, { find(buf, "// <>"), 3 }) -require("litgo.chunks").goto_chunk() -wait(10000, function() return vim.api.nvim_get_current_line() == "for i := 0; i < m; i++ {" end, "gd") -check(vim.api.nvim_get_current_line() == "for i := 0; i < m; i++ {", "gd jumps from a reference to the chunk") +```go +//@ Predicate Sorted(A) ::= Forall i in [0, |A|) : +//@ Forall j in [i, |A|) : A[i] <= A[j] --- Go in a Markdown buffer: gopls, behind the language server -------------------- -local server = vim.lsp.get_clients({ bufnr = buf, name = "litgo" })[1] -if server and server.server_capabilities.completionProvider then - local function ask(method, row, col) - local params = { textDocument = vim.lsp.util.make_text_document_params(buf), position = { line = row - 1, character = col } } - local answer = server:request_sync("textDocument/" .. method, params, 30000, buf) - return answer and answer.result - end - -- wg is declared in main and used in a chunk: only the tangled program knows what it is. - local l5 = find(buf, "wg.Add(1)") - local hover - wait(60000, function() - hover = ask("hover", l5, 1) - return hover ~= nil - end, "gopls to load the program") - check(hover and hover.contents.value:find("var wg sync.WaitGroup", 1, true) ~= nil, "hover in a chunk knows a variable of the block it is spliced into") - check(hover and hover.range.start.line == l5 - 1 and hover.range.start.character == 1, "and answers about the Markdown") - check(ask("hover", find(buf, "## The worker"), 4) == nil, "prose is not Go") +//@ Predicate Between(lo, x, hi) ::= lo <= x ^ x <= hi +``` - vim.api.nvim_buf_set_lines(buf, l5, l5, false, { "\twg." }) - local items = ask("completion", l5 + 1, 4) - local labels = {} - for _, item in ipairs(items and (items.items or items) or {}) do - labels[item.label] = item - end - check(labels.Wait ~= nil and labels.Done ~= nil, "completion in a chunk offers the methods of wg") - local edit = labels.Wait and labels.Wait.textEdit or {} - check((edit.insert or edit.range or { start = {} }).start.line == l5, "with edits that are edits of the Markdown") - vim.api.nvim_buf_set_lines(buf, l5, l5 + 1, false, {}) +## Calls - local l6 = find(buf, "go worker(ctx") - vim.api.nvim_win_set_cursor(0, { l6, 4 }) - require("litgo.chunks").goto_chunk() - wait(10000, function() return vim.api.nvim_get_current_line():find("^func worker%(") ~= nil end, "gd on a Go name") - check(vim.api.nvim_get_current_line():find("^func worker%(") ~= nil, "gd on a Go name jumps to its definition, in another chunk") -else - print("skip gopls is not installed") -end +A call is where verification is modular. The caller never looks inside the +callee: it proves the callee's `Requires`, and what it learns about the result +is the callee's `Ensures`, and nothing else. --- :TangleCompileAndRun, success ------------------------------------------------ -vim.cmd("silent write") -vim.cmd("TangleCompileAndRun") -local function panel_text() - local pb = vim.fn.bufnr("litgo://output") - return pb > 0 and table.concat(vim.api.nvim_buf_get_lines(pb, 0, -1, false), "\n") or "" -end -wait(60000, function() return panel_text():find("exit %d+ in") ~= nil end, "the program to finish") -check(panel_text():find("jobs finished") ~= nil, "program output reaches the panel") -check(panel_text():find("βœ“ built") ~= nil, "the build is reported") -check(#vim.diagnostic.get(buf, { namespace = run.ns }) == 0, "a clean run leaves no diagnostics") +```go +//@ Requires Between(0, x, 10) +func Twice(x int) (r int) { + return x + x +} +//@ Ensures r = double(x) ^ Between(0, r, 20) --- The panel has the cursor when the run is over, and q gives it back. -local function panel_win() - local pb = vim.fn.bufnr("litgo://output") - for _, w in ipairs(vim.api.nvim_list_wins()) do - if vim.api.nvim_win_get_buf(w) == pb then - return w - end - end -end -wait(10000, function() return vim.api.nvim_get_current_win() == panel_win() end, "the panel to take the cursor") -check(vim.api.nvim_get_current_win() == panel_win(), "the output panel takes the cursor when the run ends") -vim.api.nvim_feedkeys("q", "x", false) -check(panel_win() == nil, "q closes it") -check(vim.api.nvim_get_current_buf() == buf, "and the cursor is back in the document") +//@ Requires Between(1, k, 5) +func UseTwice(k int) (r int) { + return Twice(k) + Twice(k+5) +} +//@ Ensures r = 4*k + 10 +``` --- Errors as you type: the language server, nothing run ------------------------- -local function diag(pattern, ns) - for _, x in ipairs(vim.diagnostic.get(buf, { namespace = ns })) do - if x.message:find(pattern, 1, true) then - return x - end - end -end -check(require("litgo.lsp").active(buf), "the language server is attached") +Make that `Between(1, k, 6)` and the second call is refused: *this call does +not establish what Twice requires*. --- What a completion plugin needs to keep Markdown snippets out of Go blocks. -local blocks = require("litgo") -check(blocks.in_go_block(buf, find(buf, "fmt.Println(r)") - 1), "a line of Go is in a Go block") -check(not blocks.in_go_block(buf, find(buf, "# Worker pool") - 1), "a heading is not") -check(not blocks.in_go_block(buf, find(buf, "flowchart LR") - 1), "and neither is a diagram") +`double` in that `Ensures` is a Go function. A function whose whole body is +`return` and one expression can be used in a formula, where it means that +expression. It is a way to give the contract a vocabulary that the program +shares. -local l3 = find(buf, "fmt.Println(r)") -vim.api.nvim_buf_set_lines(buf, l3 - 1, l3, false, { "\tfmt.Println(rr)" }) -wait(60000, function() return diag("undefined: rr") ~= nil end, "the language server to find the type error") -local d = diag("undefined: rr") -check(d and d.lnum == l3 - 1 and d.col == 13 and d.end_col == 15, "a type error inside a chunk is underlined where it is, unsaved and unrun") -local first = diag("declared and not used: r") -check(first ~= nil, "and so is the variable that is now unused") +```go +func double(x int) int { return 2 * x } +``` + +## Recursion + +Recursion is induction. Inside `Triangle`, the call `Triangle(n-1)` is known +by the contract being proved: that is the induction hypothesis, and using it is +sound as long as the induction is well-founded. `Measure` is what makes it so: +an expression that is not negative and is smaller at every recursive call. +`BaseCase` marks where the induction starts, and the verifier checks that no +recursive call can be reached once it has held. `InductionHypothesis` states, +and proves, what makes the recursive call legal. + +```go +//@ Requires n >= 0 +//@ Measure n +func Triangle(n int) (t int) { + //@ BaseCase n = 0 + if n == 0 { + return 0 + } + //@ InductionHypothesis n - 1 >= 0 + return n + Triangle(n-1) +} +//@ Ensures 2*t = n*(n+1) +``` + +Write `Triangle(n)` instead and the contract would still "follow" from itself. +The `Measure` is what catches it: *Measure is not shown to get smaller in the +recursive call*. + +## Assertions and primes + +`Assert` is a claim about the state at one point. It is proved there, and +known from there on, so it doubles as a stepping stone when the prover needs +to be led. The word is optional: a bare formula is an assertion. + +A primed name is a value *after* an assignment. In a formula that mentions +`x'`, the `x'` is the value now and the plain `x` is the value one assignment +earlier, and with `x''` everything moves one further back. This is the old +trick of swapping two numbers without a third: + +```go +func Swap(a, b int) (x, y int) { + x, y = a, b + x = x + y + y = x - y + x = x - y + //@ Assert x = a + b ^ x' = b + //@ y = a + return x, y +} +//@ Ensures x = b ^ y = a +``` --- A run adds nothing the server has said, but still jumps and fills quickfix. -vim.api.nvim_win_set_cursor(0, { 1, 0 }) -vim.cmd("TangleCompileAndRun") -wait(60000, function() return panel_text():find("build failed") ~= nil end, "the build to fail") -wait(2000, function() return #vim.fn.getqflist() == 2 end, "the quickfix list") -check(vim.api.nvim_get_current_win() ~= panel_win(), "a run that fails leaves the cursor in the document") -check(first and vim.api.nvim_win_get_cursor(0)[1] == first.lnum + 1, "the cursor jumped to the first error") -check(#vim.fn.getqflist() == 2, "the quickfix list mirrors the compiler") -check(diag("undefined: rr", run.ns) == nil, "the run does not repeat what the server reports") +In an `Ensures`, a parameter without a prime is the value the caller passed, +whatever the body has done to its copy since. -vim.api.nvim_buf_set_lines(buf, l3 - 1, l3, false, { "\tfmt.Println(r)" }) -wait(20000, function() return diag("undefined: rr") == nil and diag("declared and not used") == nil end, "the errors to clear") -check(#vim.diagnostic.get(buf) == 0, "fixing the line clears the errors") +## Promises that last: `Preserves` --- Diagnostics from a run (a panic, a failing test) stay accurate between runs. -vim.diagnostic.set(run.ns, buf, { - { lnum = l3 - 1, col = 0, message = "panic: one", severity = 1 }, - { lnum = l3 + 1, col = 0, message = "panic: two", severity = 1 }, -}) -vim.api.nvim_buf_set_lines(buf, l3 - 1, l3, false, { "\tfmt.Println(r) " }) -vim.api.nvim_buf_set_lines(buf, 0, 0, false, { "", "" }) -wait(2000, function() local x = diag("panic: two", run.ns) return x and x.lnum == l3 + 3 end, "run diagnostics to move") -check(diag("panic: one", run.ns) == nil, "editing a line dismisses the run diagnostic on it") -check(diag("panic: two", run.ns).lnum == l3 + 3, "and the ones below follow their lines") -vim.api.nvim_buf_set_lines(buf, 0, 2, false, {}) -run.clear(buf) +`Preserves` states something once and has it checked after every assignment +until the block ends. A loop inside that block may assume it, so it does not +have to be repeated in the invariant. --- Structure is the server's business too. -local l4 = find(buf, "// <>") -vim.api.nvim_buf_set_lines(buf, l4 - 1, l4, false, { "\t// <>" }) -wait(20000, function() return diag("is not defined") ~= nil and diag("is never used") ~= nil end, "the structural diagnostics") -check(diag("spawn teh workers⟩ is not defined") and diag("spawn the workers⟩ is never used"), "a misspelt reference is flagged at both ends") -vim.cmd("normal u") +```go +//@ Requires amount >= 0 +func Transfer(from, to, amount int) (f, t int) { + f, t = from, to + //@ Preserves f + t = from + to + //@ Invariant 0 <= moved <= amount ^ f = from - moved + //@ Variant amount - moved + for moved := 0; moved < amount; moved++ { + f, t = f-1, t+1 + } + return f, t +} +//@ Ensures f + t = from + to ^ f = from - amount +``` --- :Weave ------------------------------------------------------------------------ -if vim.fn.executable("pandoc") == 1 then - vim.cmd("silent write") - vim.cmd("Weave html") - wait(20000, function() return vim.uv.fs_stat(dir .. "/pool.html") ~= nil end, "the woven page") - check(vim.uv.fs_stat(dir .. "/pool.html") ~= nil, ":Weave html writes pool.html") - if vim.fn.executable("typst") == 1 and vim.system({ "typst", "--version" }):wait().code == 0 then - vim.cmd("Weave") - wait(30000, function() return vim.uv.fs_stat(dir .. "/pool.pdf") ~= nil end, "the woven PDF") - check(vim.uv.fs_stat(dir .. "/pool.pdf") ~= nil, ":Weave writes pool.pdf") - end -end +No money is created or destroyed. The parallel assignment is the point: `f--` +followed by `t++` would break the promise for the length of one statement, and +the verifier would say which one. --- Tables --------------------------------------------------------------------------- -vim.cmd("enew!") -local tbuf = vim.api.nvim_get_current_buf() -vim.api.nvim_buf_set_name(tbuf, dir .. "/table.lit.md") -vim.api.nvim_buf_set_lines(tbuf, 0, -1, false, { - "# Keys", - "", - "| Key | What it does | Cost |", - "|:-:|---|--:|", - "| `K` | shows the *documentation* | $x^2$ |", - "| `` | completes | 10 |", - "", - "The end.", -}) -require("litgo").attach(tbuf) -wait(3000, function() - local st = render.get(tbuf) - return st and st.items ~= nil -end, "the analysis of the table") +## The escape hatch: `Axiom` --- What a row looks like: the text that is not concealed, and the virtual --- text where it is put in. -local function shown(row) - local text = vim.api.nvim_buf_get_lines(tbuf, row, row + 1, false)[1] - local hidden, put = {}, {} - for _, m in ipairs(vim.api.nvim_buf_get_extmarks(tbuf, ns, { row, 0 }, { row, -1 }, { details = true })) do - local d = m[4] - if d.conceal then - for i = m[3] + 1, d.end_col do hidden[i] = true end - elseif d.virt_text_pos == "inline" then - put[m[3]] = (put[m[3]] or "") .. d.virt_text[1][1] - end - end - local out = "" - for i = 1, #text + 1 do - out = out .. (put[i - 1] or "") - if i <= #text and not hidden[i] then out = out .. text:sub(i, i) end - end - return out -end -check(vim.api.nvim_buf_line_count(tbuf) == 8 and #vim.api.nvim_buf_get_extmarks(tbuf, ns, 0, -1, { details = true }) > 0, - "a table stays where it is") -check(shown(2) == "| Key | What it does | Cost |", "the header is padded, and its first column is centred: " .. shown(2)) -check(shown(3) == "|:----------:|-------------------------|-----:|", "the rule is as wide as the columns: " .. shown(3)) -check(shown(4) == "| K | shows the documentation | π‘₯Β² |", "cells are as wide as they look, not as their source: " .. shown(4)) -check(shown(5) == "| | completes | 10 |", "the widest cell decides: " .. shown(5)) +The prover is complete for nothing. Its arithmetic is linear, with products +treated as unknowns about which it knows a few facts, and the cube of a +number is past what it knows. `Axiom` tells it, and it believes you without +proof. That is exactly as dangerous as it sounds, so every `Axiom` is reported +as a warning, every time. --- The same thing as the screen shows it, which is what counts. -vim.cmd("redraw") -local screen = {} -for r = 3, 6 do - local s = "" - for c = 1, 60 do s = s .. vim.fn.screenstring(r, c) end - screen[#screen + 1] = vim.trim(s) -end -check(screen[3] == shown(4) and screen[4] == shown(5), "and that is what is on the screen: " .. screen[3]) -vim.api.nvim_win_set_cursor(0, { 5, 0 }) -vim.cmd("doautocmd CursorMoved") -check(shown(4) == "| `K` | shows the *documentation* | $x^2$ |", "the row under the cursor is left as it was typed") -check(shown(5):find("completes |", 1, true) ~= nil, "and the other rows stay lined up") -vim.cmd("bwipeout!") +```go +//@ Requires x >= 0 +func Cube(x int) (c int) { + //@ Axiom x*x*x >= 0 + return x * x * x +} +//@ Ensures c >= 0 +``` -vim.fn.delete(dir, "rf") -print(failures == 0 and "\nall passed" or ("\n" .. failures .. " failed")) -os.exit(failures == 0 and 0 or 1) +## All of it at once: binary search + +The most famous loop with a bug in it. The invariant says that everything to +the left of `lo` is too small and everything from `hi` on is too big, and +`Sorted` is what lets one comparison with `A[mid]` speak for a whole half of +the slice. + +```go +//@ Requires Sorted(A) +func Search(A []int, x int) (at int, found bool) { + lo, hi := 0, len(A) + //@ Invariant 0 <= lo <= hi <= |A| + //@ Invariant A[0:lo) < x ^ x < A[hi:|A|) + //@ Variant hi - lo + for lo < hi { + mid := lo + (hi-lo)/2 + if A[mid] == x { + return mid, true + } else if A[mid] < x { + lo = mid + 1 + } else { + hi = mid + } + } + return 0, false +} +//@ Ensures found -> 0 <= at < |A| ^ A[at] = x +//@ Ensures ~found -> x <> A[:] ``` -# An example +Four things are proved here, and only two of them were written down: if +`found`, then `at` is a position of `x`; if not, `x` is nowhere in `A`; and, +unasked, `A[mid]` is always inside the slice, and the loop ends. Try +`lo = mid` (it loops forever, and the `Variant` says so), `hi = mid - 1` (it +skips an element, and the invariant is not maintained), or drop the +`Requires` (and nothing about the halves follows). + +## What is left out + +litgo's checker is its own, because the paper's `vegop` has not been +published, and it is deliberately smaller than what the paper describes. + +* **The fragment.** Integers, booleans, slices of integers that are read and + not written, integer and boolean fields of structs, `if`, `for`, `range`, + `break`, `continue`, `return`, and calls of annotated functions. An + annotated function that uses anything else is reported as *not verified*, + with the line that was too much. It is never silently passed. +* **No writes through a slice or a pointer.** `A[i] = x` may change another + slice that shares the array, and a proof that did not notice would prove + something false. So sorting is out, for now. +* **Not Go, so not here.** VeGo's `.vgo` dialect has `while`, `skip` and + functions inside functions. A litgo block is Go, where `for cond {}`, an + empty statement and a second function say the same. +* **Read but not checked.** `Property`, and contracts on interface methods. + They get a warning rather than silence. +* **The integers are mathematical.** Overflow is not modelled, here or in the + paper. +* **"Not proved" is not "false".** The prover never calls a false thing + proved, and it sometimes fails to prove a true one. The cure is usually a + stronger invariant or an `Assert` on the way. + +## Running it -This is a complete literate program. The tests above use it, and it is a good -first file to open in the editor. It is a document inside a document, so its -fence is one backtick longer than the fences it contains. It is also -`verbatim`, because its references belong to it and not to this document. +```go +func main() { + fmt.Println("AddPositives(2, 3) =", AddPositives(2, 3)) + fmt.Println("ElementOver({7, 9}, 1, 2) =", ElementOver([]int{7, 9}, 1, 2)) + fmt.Println("Halve(7) =", Halve(7)) + q, r := Divide(17, 5) + fmt.Println("Divide(17, 5) =", q, r) + fmt.Println("SumTo(100) =", SumTo(100)) + A := []int{1, 3, 3, 7, 9, 12} + fmt.Println("Count(A, 3):") + fmt.Println(" =", Count(A, 3)) + fmt.Println("Find(A, 7) =", Find(A, 7)) + fmt.Println("Max(A) =", Max(A)) + fmt.Println("UseTwice(2) =", UseTwice(2)) + fmt.Println("Triangle(10) =", Triangle(10)) + x, y := Swap(1, 2) + fmt.Println("Swap(1, 2) =", x, y) + f, t := Transfer(10, 0, 4) + fmt.Println("Transfer(10, 0, 4) =", f, t) + fmt.Println("Cube(3) =", Cube(3)) + at, found := Search(A, 9) + fmt.Println("Search(A, 9) =", at, found) +} +``` +```` - +# An example with a tree in it + +The fourth proves things about data structures: a binary heap, which is a tree +kept in a slice, and a ring buffer. The numbering of the heap really is a +tree, the heap property really does put the maximum at the root, and the +operations of the ring really do keep its invariant. + + ````markdown - - - -# Worker pool in Go + -A pool of $n$ workers processes $m$ jobs concurrently. The jobs travel through -a channel, and the context is cancellable, so we can stop early and still keep -the partial results. +# A tree in an array, and a queue in a circle -If every job costs $t$ seconds, the pool finishes in about +Two data structures that are nothing but integers and a slice, which is what +makes them good first subjects for a proof: a binary heap, which is a tree +with no pointers in it, and a ring buffer, which is a queue with no end. -$$ -T(n, m) = \left\lceil \frac{m}{n} \right\rceil \cdot t -$$ +Everything here is proved by `litgo prove`, in the notation of +[VeGo](https://arxiv.org/abs/2608.22630). What "proved" means for a data +structure comes in three kinds, and each shows up below: -instead of the $m \cdot t$ a single goroutine would need: a speed-up of -$\frac{m t}{T} \approx n$ for as long as $n \le m$. +* **The shape is right.** The arithmetic that finds a parent or a child really + does describe a tree: children know their parent, nobody is their own + ancestor, no index falls off the slice. +* **The invariant means something.** From "every node is at most its parent", + which is a local fact, it follows that the root is the maximum, which is a + global one. That step is an induction, and a loop is how an induction is + written. +* **The operations keep the invariant.** Whatever state a ring buffer is in, + if it was a valid one before `Push`, it is a valid one after. ```mermaid -flowchart LR - main -->|jobs| w1[worker 1] & w2[worker 2] & w3[worker 3] - w1 & w2 & w3 -->|results| collect +flowchart TD + n0["A[0] = 9"] --> n1["A[1] = 7"] & n2["A[2] = 8"] + n1 --> n3["A[3] = 3"] & n4["A[4] = 5"] + n2 --> n5["A[5] = 6"] ``` -## The shape of the program +## The shape of the tree -The whole program fits on one screen, because each step is a named chunk that -is explained further down. Chunks are spliced in *textually*, so they see -`ctx`, `jobs`, `results` and `wg` exactly as if they had been written here. +The tree is in the numbering. Node $i$ has its children at $2i+1$ and $2i+2$, +and so its parent at $\lfloor (i-1)/2 \rfloor$. Three one-line functions, and +their contracts say that they fit together. ```go -// <> +//@ Requires i >= 0 +func Left(i int) (c int) { return 2*i + 1 } +//@ Ensures Parent(c) = i ^ c > i + +//@ Requires i >= 0 +func Right(i int) (c int) { return 2*i + 2 } +//@ Ensures Parent(c) = i ^ c = Left(i) + 1 + +//@ Requires i > 0 +func Parent(i int) (p int) { return (i - 1) / 2 } +//@ Ensures 0 <= p < i +//@ Ensures i = Left(p) v i = Right(p) +``` -func main() { - // <> +`Parent(Left(i)) = i` needs the verifier to know that $2i/2 = i$ and that +$(2i+1)/2 = i$ too, which is what rounding down means. A function that is one +`return` of one expression can be used inside a formula, where it stands for +that expression, so the contracts can talk about each other. - n, m := 3, 10 - jobs := make(chan int, m) - results := make(chan int, m) - var wg sync.WaitGroup +`0 <= p < i` is the important one. It says a parent's number is smaller than +its child's. So walking upwards always makes progress, there are no cycles, +and every walk ends at node 0. That is what makes this numbering a *tree*. - // <> - // <> - // <> - // <> -} -``` +## The heap property -## The worker +A max-heap is a tree in which no node is larger than its parent. As a +predicate over the whole slice: -A worker drains `jobs` until the channel closes or the context is cancelled, -whichever comes first. Doubling stands in for real work. +```go +//@ Predicate Heap(A) ::= Forall k in [1, |A|) : A[Parent(k)] >= A[k] +``` + +Checking it is a loop over the nodes that have a parent. The contract is an +*if and only if*: `true` means it is a heap, and `false` means it is not, which +is the half people forget to promise. - ```go -func worker(ctx context.Context, id int, jobs <-chan int, results chan<- int, wg *sync.WaitGroup) { - defer wg.Done() - for job := range jobs { - select { - case <-ctx.Done(): - return - default: - // NOTE: this line is dropped by the tangler-exclude directive. - time.Sleep(20 * time.Millisecond) - results <- job * 2 +func IsHeap(A []int) (ok bool) { + //@ Invariant 1 <= i + //@ Invariant Forall k in [1, i) : k < |A| -> A[Parent(k)] >= A[k] + //@ Variant |A| - i + for i := 1; i < len(A); i++ { + if A[Parent(i)] < A[i] { + return false } } + return true } +//@ Ensures ok <-> Heap(A) ``` -```mermaid -sequenceDiagram - participant M as main - participant W as worker - M->>W: job - W-->>M: job * 2 - M-xW: cancel -``` +Nobody wrote that `A[Parent(i)]` is inside the slice. The verifier works it +out from `1 <= i < len(A)` and the meaning of division, or refuses the +function. -## Cancellation +## From local to global: the root is the maximum -The deadline is a tiny chunk of its own: an expression, interpolated inline. +The heap property only compares a node with its parent. That the root beats +*every* node follows by climbing: if $A[j] \ge A[i]$ and $j$ is not the root, +then $A[\mathrm{Parent}(j)] \ge A[j] \ge A[i]$, and the parent is closer to +the root. - ```go -ctx, cancel := context.WithCancel(context.Background()) -defer cancel() -time.AfterFunc(/*<>*/, cancel) +//@ Requires Heap(A) ^ 0 <= i < |A| +func Depth(A []int, i int) (d int) { + j := i + //@ Invariant 0 <= j <= i ^ A[j] >= A[i] ^ d >= 0 + //@ Variant j + for j > 0 { + j = Parent(j) + d++ + } + return d +} +//@ Ensures d >= 0 ^ A[0] >= A[i] ``` -Long enough for roughly half of the jobs. +The function counts how deep node `i` is. Its proof is the interesting part: +the invariant carries `A[j] >= A[i]` up the tree, and when the loop ends `j` +is 0. The `Variant` is `j` itself, which shrinks because `Parent(j) < j`. So +the same contract proves that the climb is correct and that it ends. + +That was one node. For all of them at once, the induction goes over the +numbering instead of up a path: if the root beats every node below `k`, it +beats `Parent(k)`, which beats `k`. - ```go -50 * time.Millisecond +//@ Requires Heap(A) ^ |A| > 0 +func Max(A []int) (m int) { + //@ Invariant 1 <= k <= |A| ^ A[0:k) <= A[0] + //@ Variant |A| - k + for k := 1; k < len(A); k++ { + //@ Assert A[Parent(k)] <= A[0] + } + return A[0] +} +//@ Ensures A[:] <= m ^ m in A[:] ``` -## Spawn, feed, collect +This loop does nothing. (The `Assert` is the step of the induction, written +out for the reader. The prover finds it without being told.) The loop is there +because it is the proof: VeGo has no ghost +code, so an induction has to be a loop that could run, and this one runs. It +costs a pass over the slice to return `A[0]`, which a real heap would not pay, +and the honest way to use it is as a lemma, once, in a test. It is worth +seeing anyway, because it shows what a loop invariant *is*: the induction +hypothesis, with the loop as the induction. + +## One step of sifting down + +Repairing a heap means swapping a node with its larger child. Choosing that +child is where the off-by-one errors live, because a node can have two +children, one, or none. - ```go -for i := 0; i < n; i++ { - wg.Add(1) - go worker(ctx, i, jobs, results, &wg) +//@ Requires i >= 0 ^ Left(i) < |A| +func LargerChild(A []int, i int) (c int) { + c = Left(i) + if Right(i) < len(A) && A[Right(i)] > A[c] { + c = Right(i) + } + return c } +//@ Ensures (c = Left(i) v c = Right(i)) ^ c < |A| ^ Parent(c) = i +//@ Ensures A[c] >= A[Left(i)] ^ (Right(i) < |A| -> A[c] >= A[Right(i)]) ``` -The channel is buffered with room for all $m$ jobs, so feeding never blocks. +Go stops evaluating `&&` at the first `false`, and the verifier knows it, so +`A[Right(i)]` is only obliged to be in range when `Right(i) < len(A)` has just +been checked. Swap the two halves of that condition and the proof fails at the +index. + +The swap itself is `A[i], A[c] = A[c], A[i]`, and that is where this verifier +stops: it does not follow writes into a slice, because another slice may share +the array. So `main` below does the swapping, unproved, with the proved +function telling it where. + +## A queue in a circle + +A ring buffer is three integers and a slice: where the oldest element is, how +many there are, and how much room. Its invariant is what makes those three +numbers a queue. - ```go -for i := 0; i < m; i++ { - jobs <- i +//@ Predicate Ring(head, size, room) ::= room > 0 ^ 0 <= head < room ^ 0 <= size <= room +``` + +Every operation has the same contract in outline: *given a valid ring, here is +a slot inside the buffer, and a valid ring again*. The parameters cannot +change, being integers passed by value, so the new state comes back as +results. + +```go +//@ Requires Ring(head, size, room) ^ size < room +func Push(head, size, room int) (slot, h, s int) { + return (head + size) % room, head, size + 1 } -close(jobs) +//@ Ensures 0 <= slot < room ^ Ring(h, s, room) ^ s = size + 1 + +//@ Requires Ring(head, size, room) ^ size > 0 +func Pop(head, size, room int) (slot, h, s int) { + return head, (head + 1) % room, size - 1 +} +//@ Ensures 0 <= slot < room ^ Ring(h, s, room) ^ s = size - 1 ``` - +`size < room` and `size > 0` are the two ways to misuse a queue, and they are +now the caller's problem, in writing. A caller that is itself verified has to +prove them: + ```go -go func() { - wg.Wait() - close(results) -}() +//@ Requires Ring(head, size, room) ^ size < room +func PushThenPop(head, size, room int) (h, s int) { + _, h, s = Push(head, size, room) + _, h, s = Pop(h, s, room) + return h, s +} +//@ Ensures Ring(h, s, room) ^ s = size ``` -Ranging over `results` ends when the channel closes, which happens after the -last worker returns β€” whether it ran out of jobs or was cancelled. +`Pop` is legal there because `Push` promised `s = size + 1`, which is more +than zero. Take that out of `Push`'s contract and this function stops +verifying, though `Push` itself still does: a contract is everything a caller +gets to know. + +## Running it - ```go -done := 0 -for r := range results { - fmt.Println(r) - done++ +func main() { + A := []int{9, 7, 8, 3, 5, 6} + fmt.Println("IsHeap:", IsHeap(A), " Max:", Max(A), " Depth of node 5:", Depth(A, 5)) + + // Replace the root and sift it down, one proved step at a time. + A[0] = 1 + for i := 0; Left(i) < len(A); { + c := LargerChild(A, i) + if A[i] >= A[c] { + break + } + A[i], A[c] = A[c], A[i] + i = c + } + fmt.Println("after sifting 1 down:", A, " IsHeap:", IsHeap(A)) + + buf := make([]int, 3) + head, size := 0, 0 + for _, x := range []int{10, 20, 30} { + var slot int + slot, head, size = Push(head, size, len(buf)) + buf[slot] = x + } + var slot int + slot, head, size = Pop(head, size, len(buf)) + fmt.Println("popped", buf[slot]) + slot, head, size = Push(head, size, len(buf)) + buf[slot] = 40 + fmt.Println("buffer:", buf, " head:", head, " size:", size) } -fmt.Printf("%d of %d jobs finished\n", done, m) ``` -```` +## Things to break + +* In `Parent`, return `i / 2`. `Left` still verifies. `Right` does not, nor + does `Parent` itself, nor `LargerChild`: the tree has lost its right + children, and everything that relied on them says so. +* In `Depth`, climb with `j = j - 1`. It still ends, and the invariant is not + maintained, because the node before `j` is no relation of it. +* In `LargerChild`, test `A[Right(i)] > A[c]` first. The index is no longer + known to be in range. +* In `Push`, drop the `% room`. The slot escapes the buffer. +```` # Publishing @@ -11280,7 +17470,8 @@ the binaries. This document is also the project's web page. Every push to `master` builds litgo from the committed sources, weaves this file to HTML with the binary it just built, and publishes the result to GitHub Pages. The tests run first, so a -push that breaks litgo doesn't replace a page that works. The HTML needs only +push that breaks litgo doesn't replace a page that works, and so do the proofs, +of this document and of the examples. The HTML needs only pandoc, because the browser runs mermaid.js and KaTeX for itself. pandoc is pinned to a release, since what Ubuntu packages is years older and the page should look the way it does when woven at home. @@ -11317,11 +17508,12 @@ jobs: run: | curl -fsSL -o pandoc.deb "https://github.com/jgm/pandoc/releases/download/$PANDOC/pandoc-$PANDOC-1-amd64.deb" sudo dpkg -i pandoc.deb - - name: Build and test + - name: Build, test and prove run: | go vet ./... go test ./... go build -o bin/litgo . + bin/litgo prove litgo.lit.md examples/isqrt.lit.md examples/vego-tutorial.lit.md examples/heap.lit.md - name: Weave run: | mkdir _site diff --git a/lua/litgo/init.lua b/lua/litgo/init.lua index 9df2c42..7737c03 100644 --- a/lua/litgo/init.lua +++ b/lua/litgo/init.lua @@ -14,6 +14,7 @@ M.config = { math = true, mermaid = true, tables = true, -- line up the columns of Markdown tables + proofs = true, -- typeset the formulas of //@ annotations: ^ as ∧, <= as ≀, Forall as βˆ€ chunks = true, -- show `// <>` as ⟨name⟩ and chunk directives as ⟨name⟩ ≑ conceal_source = true, -- hide diagram/display-math source while the cursor is elsewhere upright = false, -- true: keep math letters upright instead of π‘–π‘‘π‘Žπ‘™π‘–π‘ @@ -29,7 +30,7 @@ M.config = { panel_height = 12, }, lsp = { - enabled = true, -- run `litgo lsp`: chunk, syntax and type errors as you type + enabled = true, -- run `litgo lsp`: chunk, syntax, type and proof errors as you type virtual_lines = true, -- show them under the offending line rather than beside it gopls = true, -- completion, hover, definitions, references, rename in Go blocks; or a path, or false settings = nil, -- gopls settings, e.g. { buildFlags = { "-tags=integration" } } @@ -43,6 +44,7 @@ M.config = { stop = "s", tangle = "t", weave = "w", + prove = "p", untangle = "u", -- visual mode rename = "n", toggle_render = "v", @@ -127,6 +129,19 @@ local function define_highlights() for name, target in pairs(links) do vim.api.nvim_set_hl(0, name, { link = target, default = true }) end + local dark = vim.o.background ~= "light" + local violet = dark and "#c792ea" or "#7c3aed" + local faded = dark and "#a58fc4" or "#8b6fc0" + for name, spec in pairs({ + LitgoProofKeyword = { fg = violet, bold = true }, + LitgoProof = { fg = faded }, + LitgoProofError = { fg = violet }, + LitgoProofWarn = { fg = faded, italic = true }, + LitgoProofUnderline = { undercurl = true, sp = violet }, + }) do + spec.default = true + vim.api.nvim_set_hl(0, name, spec) + end end --- Is a line (0-based; the cursor's by default) inside a Go block? @@ -173,6 +188,9 @@ function M.attach(buf) command("TangleCompileAndRun", function(o) run.run(buf, o.fargs) end, { nargs = "*", desc = "Tangle, compile and run; errors appear inline" }) + command("Prove", function() + run.prove(buf) + end, { desc = "Check the //@ annotations: contracts, invariants, termination" }) command("LitStop", function() run.stop() end, { desc = "Kill the running program" }) @@ -203,6 +221,7 @@ function M.attach(buf) map("n", keys.stop, "LitStop", "stop the program") map("n", keys.tangle, "Tangle", "tangle") map("n", keys.weave, "Weave", "weave") + map("n", keys.prove, "Prove", "prove the annotations") map("x", keys.untangle, ":Untangle", "untangle selection into a chunk") map("n", keys.rename, "RenameChunk", "rename chunk") map("n", keys.toggle_render, "LitRender", "toggle rendering") @@ -227,6 +246,7 @@ function M.setup(opts) M.config = vim.tbl_deep_extend("force", M.config, opts or {}) M._bin = nil define_highlights() + require("litgo.proof").setup() local group = vim.api.nvim_create_augroup("litgo", { clear = true }) vim.api.nvim_create_autocmd({ "BufReadPost", "BufNewFile", "BufWinEnter" }, { group = group, diff --git a/lua/litgo/lsp.lua b/lua/litgo/lsp.lua index f25741b..1a6dda4 100644 --- a/lua/litgo/lsp.lua +++ b/lua/litgo/lsp.lua @@ -67,6 +67,7 @@ function M.start(buf) virtual_lines = cfg.virtual_lines, virtual_text = not cfg.virtual_lines, severity_sort = true, + [require("litgo.proof").handler] = true, }, vim.lsp.diagnostic.get_namespace(client.id)) if cfg.completion == "auto" then complete_with(client.id, bufnr) diff --git a/lua/litgo/proof.lua b/lua/litgo/proof.lua new file mode 100644 index 0000000..b93b3e3 --- /dev/null +++ b/lua/litgo/proof.lua @@ -0,0 +1,84 @@ +-- Code generated by litgo from litgo.lit.md; DO NOT EDIT. + +-- Diagnostics from the verifier, drawn apart from the compiler's. +local M = {} + +M.handler = "litgo/proof" + +local drawn = {} -- diagnostic namespace -> extmark namespace + +function M.is_proof(d) + return type(d.source) == "string" and d.source:find("vego", 1, true) ~= nil +end + +local function draw(buf, ns, d) + local line = vim.api.nvim_buf_get_lines(buf, d.lnum, d.lnum + 1, false)[1] or "" + local warn = d.severity ~= vim.diagnostic.severity.ERROR + local hl = warn and "LitgoProofWarn" or "LitgoProofError" + local pad = string.rep(" ", vim.fn.strdisplaywidth(line:match("^%s*"))) + local lines = {} + for i, text in ipairs(vim.split(d.message, "\n")) do + lines[i] = { { pad .. (i == 1 and "└─ ∴ " or " ") .. text, hl } } + end + local end_col = math.min(d.end_col or #line, #line) + pcall(vim.api.nvim_buf_set_extmark, buf, ns, d.lnum, math.min(d.col, #line), { + end_col = math.max(end_col, math.min(d.col, #line)), + hl_group = not warn and "LitgoProofUnderline" or nil, + sign_text = "∴", + sign_hl_group = hl, + virt_lines = lines, + priority = 220, + strict = false, + }) +end + +--- Install the handler, and keep the built-in ones away from proofs. +function M.setup() + if vim.diagnostic.handlers[M.handler] then + return + end + local function is_lit(buf) + return require("litgo").is_lit(buf) + end + vim.diagnostic.handlers[M.handler] = { + show = function(namespace, buf, diagnostics) + drawn[namespace] = drawn[namespace] or vim.api.nvim_create_namespace("litgo-proof-" .. namespace) + vim.api.nvim_buf_clear_namespace(buf, drawn[namespace], 0, -1) + if not is_lit(buf) then + return + end + for _, d in ipairs(diagnostics) do + if M.is_proof(d) then + draw(buf, drawn[namespace], d) + end + end + end, + hide = function(namespace, buf) + if drawn[namespace] and vim.api.nvim_buf_is_valid(buf) then + vim.api.nvim_buf_clear_namespace(buf, drawn[namespace], 0, -1) + end + end, + } + for _, name in ipairs({ "signs", "underline", "virtual_text", "virtual_lines" }) do + local builtin = vim.diagnostic.handlers[name] + if builtin then + vim.diagnostic.handlers[name] = { + show = function(namespace, buf, diagnostics, opts) + if is_lit(buf) then + diagnostics = vim.tbl_filter(function(d) + return not M.is_proof(d) + end, diagnostics) + end + builtin.show(namespace, buf, diagnostics, opts) + end, + hide = function(namespace, buf) + if builtin.hide then + builtin.hide(namespace, buf) + end + end, + } + end + end +end + +return M diff --git a/lua/litgo/render.lua b/lua/litgo/render.lua index ae2ce3c..880022d 100644 --- a/lua/litgo/render.lua +++ b/lua/litgo/render.lua @@ -43,6 +43,8 @@ local function wanted(item, cfg) return cfg.mermaid elseif item.kind == "table" then return cfg.tables + elseif item.kind == "vego-math" then + return cfg.proofs end return cfg.chunks end @@ -67,7 +69,18 @@ function M.draw(buf) local last = vim.api.nvim_buf_line_count(buf) - 1 for _, it in ipairs(st.items) do - if wanted(it, cfg) then + if it.kind == "vego" then + -- An annotation keeps its colour under the cursor too: nothing is + -- hidden, so there is nothing to reveal. + local key_end = it.col + 3 + if it.name and it.name ~= "" then + local line = vim.api.nvim_buf_get_lines(buf, it.line, it.line + 1, false)[1] or "" + local _, e = line:find(it.name, it.col + 4, true) + key_end = e or key_end + end + mark(buf, it.line, it.col, { end_col = key_end, hl_group = "LitgoProofKeyword", priority = 210 }) + mark(buf, it.line, key_end, { end_col = it.end_col, hl_group = "LitgoProof", priority = 210 }) + elseif wanted(it, cfg) then local under_cursor = cursor >= it.line and cursor <= it.end_line if it.lines then local hl = it.kind == "mermaid" and "LitgoDiagram" or "LitgoMath" @@ -133,7 +146,7 @@ function M.draw(buf) end end else - local hl = ({ math = "LitgoMath", file = "LitgoFile" })[it.kind] or "LitgoChunk" + local hl = ({ math = "LitgoMath", file = "LitgoFile", ["vego-math"] = "LitgoProof" })[it.kind] or "LitgoChunk" if it.info == "undefined" then hl = "LitgoError" end @@ -295,7 +308,7 @@ function M.attach(buf) state[buf] = nil end, }) - vim.diagnostic.config({ virtual_text = true, signs = true, underline = true }, check_ns) + vim.diagnostic.config({ virtual_text = true, signs = true, underline = true, [require("litgo.proof").handler] = true }, check_ns) window_options(buf) M.refresh(buf) end diff --git a/lua/litgo/run.lua b/lua/litgo/run.lua index 6302a7a..876bab3 100644 --- a/lua/litgo/run.lua +++ b/lua/litgo/run.lua @@ -128,10 +128,11 @@ local function publish(diags, origin) for file, list in pairs(by_file) do local buf = vim.fn.bufadd(file) vim.fn.bufload(buf) - -- What the compiler says, the language server has usually said already. + -- What the compiler and the verifier say, the language server has usually + -- said already. if require("litgo.lsp").reporting(buf) then list = vim.tbl_filter(function(d) - return d.source ~= "go build" + return d.source ~= "go build" and d.source ~= "vego" end, list) end vim.diagnostic.set(run_ns, buf, require("litgo.render").to_diagnostics(list, buf)) @@ -155,6 +156,7 @@ function M.attach(buf) signs = true, underline = true, severity_sort = true, + [require("litgo.proof").handler] = true, }, run_ns) -- Several edits can arrive before the scheduled update runs (a macro, a -- :global command), so each works on the result of the one before. @@ -246,6 +248,33 @@ function M.weave(buf, format) end) end +--- Check the //@ annotations without compiling or running anything. +function M.prove(buf) + local bin = litgo().bin() + if not bin then + return + end + save(buf) + M.clear(buf) + vim.system({ bin, "prove", "--json", vim.api.nvim_buf_get_name(buf) }, { text = true }, function(res) + vim.schedule(function() + local ok, decoded = pcall(vim.json.decode, res.stdout or "") + if not ok or type(decoded) ~= "table" then + vim.notify("litgo: prove failed\n" .. (res.stderr or ""), vim.log.levels.ERROR) + return + end + publish(decoded.diagnostics or {}, buf) + vim.g.litgo_last_proof = ("%d/%d"):format(decoded.proved, decoded.functions) + local level = decoded.proved == decoded.functions and vim.log.levels.INFO or vim.log.levels.WARN + if decoded.functions == 0 then + vim.notify("litgo: no //@ annotations, so nothing to prove") + else + vim.notify(("litgo: ∴ proved %d of %d annotated functions (%d obligations)"):format(decoded.proved, decoded.functions, decoded.obligations), level) + end + end) + end) +end + function M.stop() if job then job:kill("sigterm") @@ -289,12 +318,20 @@ function M.run(buf, args) elseif ev.event == "diagnostic" then diags[#diags + 1] = ev local where = ev.file and (ev.file .. ":" .. (ev.line + 1) .. ":" .. (ev.col + 1) .. ": ") or "" - panel_add(where .. ev.message, ev.severity == 1 and "LitgoError" or "DiagnosticWarn") + local hl = ev.severity == 1 and "LitgoError" or "DiagnosticWarn" + if require("litgo.proof").is_proof(ev) then + hl = ev.severity == 1 and "LitgoProofError" or "LitgoProofWarn" + end + panel_add(where .. ev.message, hl) + elseif ev.event == "prove" then + local all = ev.proved == ev.functions + panel_add(("∴ proved %d of %d annotated functions (%d obligations, %d ms)"):format(ev.proved, ev.functions, ev.obligations, ev.ms), all and "LitgoProofKeyword" or "LitgoProofError") elseif ev.event == "build" then if ev.ok then panel_add(("βœ“ built in %d ms"):format(ev.ms), "LitgoOk") else - panel_add("βœ— " .. (ev.stage == "tangle" and "tangle failed" or "build failed"), "LitgoError") + local what = { tangle = "tangle failed", prove = "not proved, so not run" } + panel_add("βœ— " .. (what[ev.stage] or "build failed"), ev.stage == "prove" and "LitgoProofError" or "LitgoError") end elseif ev.event == "output" then panel_add(ev.text, ev.stream == "stderr" and "DiagnosticWarn" or nil) diff --git a/main.go b/main.go index 44d46cf..23f2626 100644 --- a/main.go +++ b/main.go @@ -16,15 +16,18 @@ import ( "github.com/tlehman/litgo/internal/check" "github.com/tlehman/litgo/internal/lit" "github.com/tlehman/litgo/internal/lsp" + "github.com/tlehman/litgo/internal/vego" "github.com/tlehman/litgo/internal/weave" ) const usage = `litgo β€” literate Go litgo tangle [--force] [--stdout] FILE.lit.md... write the Go source - litgo run [--vet] [--timeout 30s] FILE [-- ARGS] tangle, compile and run + litgo run [--vet] [--no-prove] [--timeout 30s] FILE [-- ARGS] + tangle, prove, compile and run litgo weave [--to pdf|html|typ|md] [-o OUT] FILE write a PDF through Typst, or a web page - litgo check FILE.lit.md... chunk, syntax and type errors; writes nothing + litgo check FILE.lit.md... chunk, syntax, type and proof errors; writes nothing + litgo prove [--json] FILE.lit.md... check the //@ annotations: contracts, invariants, termination litgo cat FILE.lit.md print with math and diagrams rendered litgo fmt [-w] FILE.lit.md gofmt the code blocks @@ -39,7 +42,7 @@ Editor protocol (document on stdin, JSON on stdout): Positions in the editor protocol are 0-based; columns are bytes. ` -const VERSION = "0.1.6" +const VERSION = "0.2.1" func main() { if len(os.Args) < 2 { @@ -56,6 +59,8 @@ func main() { os.Exit(cmdWeave(args)) case "check": os.Exit(cmdCheck(args)) + case "prove": + os.Exit(cmdProve(args)) case "cat": os.Exit(cmdCat(args)) case "fmt": @@ -177,6 +182,7 @@ func cmdRun(args []string) int { fs.BoolVar(&o.json, "json", false, "emit one JSON event per line") fs.BoolVar(&o.force, "force", false, "overwrite files litgo did not generate") fs.BoolVar(&o.vet, "vet", false, "also run go vet") + fs.BoolVar(&o.noProve, "no-prove", false, "run even if the //@ annotations are not proved") fs.DurationVar(&o.timeout, "timeout", 0, "kill the program after this long") fs.Parse(args) if fs.NArg() == 0 { @@ -256,6 +262,10 @@ func cmdCheck(args []string) int { if len(diags) == len(res.Diags) { diags = append(diags, check.Types(res, cache)...) } + if len(diags) == len(res.Diags) { + proofs, _ := vego.Prove(res) + diags = append(diags, proofs...) + } if report(diags) > 0 { status = 1 } @@ -263,6 +273,61 @@ func cmdCheck(args []string) int { return status } +func cmdProve(args []string) int { + fs := flag.NewFlagSet("prove", flag.ExitOnError) + asJSON := fs.Bool("json", false, "print the findings and the count as JSON") + fs.Parse(args) + if fs.NArg() == 0 { + fmt.Fprintln(os.Stderr, "litgo prove: no input files") + return 2 + } + status := 0 + for _, path := range fs.Args() { + doc, err := load(path) + if err != nil { + fmt.Fprintln(os.Stderr, "litgo:", err) + status = 1 + continue + } + res := doc.Tangle() + diags := append(res.Diags, res.SyntaxCheck()...) + var sum vego.Summary + if len(diags) == len(res.Diags) && !res.HasErrors() { + var proofs []lit.Diag + proofs, sum = vego.Prove(res) + diags = append(diags, proofs...) + } + failed := false + for _, d := range diags { + failed = failed || d.Severity == lit.SevError + } + if failed { + status = 1 + } + if *asJSON { + if diags == nil { + diags = []lit.Diag{} + } + printJSON(map[string]any{"file": doc.Path, "diagnostics": diags, "functions": sum.Functions, + "proved": sum.Proved, "obligations": sum.Obligations, "names": sum.Names}) + continue + } + report(diags) + switch { + case failed && sum.Functions == 0: + case sum.Functions == 0: + fmt.Fprintf(os.Stderr, "litgo: %s has no //@ annotations, so there is nothing to prove\n", relPath(doc.Path)) + default: + fmt.Fprintf(os.Stderr, "litgo: %s: proved %d of %d annotated functions (%d obligations)\n", + relPath(doc.Path), sum.Proved, sum.Functions, sum.Obligations) + for _, name := range sum.Names { + fmt.Fprintf(os.Stderr, " βœ“ %s\n", name) + } + } + } + return status +} + // cmdCat prints the document the way the editor shows it: math as Unicode, // diagrams drawn, chunk syntax prettified. func cmdCat(args []string) int { @@ -279,8 +344,8 @@ func cmdCat(args []string) int { blocks := map[int]lit.Item{} inline := map[int][]lit.Item{} for _, it := range a.Items { - if it.Kind == "table" { - continue // lining up a table is the editor's business + if it.Kind == "table" || it.Kind == "vego" { + continue // lining up a table and colouring an annotation are the editor's business } else if len(it.Lines) > 0 { blocks[it.Line] = it } else { diff --git a/run.go b/run.go index 3786402..12dcffc 100644 --- a/run.go +++ b/run.go @@ -17,6 +17,7 @@ import ( "time" "github.com/tlehman/litgo/internal/lit" + "github.com/tlehman/litgo/internal/vego" ) // reporter carries the run's progress to a person (plain text) or to an @@ -210,6 +211,7 @@ type runOptions struct { json bool force bool vet bool + noProve bool timeout time.Duration args []string } @@ -244,7 +246,7 @@ func runFile(path string, o runOptions) int { } m := &remapper{dir: dir, outputs: map[string]*mapped{}} - ok := true + ok, unproved := true, false for _, d := range docs { res := d.Tangle() for _, dg := range res.Diags { @@ -269,9 +271,30 @@ func runFile(path string, o runOptions) int { r.emit("tangle", map[string]any{"file": d.Path, "out": fmt.Sprintf("%d files", n)}) r.status("tangled %s β†’ %d files", relPath(d.Path), n) } + if !o.noProve { + start := time.Now() + proofs, sum := vego.Prove(res) + for _, dg := range proofs { + r.diag(dg) + if dg.Severity == lit.SevError { + ok, unproved = false, true + } + } + if sum.Functions > 0 { + r.emit("prove", map[string]any{"file": d.Path, "functions": sum.Functions, "proved": sum.Proved, + "obligations": sum.Obligations, "ms": time.Since(start).Milliseconds()}) + r.status("proved %d of %d annotated functions in %s (%d obligations, %d ms)", sum.Proved, sum.Functions, + relPath(d.Path), sum.Obligations, time.Since(start).Milliseconds()) + } + } } if !ok { - r.emit("build", map[string]any{"ok": false, "ms": 0, "stage": "tangle"}) + stage := "tangle" + if unproved { + stage = "prove" + r.status("not proved, so not run (--no-prove runs it anyway)") + } + r.emit("build", map[string]any{"ok": false, "ms": 0, "stage": stage}) return 1 } // A document that writes several files is a package or a module, and diff --git a/test/plugin.lua b/test/plugin.lua index 19af08b..ce671e7 100644 --- a/test/plugin.lua +++ b/test/plugin.lua @@ -306,5 +306,100 @@ check(shown(5):find("completes |", 1, true) ~= nil, "and the other vim.cmd("bwipeout!") vim.fn.delete(dir, "rf") +-- Proofs --------------------------------------------------------------------------- +local pdir = vim.fn.tempname() +vim.fn.mkdir(pdir, "p") +local pfile = pdir .. "/isqrt.lit.md" +vim.fn.writefile(vim.fn.readfile(root .. "/examples/isqrt.lit.md"), pfile) +vim.cmd.edit(pfile) +local pbuf = vim.api.nvim_get_current_buf() +wait(3000, function() + local st = render.get(pbuf) + return st and st.items ~= nil +end, "analysis of the proved example") +local keywords, formulas = 0, 0 +for _, m in ipairs(vim.api.nvim_buf_get_extmarks(pbuf, ns, 0, -1, { details = true })) do + if m[4].hl_group == "LitgoProofKeyword" then keywords = keywords + 1 end + if m[4].hl_group == "LitgoProof" then formulas = formulas + 1 end +end +check(keywords == 5 and formulas == 5, "the five annotations have a colour of their own (" .. keywords .. ")") +local inv = find(pbuf, "//@ Invariant lo*lo") +local km = vim.api.nvim_buf_get_extmarks(pbuf, ns, { inv - 1, 0 }, { inv - 1, -1 }, { details = true })[1] +check(km and km[3] == 0 and km[4].end_col == #"//@ Invariant", "the keyword is set apart from the formula") + +-- The formula is typeset like the math in the prose, except under the cursor. +local function typeset(row) + local text, hidden + for _, m in ipairs(vim.api.nvim_buf_get_extmarks(pbuf, ns, { row, 0 }, { row, -1 }, { details = true })) do + if m[4].virt_text_pos == "inline" then text = m[4].virt_text[1][1] end + if m[4].conceal then hidden = { m[3], m[4].end_col } end + end + return text, hidden +end +vim.api.nvim_win_set_cursor(0, { 1, 0 }) +render.draw(pbuf) +local text, hidden = typeset(inv - 1) +check(text == "π‘™π‘œΒ² ≀ 𝑛 ∧ 𝑛 < β„Žπ‘–Β²", "a formula is typeset: " .. tostring(text)) +check(hidden and hidden[1] == #"//@ Invariant " and hidden[2] == #"//@ Invariant lo*lo <= n ^ n < hi*hi", "in place of its source, and the keyword stays") +vim.api.nvim_win_set_cursor(0, { inv, 0 }) +render.draw(pbuf) +check(typeset(inv - 1) == nil, "and the source comes back under the cursor") +vim.api.nvim_win_set_cursor(0, { 1, 0 }) + +local function proof_marks() + local out = {} + for name, id in pairs(vim.api.nvim_get_namespaces()) do + if name:find("litgo-proof-", 1, true) then + vim.list_extend(out, vim.api.nvim_buf_get_extmarks(pbuf, id, 0, -1, { details = true })) + end + end + return out +end +local function proofs() + return vim.tbl_filter(require("litgo.proof").is_proof, vim.diagnostic.get(pbuf)) +end +wait(10000, function() return require("litgo.lsp").active(pbuf) end, "the server on the proved example") +check(#proofs() == 0 and #proof_marks() == 0, "a proved document has nothing violet under it") + +-- Break the bisection. It still compiles, so only the verifier can object. +local lo = find(pbuf, "\tlo = mid") +vim.api.nvim_buf_set_lines(pbuf, lo - 1, lo, false, { "\tlo = mid + 1" }) +wait(15000, function() return #proofs() == 2 end, "the verifier's objections") +local failed = proofs() +check(#failed == 2 and failed[1].lnum == inv - 2 and failed[1].message:find("Invariant is not maintained", 1, true) ~= nil, + "a broken invariant is reported at the invariant, as you type") +check(#vim.diagnostic.get(pbuf) == 2, "and the compiler has nothing to add") +local violet = proof_marks() +check(#violet == 2 and violet[1][4].hl_group == "LitgoProofUnderline" and violet[1][4].virt_lines[1][1][2] == "LitgoProofError" + and vim.trim(violet[1][4].sign_text) == "∴", "it is drawn in the proof colour, not the compiler's") +-- Neovim keeps marks of its own to follow a diagnostic through edits; what +-- counts is a mark that draws something. +local builtin = 0 +for name, id in pairs(vim.api.nvim_get_namespaces()) do + if not name:find("litgo-", 1, true) then + for _, m in ipairs(vim.api.nvim_buf_get_extmarks(pbuf, id, 0, -1, { details = true })) do + local d = m[4] + if d.hl_group or d.virt_text or d.virt_lines or d.sign_text then + builtin = builtin + 1 + end + end + end +end +check(builtin == 0, "and the built-in handlers leave it alone") + +-- A run refuses to go further than the proof. +vim.cmd("TangleCompileAndRun") +wait(20000, function() return panel_text():find("not proved, so not run", 1, true) ~= nil end, "the run that is not proved") +check(panel_text():find("proved 0 of 1", 1, true) ~= nil, "a run stops at a failed proof") +check(panel_text():find("isqrt(", 1, true) == nil, "and the program was not run") + +vim.api.nvim_set_current_buf(pbuf) +vim.api.nvim_buf_set_lines(pbuf, lo - 1, lo, false, { "\tlo = mid" }) +wait(15000, function() return #proofs() == 0 and #proof_marks() == 0 end, "the proof to come back") +check(#proofs() == 0, "fixing the code takes the violet away") +vim.cmd("Prove") +wait(10000, function() return vim.g.litgo_last_proof ~= nil end, ":Prove") +check(vim.g.litgo_last_proof == "1/1", ":Prove proves it without running it") + print(failures == 0 and "\nall passed" or ("\n" .. failures .. " failed")) os.exit(failures == 0 and 0 or 1)