Carry manifests and lock hashes in the forge messages, keep a module's new-form inputs, and lay out the build modules' toolchain keys (design 47 R4c-1, issues 1442, 1443)

The indexer needs a module's manifest, lock hashes and toolchain to compute its new-form fingerprint, and has none of them (ADR 0465 clauses 1, 2, 5): the forge messages now carry the first two, a module keeps its inputs so it can be computed again when a toolchain is reported, and the build modules' toolchain keys have a layout and an agreement rule.

go/delivery/forge_inputs_test.go

@@ -0,0 +1,135 @@
+package delivery
+
+import (
+	"strings"
+	"testing"
+)
+
+const oddManifest = "{\n\t\"module\":   \"a\",\n  \"z\": 1, \"a\": [ \"x\",\"y\" ],\n  \"why\": \"é — <b> & \\n\"\n}\n\n"
+
+func mergedWithInputs() Forge {
+	return Forge{Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 36, Head: "aaaa", Merge: "cccc", Root: "t0",
+		Trees:     map[string]string{"a": "t1", "b": "t2"},
+		Manifests: map[string]string{"a": oddManifest},
+		Locks:     map[string]map[string]string{"a": {"go.sum": strings.Repeat("ab", 32)}}}
+}
+
+// 1442/AC1
+func TestForgeMergedCarriesManifestsAndLocks(t *testing.T) {
+	f := mergedWithInputs()
+	data, err := EncodeForge(f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := DecodeForge(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.Manifests["a"] != oddManifest || !SameForge(got, withV(f)) {
+		t.Errorf("the message changed on the way: %q", got.Manifests["a"])
+	}
+}
+
+func withV(f Forge) Forge { f.V = Version; return f }
+
+// 1442/AC2
+func TestForgeMergedRefusesBadInputs(t *testing.T) {
+	for name, tc := range map[string]struct {
+		mut  func(*Forge)
+		want string
+	}{
+		"manifest outside trees": {func(f *Forge) { f.Manifests["zz"] = "{}" }, `"zz"`},
+		"locks outside trees":    {func(f *Forge) { f.Locks["yy"] = map[string]string{"go.sum": "h"} }, `"yy"`},
+		"empty manifest":         {func(f *Forge) { f.Manifests["b"] = "" }, `"b"`},
+		"empty lock hash":        {func(f *Forge) { f.Locks["a"]["go.sum"] = "" }, `"a"`},
+	} {
+		f := mergedWithInputs()
+		tc.mut(&f)
+		if _, err := EncodeForge(f); err == nil || !strings.Contains(err.Error(), tc.want) {
+			t.Errorf("%s: %v, want a refusal naming %s", name, err, tc.want)
+		}
+	}
+}
+
+// 1442/AC3
+func TestForgePullCarriesManifestsAndLocks(t *testing.T) {
+	pull := func(action string) Forge {
+		return Forge{Kind: ForgePull, Repository: "novox/mesh-sdk", Number: 1, Head: "h", Action: action,
+			Dirs: []string{"a"}, DirsKnown: true, Manifests: map[string]string{"a": oddManifest},
+			Locks: map[string]map[string]string{"a": {"go.sum": strings.Repeat("0", 64)}}}
+	}
+	for _, action := range []string{PullOpened, PullUpdated} {
+		data, err := EncodeForge(pull(action))
+		if err != nil {
+			t.Fatalf("%s: %v", action, err)
+		}
+		if got, err := DecodeForge(data); err != nil || got.Manifests["a"] != oddManifest {
+			t.Errorf("%s: %v", action, err)
+		}
+	}
+	f := pull(PullOpened)
+	f.Manifests["b"] = "{}"
+	if _, err := EncodeForge(f); err == nil || !strings.Contains(err.Error(), `"b"`) {
+		t.Errorf("a manifest outside Dirs: %v", err)
+	}
+	f = pull(PullOpened)
+	f.Locks["c"] = map[string]string{"go.sum": "h"}
+	if _, err := EncodeForge(f); err == nil || !strings.Contains(err.Error(), `"c"`) {
+		t.Errorf("locks outside Dirs: %v", err)
+	}
+	for name, f := range map[string]Forge{
+		"refused manifests": {Kind: ForgeRefused, Repository: "a/b", Act: "x", Words: "no", Manifests: map[string]string{"a": "{}"}},
+		"refused locks":     {Kind: ForgeRefused, Repository: "a/b", Act: "x", Words: "no", Locks: map[string]map[string]string{"a": {"go.sum": "h"}}},
+		"open manifests":    {Kind: ForgeOpen, Manifests: map[string]string{"a": "{}"}},
+		"open locks":        {Kind: ForgeOpen, Locks: map[string]map[string]string{"a": {"go.sum": "h"}}},
+	} {
+		if _, err := EncodeForge(f); err == nil {
+			t.Errorf("%s was encoded", name)
+		}
+	}
+}
+
+// 1442/AC4
+func TestForgeFromAnOlderBridge(t *testing.T) {
+	old := `{"v":1,"kind":"merged","repository":"novox/mesh-sdk","number":36,"head":"aaaa","merge":"cccc","trees":{"a":"t1"},"root":"t0"}`
+	f, err := DecodeForge([]byte(old))
+	if err != nil || f.Manifests != nil || f.Locks != nil {
+		t.Fatalf("%+v, %v", f, err)
+	}
+	data, err := EncodeForge(f)
+	if err != nil || strings.Contains(string(data), "manifests") || strings.Contains(string(data), "locks") {
+		t.Errorf("%s, %v", data, err)
+	}
+}
+
+// 1442/AC5
+func TestModuleKeepsInputs2(t *testing.T) {
+	kept := KeptInputs2{Tree: "t1", Bases: []string{"src2:" + strings.Repeat("1", 64), "src2:" + strings.Repeat("2", 64)},
+		Builder: "src2:" + strings.Repeat("3", 64), Contexts: map[string]string{"web": "t9"},
+		Locks: map[string]string{"web": strings.Repeat("cd", 32)}}
+	m := Module{FP: "src1:x", Manifest: strings.Repeat("ef", 32), Inputs2: &kept, Repository: "novox/m", Commit: "c"}
+	m.FP = SourcePrefix + "x"
+	data, err := EncodeModule(m)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := DecodeModule(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	m.V = Version
+	if !SameModule(got, m) {
+		t.Errorf("Inputs2 did not come back equal: %+v", got.Inputs2)
+	}
+	tc := map[string]Toolchain2{"web": {Language: "typescript", Version: "5.9", Recipe: "2"}}
+	want, none := SourceFingerprint2(SourceInputs2{Module: "m", Tree: kept.Tree, Bases: kept.Bases, Builder: kept.Builder,
+		Contexts: kept.Contexts, Locks: kept.Locks, Toolchains: tc})
+	have, none2 := SourceFingerprint2(got.Inputs2.With("m", tc))
+	if want == "" || none != "" || none2 != "" || have != want {
+		t.Errorf("fingerprints differ: %q %q (%q %q)", have, want, none, none2)
+	}
+	m.Manifest = ""
+	if _, err := EncodeModule(m); err == nil {
+		t.Error("Inputs2 without a manifest digest was encoded")
+	}
+}

go/delivery/source.go

@@ -377,15 +377,34 @@ func (c Candidate) check() error {
 // Manifest, the digest of the module's manifest, the key of the manifests bucket. A value written before has none of
 // the three, and a reader of today's shape ignores them.
 type Module struct {
-	V          int    `json:"v"`
+	V        int    `json:"v"`
-	FP         string `json:"fp,omitempty"`
+	FP       string `json:"fp,omitempty"`
-	None       string `json:"none,omitempty"`
+	None     string `json:"none,omitempty"`
-	FP2        string `json:"fp2,omitempty"`
+	FP2      string `json:"fp2,omitempty"`
-	None2      string `json:"none2,omitempty"`
+	None2    string `json:"none2,omitempty"`
-	Manifest   string `json:"manifest,omitempty"`
+	Manifest string `json:"manifest,omitempty"`
-	Src        uint64 `json:"src,omitempty"`
+	// Inputs2 are the new-form inputs the merge gave the module, kept so the indexer computes FP2 again when a build
-	Repository string `json:"repository"`
+	// module's toolchain versions appear, with no merge (novox/hq ADR 0465 clause 2). It needs Manifest.
-	Commit     string `json:"commit"`
+	Inputs2    *KeptInputs2 `json:"inputs2,omitempty"`
+	Src        uint64       `json:"src,omitempty"`
+	Repository string       `json:"repository"`
+	Commit     string       `json:"commit"`
+}
+
+// KeptInputs2 are SourceInputs2 without the module's name (the key has it) and without toolchains (those come from
+// the build modules' toolchain keys when the fingerprint is computed): what a Module keeps of a merge.
+type KeptInputs2 struct {
+	Tree     string            `json:"tree"`
+	Bases    []string          `json:"bases,omitempty"`
+	Builder  string            `json:"builder,omitempty"`
+	Contexts map[string]string `json:"contexts,omitempty"`
+	Locks    map[string]string `json:"locks,omitempty"`
+}
+
+// With is the full inputs of module m: the kept ones and the given toolchains.
+func (k KeptInputs2) With(module string, toolchains map[string]Toolchain2) SourceInputs2 {
+	return SourceInputs2{Module: module, Tree: k.Tree, Bases: k.Bases, Builder: k.Builder, Contexts: k.Contexts,
+		Locks: k.Locks, Toolchains: toolchains}
 }
 
 func (m Module) check() error {
@@ -399,6 +418,9 @@ func (m Module) check() error {
 	case m.FP2 != "" && m.None2 != "":
 		return fmt.Errorf("delivery: a module has both fp2 and none2")
 	}
+	if m.Inputs2 != nil && m.Manifest == "" {
+		return fmt.Errorf("delivery: a module keeps new-form inputs but has no manifest digest")
+	}
 	return checkNewForm(m.FP2, m.Manifest)
 }
 
@@ -421,9 +443,14 @@ type Forge struct {
 	Merge      string            `json:"merge,omitempty"`
 	Trees      map[string]string `json:"trees,omitempty"`
 	Root       string            `json:"root,omitempty"`
-	Act        string            `json:"act,omitempty"`
+	// Manifests are the module.json bytes, exact, of each module directory: of the merge's trees on a merged message,
-	Words      string            `json:"words,omitempty"`
+	// of Dirs at the head on a pull message. Locks are, of each such directory, its lock files' hex sha256 by path
-	Open       []OpenPull        `json:"open,omitempty"`
+	// relative to it (novox/hq ADR 0465 clause 1). Older writers send neither.
+	Manifests map[string]string            `json:"manifests,omitempty"`
+	Locks     map[string]map[string]string `json:"locks,omitempty"`
+	Act       string                       `json:"act,omitempty"`
+	Words     string                       `json:"words,omitempty"`
+	Open      []OpenPull                   `json:"open,omitempty"`
 }
 
 func (f Forge) check() error {
@@ -455,6 +482,13 @@ func (f Forge) check() error {
 		if len(f.Dirs) > 0 && !f.DirsKnown {
 			return fmt.Errorf("delivery: a pull message with module directories must say it knows them (dirs-known)")
 		}
+		dirs := map[string]bool{}
+		for _, d := range f.Dirs {
+			dirs[d] = true
+		}
+		if err := f.checkInputs(func(d string) bool { return dirs[d] }, "its module directories"); err != nil {
+			return err
+		}
 	case ForgeMerged:
 		if len(f.Dirs) > 0 || f.DirsKnown {
 			return fmt.Errorf("delivery: a merged message carries no module directories")
@@ -479,10 +513,16 @@ func (f Forge) check() error {
 				return fmt.Errorf("delivery: a merged message's tree of the directory %q is empty", dir)
 			}
 		}
+		if err := f.checkInputs(func(d string) bool { _, ok := f.Trees[d]; return ok }, "its trees"); err != nil {
+			return err
+		}
 	case ForgeRefused:
 		if len(f.Dirs) > 0 || f.DirsKnown {
 			return fmt.Errorf("delivery: a refused message carries no module directories")
 		}
+		if len(f.Manifests) > 0 || len(f.Locks) > 0 {
+			return fmt.Errorf("delivery: a refused message carries no manifests or lock hashes")
+		}
 		if f.Act == "" {
 			return fmt.Errorf("delivery: a refused message needs the act it answers")
 		}
@@ -493,11 +533,35 @@ func (f Forge) check() error {
 	return nil
 }
 
+// checkInputs refuses a manifest or a lock hash of a directory that known does not know (of is what it knows them
+// by), an empty manifest and an empty lock path or hash, naming the directory.
+func (f Forge) checkInputs(known func(dir string) bool, of string) error {
+	for dir, m := range f.Manifests {
+		if !known(dir) {
+			return fmt.Errorf("delivery: the manifest of the directory %q is not of %s", dir, of)
+		}
+		if m == "" {
+			return fmt.Errorf("delivery: the manifest of the directory %q is empty", dir)
+		}
+	}
+	for dir, locks := range f.Locks {
+		if !known(dir) {
+			return fmt.Errorf("delivery: the lock hashes of the directory %q are not of %s", dir, of)
+		}
+		for path, hash := range locks {
+			if path == "" || hash == "" {
+				return fmt.Errorf("delivery: the lock %q of the directory %q has an empty path or hash", path, dir)
+			}
+		}
+	}
+	return nil
+}
+
 // checkOpen refuses an open message that names a repository, a branch, a head or anything of the other kinds, or
 // an open pull request that is not `<owner>/<name>` and a number.
 func (f Forge) checkOpen() error {
 	if f.Repository != "" || f.Branch != "" || f.Number != 0 || f.Head != "" || f.Action != "" || len(f.Dirs) > 0 || f.DirsKnown ||
-		f.Merge != "" || len(f.Trees) > 0 || f.Root != "" || f.Act != "" || f.Words != "" {
+		f.Merge != "" || len(f.Trees) > 0 || len(f.Manifests) > 0 || len(f.Locks) > 0 || f.Root != "" || f.Act != "" || f.Words != "" {
 		return fmt.Errorf("delivery: an open message carries only the open pull requests it names")
 	}
 	for _, p := range f.Open {
@@ -603,7 +667,7 @@ func SameSource(a, b Source) bool { return reflect.DeepEqual(a, b) }
 func SameCandidate(a, b Candidate) bool { return reflect.DeepEqual(a, b) }
 
 // SameModule says whether two modules are the same value.
-func SameModule(a, b Module) bool { return a == b }
+func SameModule(a, b Module) bool { return reflect.DeepEqual(a, b) }
 
 // SameForge says whether two forge messages are the same value.
 func SameForge(a, b Forge) bool { return reflect.DeepEqual(a, b) }

go/delivery/toolchains.go

@@ -0,0 +1,127 @@
+package delivery
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+)
+
+// ToolchainsState is the name of the state a build module keeps its toolchains in; its bucket is
+// `<module>_toolchains`, built as the SDK names a module's state, by ToolchainsBucket (novox/hq ADR 0465 clause 2).
+const ToolchainsState = "toolchains"
+
+// ToolchainsBucket is the bucket of a build module's toolchains.
+func ToolchainsBucket(module string) string { return module + "_" + ToolchainsState }
+
+// ToolchainNoReport is the reason AgreedToolchain gives when no node has reported a language's toolchain.
+const ToolchainNoReport = "toolchain not yet reported"
+
+func checkToolchainLanguage(language string) error {
+	switch language {
+	case "go", "typescript":
+		return nil
+	}
+	return fmt.Errorf("delivery: the toolchain language %q is not go or typescript", language)
+}
+
+// ToolchainKey is the key of a node's toolchain of one language: `<node>.<language>`. It refuses a node that is not a
+// name and a language other than go or typescript.
+func ToolchainKey(node, language string) (string, error) {
+	if err := checkName("node", node); err != nil {
+		return "", err
+	}
+	if err := checkToolchainLanguage(language); err != nil {
+		return "", err
+	}
+	return node + "." + language, nil
+}
+
+// ParseToolchainKey is the inverse of ToolchainKey.
+func ParseToolchainKey(key string) (node, language string, err error) {
+	node, language, ok := strings.Cut(key, ".")
+	if !ok {
+		return "", "", fmt.Errorf("delivery: %q is not a toolchain key, <node>.<language>", key)
+	}
+	if _, err := ToolchainKey(node, language); err != nil {
+		return "", "", fmt.Errorf("delivery: %q is not a toolchain key: %w", key, err)
+	}
+	return node, language, nil
+}
+
+// Toolchain is the value of a toolchains key: the version of the language's toolchain installed on the node and the
+// build module's recipe version, and when the module saw them.
+type Toolchain struct {
+	V        int       `json:"v"`
+	Language string    `json:"language"`
+	Version  string    `json:"version"`
+	Recipe   string    `json:"recipe"`
+	At       time.Time `json:"at"`
+}
+
+func (t Toolchain) check() error {
+	if err := checkToolchainLanguage(t.Language); err != nil {
+		return err
+	}
+	if t.Version == "" || t.Recipe == "" {
+		return fmt.Errorf("delivery: a %s toolchain needs its version and its recipe", t.Language)
+	}
+	return nil
+}
+
+// EncodeToolchain is t as JSON at Version; it refuses a language other than go or typescript and an empty version or
+// recipe.
+func EncodeToolchain(t Toolchain) ([]byte, error) {
+	if err := t.check(); err != nil {
+		return nil, err
+	}
+	t.V = Version
+	return json.Marshal(t)
+}
+
+// DecodeToolchain reads a toolchains value; it refuses an unknown version (ErrUnknownVersion) and what EncodeToolchain
+// refuses.
+func DecodeToolchain(data []byte) (Toolchain, error) {
+	var t Toolchain
+	if err := decode(data, &t); err != nil {
+		return Toolchain{}, err
+	}
+	if err := t.check(); err != nil {
+		return Toolchain{}, err
+	}
+	return t, nil
+}
+
+// NodeToolchain is one node's entry of a language, as read from the toolchains bucket.
+type NodeToolchain struct {
+	Node string
+	Toolchain
+}
+
+// AgreedToolchain is the one Toolchain2 every node's entry of a language agrees on, or none and why: "toolchain not
+// yet reported" with no entries, and "builders disagree" naming each differing version and recipe with the nodes
+// reporting it when two differ (novox/hq ADR 0405 clause 2). The answer does not depend on the entries' order. The
+// entries must all be of one language.
+func AgreedToolchain(entries []NodeToolchain) (tc Toolchain2, none string) {
+	if len(entries) == 0 {
+		return Toolchain2{}, ToolchainNoReport
+	}
+	groups := map[Toolchain2][]string{}
+	for _, e := range entries {
+		k := Toolchain2{Language: e.Language, Version: e.Version, Recipe: e.Recipe}
+		groups[k] = append(groups[k], e.Node)
+	}
+	if len(groups) == 1 {
+		for k := range groups {
+			return k, ""
+		}
+	}
+	parts := make([]string, 0, len(groups))
+	for k, nodes := range groups {
+		sort.Strings(nodes)
+		parts = append(parts, fmt.Sprintf("%s recipe %s on %s", k.Version, k.Recipe, strings.Join(nodes, ", ")))
+	}
+	sort.Strings(parts)
+	return Toolchain2{}, "builders disagree: " + strings.Join(parts, "; ")
+}

go/delivery/toolchains_test.go

@@ -0,0 +1,90 @@
+package delivery
+
+import (
+	"strings"
+	"testing"
+	"time"
+)
+
+func tcEntry(node, lang, version, recipe string) NodeToolchain {
+	return NodeToolchain{Node: node, Toolchain: Toolchain{Language: lang, Version: version, Recipe: recipe, At: time.Unix(100, 0).UTC()}}
+}
+
+// 1443/AC1
+func TestToolchainKeyAndValue(t *testing.T) {
+	key, err := ToolchainKey("ace", "go")
+	if err != nil || key != "ace.go" {
+		t.Fatalf("%q %v", key, err)
+	}
+	node, lang, err := ParseToolchainKey(key)
+	if err != nil || node != "ace" || lang != "go" {
+		t.Errorf("%q %q %v", node, lang, err)
+	}
+	for _, bad := range []string{"", "ace", "ace.rust", ".go", "a.b.go", "ace."} {
+		if _, _, err := ParseToolchainKey(bad); err == nil {
+			t.Errorf("ParseToolchainKey(%q) passed", bad)
+		}
+	}
+	if _, err := ToolchainKey("ace", "python"); err == nil {
+		t.Error("language python keyed")
+	}
+	want := Toolchain{Language: "go", Version: "1.26.1", Recipe: "3", At: time.Unix(100, 0).UTC()}
+	data, err := EncodeToolchain(want)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got, err := DecodeToolchain(data)
+	want.V = Version
+	if err != nil || got != want {
+		t.Errorf("%+v %v", got, err)
+	}
+	for name, bad := range map[string]Toolchain{
+		"python":     {Language: "python", Version: "3", Recipe: "1"},
+		"no version": {Language: "go", Recipe: "1"},
+		"no recipe":  {Language: "typescript", Version: "5"},
+	} {
+		if _, err := EncodeToolchain(bad); err == nil {
+			t.Errorf("%s was encoded", name)
+		}
+		if _, err := DecodeToolchain([]byte(`{"v":1,"language":"` + bad.Language + `","version":"` + bad.Version + `","recipe":"` + bad.Recipe + `"}`)); err == nil {
+			t.Errorf("%s was decoded", name)
+		}
+	}
+	if ToolchainsBucket("go-build") != "go-build_toolchains" {
+		t.Error(ToolchainsBucket("go-build"))
+	}
+}
+
+// 1443/AC2
+func TestAgreedToolchain(t *testing.T) {
+	tc, none := AgreedToolchain([]NodeToolchain{tcEntry("ace", "go", "1.26.1", "3"), tcEntry("novox", "go", "1.26.1", "3")})
+	if tc != (Toolchain2{Language: "go", Version: "1.26.1", Recipe: "3"}) || none != "" {
+		t.Errorf("%+v %q", tc, none)
+	}
+}
+
+// 1443/AC3
+func TestAgreedToolchainNoReport(t *testing.T) {
+	tc, none := AgreedToolchain(nil)
+	if tc != (Toolchain2{}) || none != "toolchain not yet reported" {
+		t.Errorf("%+v %q", tc, none)
+	}
+}
+
+// 1443/AC4
+func TestAgreedToolchainDisagree(t *testing.T) {
+	a, b, c := tcEntry("ace", "go", "1.26.1", "3"), tcEntry("novox", "go", "1.26.2", "3"), tcEntry("g14", "go", "1.26.1", "4")
+	_, first := AgreedToolchain([]NodeToolchain{a, b, c})
+	_, second := AgreedToolchain([]NodeToolchain{c, b, a})
+	if first == "" || first != second {
+		t.Errorf("order changed the answer: %q vs %q", first, second)
+	}
+	for _, want := range []string{"builders disagree", "1.26.1", "1.26.2", "ace", "novox", "g14"} {
+		if !strings.Contains(first, want) {
+			t.Errorf("%q does not name %q", first, want)
+		}
+	}
+	if tc, _ := AgreedToolchain([]NodeToolchain{a, b}); tc != (Toolchain2{}) {
+		t.Errorf("a disagreement gave %+v", tc)
+	}
+}