diff --git a/go/delivery/forge_inputs_test.go b/go/delivery/forge_inputs_test.go new file mode 100644 index 0000000..edc1bf2 --- /dev/null +++ b/go/delivery/forge_inputs_test.go @@ -0,0 +1,135 @@ +package delivery + +import ( + "strings" + "testing" +) + +const oddManifest = "{\n\t\"module\": \"a\",\n \"z\": 1, \"a\": [ \"x\",\"y\" ],\n \"why\": \"é — & \\n\"\n}\n\n" + +func mergedWithInputs() Forge { + return Forge{Kind: ForgeMerged, Repository: "novox/mesh-sdk", Number: 36, Head: "aaaa", Merge: "cccc", Root: "t0", + Trees: map[string]string{"a": "t1", "b": "t2"}, + Manifests: map[string]string{"a": oddManifest}, + Locks: map[string]map[string]string{"a": {"go.sum": strings.Repeat("ab", 32)}}} +} + +// 1442/AC1 +func TestForgeMergedCarriesManifestsAndLocks(t *testing.T) { + f := mergedWithInputs() + data, err := EncodeForge(f) + if err != nil { + t.Fatal(err) + } + got, err := DecodeForge(data) + if err != nil { + t.Fatal(err) + } + if got.Manifests["a"] != oddManifest || !SameForge(got, withV(f)) { + t.Errorf("the message changed on the way: %q", got.Manifests["a"]) + } +} + +func withV(f Forge) Forge { f.V = Version; return f } + +// 1442/AC2 +func TestForgeMergedRefusesBadInputs(t *testing.T) { + for name, tc := range map[string]struct { + mut func(*Forge) + want string + }{ + "manifest outside trees": {func(f *Forge) { f.Manifests["zz"] = "{}" }, `"zz"`}, + "locks outside trees": {func(f *Forge) { f.Locks["yy"] = map[string]string{"go.sum": "h"} }, `"yy"`}, + "empty manifest": {func(f *Forge) { f.Manifests["b"] = "" }, `"b"`}, + "empty lock hash": {func(f *Forge) { f.Locks["a"]["go.sum"] = "" }, `"a"`}, + } { + f := mergedWithInputs() + tc.mut(&f) + if _, err := EncodeForge(f); err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("%s: %v, want a refusal naming %s", name, err, tc.want) + } + } +} + +// 1442/AC3 +func TestForgePullCarriesManifestsAndLocks(t *testing.T) { + pull := func(action string) Forge { + return Forge{Kind: ForgePull, Repository: "novox/mesh-sdk", Number: 1, Head: "h", Action: action, + Dirs: []string{"a"}, DirsKnown: true, Manifests: map[string]string{"a": oddManifest}, + Locks: map[string]map[string]string{"a": {"go.sum": strings.Repeat("0", 64)}}} + } + for _, action := range []string{PullOpened, PullUpdated} { + data, err := EncodeForge(pull(action)) + if err != nil { + t.Fatalf("%s: %v", action, err) + } + if got, err := DecodeForge(data); err != nil || got.Manifests["a"] != oddManifest { + t.Errorf("%s: %v", action, err) + } + } + f := pull(PullOpened) + f.Manifests["b"] = "{}" + if _, err := EncodeForge(f); err == nil || !strings.Contains(err.Error(), `"b"`) { + t.Errorf("a manifest outside Dirs: %v", err) + } + f = pull(PullOpened) + f.Locks["c"] = map[string]string{"go.sum": "h"} + if _, err := EncodeForge(f); err == nil || !strings.Contains(err.Error(), `"c"`) { + t.Errorf("locks outside Dirs: %v", err) + } + for name, f := range map[string]Forge{ + "refused manifests": {Kind: ForgeRefused, Repository: "a/b", Act: "x", Words: "no", Manifests: map[string]string{"a": "{}"}}, + "refused locks": {Kind: ForgeRefused, Repository: "a/b", Act: "x", Words: "no", Locks: map[string]map[string]string{"a": {"go.sum": "h"}}}, + "open manifests": {Kind: ForgeOpen, Manifests: map[string]string{"a": "{}"}}, + "open locks": {Kind: ForgeOpen, Locks: map[string]map[string]string{"a": {"go.sum": "h"}}}, + } { + if _, err := EncodeForge(f); err == nil { + t.Errorf("%s was encoded", name) + } + } +} + +// 1442/AC4 +func TestForgeFromAnOlderBridge(t *testing.T) { + old := `{"v":1,"kind":"merged","repository":"novox/mesh-sdk","number":36,"head":"aaaa","merge":"cccc","trees":{"a":"t1"},"root":"t0"}` + f, err := DecodeForge([]byte(old)) + if err != nil || f.Manifests != nil || f.Locks != nil { + t.Fatalf("%+v, %v", f, err) + } + data, err := EncodeForge(f) + if err != nil || strings.Contains(string(data), "manifests") || strings.Contains(string(data), "locks") { + t.Errorf("%s, %v", data, err) + } +} + +// 1442/AC5 +func TestModuleKeepsInputs2(t *testing.T) { + kept := KeptInputs2{Tree: "t1", Bases: []string{"src2:" + strings.Repeat("1", 64), "src2:" + strings.Repeat("2", 64)}, + Builder: "src2:" + strings.Repeat("3", 64), Contexts: map[string]string{"web": "t9"}, + Locks: map[string]string{"web": strings.Repeat("cd", 32)}} + m := Module{FP: "src1:x", Manifest: strings.Repeat("ef", 32), Inputs2: &kept, Repository: "novox/m", Commit: "c"} + m.FP = SourcePrefix + "x" + data, err := EncodeModule(m) + if err != nil { + t.Fatal(err) + } + got, err := DecodeModule(data) + if err != nil { + t.Fatal(err) + } + m.V = Version + if !SameModule(got, m) { + t.Errorf("Inputs2 did not come back equal: %+v", got.Inputs2) + } + tc := map[string]Toolchain2{"web": {Language: "typescript", Version: "5.9", Recipe: "2"}} + want, none := SourceFingerprint2(SourceInputs2{Module: "m", Tree: kept.Tree, Bases: kept.Bases, Builder: kept.Builder, + Contexts: kept.Contexts, Locks: kept.Locks, Toolchains: tc}) + have, none2 := SourceFingerprint2(got.Inputs2.With("m", tc)) + if want == "" || none != "" || none2 != "" || have != want { + t.Errorf("fingerprints differ: %q %q (%q %q)", have, want, none, none2) + } + m.Manifest = "" + if _, err := EncodeModule(m); err == nil { + t.Error("Inputs2 without a manifest digest was encoded") + } +} diff --git a/go/delivery/source.go b/go/delivery/source.go index 31c1a9a..580aaa1 100644 --- a/go/delivery/source.go +++ b/go/delivery/source.go @@ -377,15 +377,34 @@ func (c Candidate) check() error { // Manifest, the digest of the module's manifest, the key of the manifests bucket. A value written before has none of // the three, and a reader of today's shape ignores them. type Module struct { - V int `json:"v"` - FP string `json:"fp,omitempty"` - None string `json:"none,omitempty"` - FP2 string `json:"fp2,omitempty"` - None2 string `json:"none2,omitempty"` - Manifest string `json:"manifest,omitempty"` - Src uint64 `json:"src,omitempty"` - Repository string `json:"repository"` - Commit string `json:"commit"` + V int `json:"v"` + FP string `json:"fp,omitempty"` + None string `json:"none,omitempty"` + FP2 string `json:"fp2,omitempty"` + None2 string `json:"none2,omitempty"` + Manifest string `json:"manifest,omitempty"` + // Inputs2 are the new-form inputs the merge gave the module, kept so the indexer computes FP2 again when a build + // module's toolchain versions appear, with no merge (novox/hq ADR 0465 clause 2). It needs Manifest. + Inputs2 *KeptInputs2 `json:"inputs2,omitempty"` + Src uint64 `json:"src,omitempty"` + Repository string `json:"repository"` + Commit string `json:"commit"` +} + +// KeptInputs2 are SourceInputs2 without the module's name (the key has it) and without toolchains (those come from +// the build modules' toolchain keys when the fingerprint is computed): what a Module keeps of a merge. +type KeptInputs2 struct { + Tree string `json:"tree"` + Bases []string `json:"bases,omitempty"` + Builder string `json:"builder,omitempty"` + Contexts map[string]string `json:"contexts,omitempty"` + Locks map[string]string `json:"locks,omitempty"` +} + +// With is the full inputs of module m: the kept ones and the given toolchains. +func (k KeptInputs2) With(module string, toolchains map[string]Toolchain2) SourceInputs2 { + return SourceInputs2{Module: module, Tree: k.Tree, Bases: k.Bases, Builder: k.Builder, Contexts: k.Contexts, + Locks: k.Locks, Toolchains: toolchains} } func (m Module) check() error { @@ -399,6 +418,9 @@ func (m Module) check() error { case m.FP2 != "" && m.None2 != "": return fmt.Errorf("delivery: a module has both fp2 and none2") } + if m.Inputs2 != nil && m.Manifest == "" { + return fmt.Errorf("delivery: a module keeps new-form inputs but has no manifest digest") + } return checkNewForm(m.FP2, m.Manifest) } @@ -421,9 +443,14 @@ type Forge struct { Merge string `json:"merge,omitempty"` Trees map[string]string `json:"trees,omitempty"` Root string `json:"root,omitempty"` - Act string `json:"act,omitempty"` - Words string `json:"words,omitempty"` - Open []OpenPull `json:"open,omitempty"` + // Manifests are the module.json bytes, exact, of each module directory: of the merge's trees on a merged message, + // of Dirs at the head on a pull message. Locks are, of each such directory, its lock files' hex sha256 by path + // relative to it (novox/hq ADR 0465 clause 1). Older writers send neither. + Manifests map[string]string `json:"manifests,omitempty"` + Locks map[string]map[string]string `json:"locks,omitempty"` + Act string `json:"act,omitempty"` + Words string `json:"words,omitempty"` + Open []OpenPull `json:"open,omitempty"` } func (f Forge) check() error { @@ -455,6 +482,13 @@ func (f Forge) check() error { if len(f.Dirs) > 0 && !f.DirsKnown { return fmt.Errorf("delivery: a pull message with module directories must say it knows them (dirs-known)") } + dirs := map[string]bool{} + for _, d := range f.Dirs { + dirs[d] = true + } + if err := f.checkInputs(func(d string) bool { return dirs[d] }, "its module directories"); err != nil { + return err + } case ForgeMerged: if len(f.Dirs) > 0 || f.DirsKnown { return fmt.Errorf("delivery: a merged message carries no module directories") @@ -479,10 +513,16 @@ func (f Forge) check() error { return fmt.Errorf("delivery: a merged message's tree of the directory %q is empty", dir) } } + if err := f.checkInputs(func(d string) bool { _, ok := f.Trees[d]; return ok }, "its trees"); err != nil { + return err + } case ForgeRefused: if len(f.Dirs) > 0 || f.DirsKnown { return fmt.Errorf("delivery: a refused message carries no module directories") } + if len(f.Manifests) > 0 || len(f.Locks) > 0 { + return fmt.Errorf("delivery: a refused message carries no manifests or lock hashes") + } if f.Act == "" { return fmt.Errorf("delivery: a refused message needs the act it answers") } @@ -493,11 +533,35 @@ func (f Forge) check() error { return nil } +// checkInputs refuses a manifest or a lock hash of a directory that known does not know (of is what it knows them +// by), an empty manifest and an empty lock path or hash, naming the directory. +func (f Forge) checkInputs(known func(dir string) bool, of string) error { + for dir, m := range f.Manifests { + if !known(dir) { + return fmt.Errorf("delivery: the manifest of the directory %q is not of %s", dir, of) + } + if m == "" { + return fmt.Errorf("delivery: the manifest of the directory %q is empty", dir) + } + } + for dir, locks := range f.Locks { + if !known(dir) { + return fmt.Errorf("delivery: the lock hashes of the directory %q are not of %s", dir, of) + } + for path, hash := range locks { + if path == "" || hash == "" { + return fmt.Errorf("delivery: the lock %q of the directory %q has an empty path or hash", path, dir) + } + } + } + return nil +} + // checkOpen refuses an open message that names a repository, a branch, a head or anything of the other kinds, or // an open pull request that is not `/` and a number. func (f Forge) checkOpen() error { if f.Repository != "" || f.Branch != "" || f.Number != 0 || f.Head != "" || f.Action != "" || len(f.Dirs) > 0 || f.DirsKnown || - f.Merge != "" || len(f.Trees) > 0 || f.Root != "" || f.Act != "" || f.Words != "" { + f.Merge != "" || len(f.Trees) > 0 || len(f.Manifests) > 0 || len(f.Locks) > 0 || f.Root != "" || f.Act != "" || f.Words != "" { return fmt.Errorf("delivery: an open message carries only the open pull requests it names") } for _, p := range f.Open { @@ -603,7 +667,7 @@ func SameSource(a, b Source) bool { return reflect.DeepEqual(a, b) } func SameCandidate(a, b Candidate) bool { return reflect.DeepEqual(a, b) } // SameModule says whether two modules are the same value. -func SameModule(a, b Module) bool { return a == b } +func SameModule(a, b Module) bool { return reflect.DeepEqual(a, b) } // SameForge says whether two forge messages are the same value. func SameForge(a, b Forge) bool { return reflect.DeepEqual(a, b) } diff --git a/go/delivery/toolchains.go b/go/delivery/toolchains.go new file mode 100644 index 0000000..4fa2361 --- /dev/null +++ b/go/delivery/toolchains.go @@ -0,0 +1,127 @@ +package delivery + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" +) + +// ToolchainsState is the name of the state a build module keeps its toolchains in; its bucket is +// `_toolchains`, built as the SDK names a module's state, by ToolchainsBucket (novox/hq ADR 0465 clause 2). +const ToolchainsState = "toolchains" + +// ToolchainsBucket is the bucket of a build module's toolchains. +func ToolchainsBucket(module string) string { return module + "_" + ToolchainsState } + +// ToolchainNoReport is the reason AgreedToolchain gives when no node has reported a language's toolchain. +const ToolchainNoReport = "toolchain not yet reported" + +func checkToolchainLanguage(language string) error { + switch language { + case "go", "typescript": + return nil + } + return fmt.Errorf("delivery: the toolchain language %q is not go or typescript", language) +} + +// ToolchainKey is the key of a node's toolchain of one language: `.`. It refuses a node that is not a +// name and a language other than go or typescript. +func ToolchainKey(node, language string) (string, error) { + if err := checkName("node", node); err != nil { + return "", err + } + if err := checkToolchainLanguage(language); err != nil { + return "", err + } + return node + "." + language, nil +} + +// ParseToolchainKey is the inverse of ToolchainKey. +func ParseToolchainKey(key string) (node, language string, err error) { + node, language, ok := strings.Cut(key, ".") + if !ok { + return "", "", fmt.Errorf("delivery: %q is not a toolchain key, .", key) + } + if _, err := ToolchainKey(node, language); err != nil { + return "", "", fmt.Errorf("delivery: %q is not a toolchain key: %w", key, err) + } + return node, language, nil +} + +// Toolchain is the value of a toolchains key: the version of the language's toolchain installed on the node and the +// build module's recipe version, and when the module saw them. +type Toolchain struct { + V int `json:"v"` + Language string `json:"language"` + Version string `json:"version"` + Recipe string `json:"recipe"` + At time.Time `json:"at"` +} + +func (t Toolchain) check() error { + if err := checkToolchainLanguage(t.Language); err != nil { + return err + } + if t.Version == "" || t.Recipe == "" { + return fmt.Errorf("delivery: a %s toolchain needs its version and its recipe", t.Language) + } + return nil +} + +// EncodeToolchain is t as JSON at Version; it refuses a language other than go or typescript and an empty version or +// recipe. +func EncodeToolchain(t Toolchain) ([]byte, error) { + if err := t.check(); err != nil { + return nil, err + } + t.V = Version + return json.Marshal(t) +} + +// DecodeToolchain reads a toolchains value; it refuses an unknown version (ErrUnknownVersion) and what EncodeToolchain +// refuses. +func DecodeToolchain(data []byte) (Toolchain, error) { + var t Toolchain + if err := decode(data, &t); err != nil { + return Toolchain{}, err + } + if err := t.check(); err != nil { + return Toolchain{}, err + } + return t, nil +} + +// NodeToolchain is one node's entry of a language, as read from the toolchains bucket. +type NodeToolchain struct { + Node string + Toolchain +} + +// AgreedToolchain is the one Toolchain2 every node's entry of a language agrees on, or none and why: "toolchain not +// yet reported" with no entries, and "builders disagree" naming each differing version and recipe with the nodes +// reporting it when two differ (novox/hq ADR 0405 clause 2). The answer does not depend on the entries' order. The +// entries must all be of one language. +func AgreedToolchain(entries []NodeToolchain) (tc Toolchain2, none string) { + if len(entries) == 0 { + return Toolchain2{}, ToolchainNoReport + } + groups := map[Toolchain2][]string{} + for _, e := range entries { + k := Toolchain2{Language: e.Language, Version: e.Version, Recipe: e.Recipe} + groups[k] = append(groups[k], e.Node) + } + if len(groups) == 1 { + for k := range groups { + return k, "" + } + } + parts := make([]string, 0, len(groups)) + for k, nodes := range groups { + sort.Strings(nodes) + parts = append(parts, fmt.Sprintf("%s recipe %s on %s", k.Version, k.Recipe, strings.Join(nodes, ", "))) + } + sort.Strings(parts) + return Toolchain2{}, "builders disagree: " + strings.Join(parts, "; ") +} diff --git a/go/delivery/toolchains_test.go b/go/delivery/toolchains_test.go new file mode 100644 index 0000000..5b19b13 --- /dev/null +++ b/go/delivery/toolchains_test.go @@ -0,0 +1,90 @@ +package delivery + +import ( + "strings" + "testing" + "time" +) + +func tcEntry(node, lang, version, recipe string) NodeToolchain { + return NodeToolchain{Node: node, Toolchain: Toolchain{Language: lang, Version: version, Recipe: recipe, At: time.Unix(100, 0).UTC()}} +} + +// 1443/AC1 +func TestToolchainKeyAndValue(t *testing.T) { + key, err := ToolchainKey("ace", "go") + if err != nil || key != "ace.go" { + t.Fatalf("%q %v", key, err) + } + node, lang, err := ParseToolchainKey(key) + if err != nil || node != "ace" || lang != "go" { + t.Errorf("%q %q %v", node, lang, err) + } + for _, bad := range []string{"", "ace", "ace.rust", ".go", "a.b.go", "ace."} { + if _, _, err := ParseToolchainKey(bad); err == nil { + t.Errorf("ParseToolchainKey(%q) passed", bad) + } + } + if _, err := ToolchainKey("ace", "python"); err == nil { + t.Error("language python keyed") + } + want := Toolchain{Language: "go", Version: "1.26.1", Recipe: "3", At: time.Unix(100, 0).UTC()} + data, err := EncodeToolchain(want) + if err != nil { + t.Fatal(err) + } + got, err := DecodeToolchain(data) + want.V = Version + if err != nil || got != want { + t.Errorf("%+v %v", got, err) + } + for name, bad := range map[string]Toolchain{ + "python": {Language: "python", Version: "3", Recipe: "1"}, + "no version": {Language: "go", Recipe: "1"}, + "no recipe": {Language: "typescript", Version: "5"}, + } { + if _, err := EncodeToolchain(bad); err == nil { + t.Errorf("%s was encoded", name) + } + if _, err := DecodeToolchain([]byte(`{"v":1,"language":"` + bad.Language + `","version":"` + bad.Version + `","recipe":"` + bad.Recipe + `"}`)); err == nil { + t.Errorf("%s was decoded", name) + } + } + if ToolchainsBucket("go-build") != "go-build_toolchains" { + t.Error(ToolchainsBucket("go-build")) + } +} + +// 1443/AC2 +func TestAgreedToolchain(t *testing.T) { + tc, none := AgreedToolchain([]NodeToolchain{tcEntry("ace", "go", "1.26.1", "3"), tcEntry("novox", "go", "1.26.1", "3")}) + if tc != (Toolchain2{Language: "go", Version: "1.26.1", Recipe: "3"}) || none != "" { + t.Errorf("%+v %q", tc, none) + } +} + +// 1443/AC3 +func TestAgreedToolchainNoReport(t *testing.T) { + tc, none := AgreedToolchain(nil) + if tc != (Toolchain2{}) || none != "toolchain not yet reported" { + t.Errorf("%+v %q", tc, none) + } +} + +// 1443/AC4 +func TestAgreedToolchainDisagree(t *testing.T) { + a, b, c := tcEntry("ace", "go", "1.26.1", "3"), tcEntry("novox", "go", "1.26.2", "3"), tcEntry("g14", "go", "1.26.1", "4") + _, first := AgreedToolchain([]NodeToolchain{a, b, c}) + _, second := AgreedToolchain([]NodeToolchain{c, b, a}) + if first == "" || first != second { + t.Errorf("order changed the answer: %q vs %q", first, second) + } + for _, want := range []string{"builders disagree", "1.26.1", "1.26.2", "ace", "novox", "g14"} { + if !strings.Contains(first, want) { + t.Errorf("%q does not name %q", first, want) + } + } + if tc, _ := AgreedToolchain([]NodeToolchain{a, b}); tc != (Toolchain2{}) { + t.Errorf("a disagreement gave %+v", tc) + } +}