From 127ac64ccefa1feea71959ee9290934e10db1409 Mon Sep 17 00:00:00 2001 From: mesh-tools Date: Tue, 20 Oct 2026 20:48:30 +0000 Subject: [PATCH] Leave an artifact without its lock file out of Locks (issue 1459) and read the build modules' toolchains (issue 1444), part 2 of 2 --- .../mesh-delivery-indexer/integration_test.go | 77 +++++++++++++++++- .../cmd/mesh-delivery-indexer/main.go | 49 ++++++++++++ .../internal/index/inputs.go | 78 +++++++++++++------ .../internal/index/inputs_test.go | 35 ++++++--- .../mesh-delivery-indexer/manifest_test.go | 4 +- modules/mesh-delivery-indexer/module.json | 2 + 6 files changed, 209 insertions(+), 36 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index dc88ca3..3962c90 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -95,7 +95,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10 // startIndexer starts the bus, the stream FORGE with the indexer's durable consumer, the buckets, lets put fill // mesh-controller_builds, and runs the program. mesh-delivery-gate_source is never created. -func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) { +func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue), before ...func(context.Context, jetstream.JetStream)) (jetstream.JetStream, context.Context, jetstream.KeyValue, jetstream.KeyValue) { dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "i") if err != nil { t.Fatal(err) @@ -154,6 +154,9 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue)) ( t.Fatal(err) } put(ctx, builds) + for _, b := range before { + b(ctx, js) + } broker := filepath.Join(t.TempDir(), "broker") cred, _ := json.Marshal(credential{URL: srv.ClientURL(), Node: "n1", Module: "mesh-delivery-indexer", User: "n1.mesh-delivery-indexer", Password: "pw"}) @@ -265,3 +268,75 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4 t.Errorf("manifest = %+v err %v, want avahi and the file's bytes", got, err) } } + +// TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain: the bucket mesh-build-go_toolchains exists (the state +// toolchains of the build module, delivery.ToolchainsBucket) and holds no key; a merged message of mesh-catalog carrying +// avahi's real manifest makes modules.avahi with none2 'toolchain not yet reported'; a put of the key n1.go (the build +// module's own layout, delivery.ToolchainKey and EncodeToolchain) makes, within 5 s and with no merge, none2 "builder's +// fingerprint not yet known". +func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *testing.T) { // 1444/AC10 + manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json") + if err != nil { + t.Fatal(err) + } + const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381" + const inputs = `{"Module":"avahi","Tree":"oldtree","Bases":null,"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}` + js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) { + rec := `{"id":"build-1","repository":"novox/mesh-catalog","ref":"main","module":"avahi","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/avahi","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}` + if _, err := builds.Put(ctx, delivery.BuildKey("avahi", "build-1"), []byte(rec)); err != nil { + t.Fatal(err) + } + if _, err := builds.Put(ctx, delivery.NewestKey("avahi"), []byte("build-1")); err != nil { + t.Fatal(err) + } + }, func(ctx context.Context, js jetstream.JetStream) { + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1}); err != nil { + t.Fatal(err) + } + }) + data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, + Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0", + Trees: map[string]string{"modules/avahi": "newtree"}, + Manifests: map[string]string{"modules/avahi": string(manifest)}, + Locks: map[string]map[string]string{"modules/avahi": {"go.sum": goSum}}}) + if err != nil { + t.Fatal(err) + } + if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.merged", data); err != nil { + t.Fatal(err) + } + none2 := func(want string) string { + var got string + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if e, err := modules.Get(ctx, "avahi"); err == nil { + if m, err := delivery.DecodeModule(e.Value()); err == nil { + if got = m.None2; got == want { + break + } + } + } + } + return got + } + if got := none2(delivery.ToolchainNoReport); got != delivery.ToolchainNoReport { + t.Fatalf("none2 = %q within 5 s, want %q", got, delivery.ToolchainNoReport) + } + key, err := delivery.ToolchainKey("n1", "go") + if err != nil { + t.Fatal(err) + } + value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "1", At: time.Now()}) + if err != nil { + t.Fatal(err) + } + kv, err := js.KeyValue(ctx, delivery.ToolchainsBucket("mesh-build-go")) + if err != nil { + t.Fatal(err) + } + if _, err := kv.Put(ctx, key, value); err != nil { + t.Fatal(err) + } + if got := none2("builder's fingerprint not yet known"); got != "builder's fingerprint not yet known" { + t.Errorf("none2 = %q within 5 s of the toolchain key, want the builder's fingerprint not yet known", got) + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index ca8db63..4abb8e3 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -15,8 +15,10 @@ import ( "fmt" "io" "log" + "maps" "os" "os/signal" + "slices" "strings" "syscall" @@ -296,6 +298,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } go func() { sourceErrs <- follow(ctx, sw, each) }() } + toolErrs, err := watchToolchains(ctx, js, ix, logger) + if err != nil { + return err + } cons, err := js.Consumer(ctx, forgeStream, cred.Node+"_"+cred.Module) if err != nil { return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, forgeStream, err) @@ -313,5 +319,48 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err case err := <-sourceErrs: return err + case err := <-toolErrs: + return err } } + +// watchToolchains reads the toolchains each build module reports (mesh-build-go_toolchains, mesh-build-node_toolchains: the +// module's state toolchains, delivery.ToolchainsBucket; novox/hq ADR 0465 clause 2, issue 1444) and follows them: every +// change of a language's agreed toolchain writes the modules of that language again (index.Indexer.Recompute). A bucket +// that does not exist yet (its build module is not delivered) is said, and not waited for (that needs a timer, ADR 0437 +// ยง8): the indexer is started again with the module, as with the gate's source. The returned channel gets the error of a +// watch that ended. +func watchToolchains(ctx context.Context, js jetstream.JetStream, ix *index.Indexer, logger *log.Logger) (<-chan error, error) { + errs := make(chan error, len(index.ToolchainModules)) + watched := 0 + for _, module := range slices.Sorted(maps.Values(index.ToolchainModules)) { + name := delivery.ToolchainsBucket(module) + kv, err := js.KeyValue(ctx, name) + if errors.Is(err, jetstream.ErrBucketNotFound) { + logger.Printf("the bucket %s does not exist yet: no toolchain of %s is read until the indexer starts again", name, module) + continue + } + if err != nil { + return nil, fmt.Errorf("the bucket %s: %w", name, err) + } + w, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { ix.PutToolchain(module, e.Key(), e.Value(), !opOf(e)) }) + if err != nil { + return nil, err + } + watched++ + go func() { + errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) { + if err := ix.ApplyToolchain(ctx, module, e.Key(), e.Value(), !opOf(e)); err != nil { + logger.Printf("%s.%s could not be taken into the modules: %v", name, e.Key(), err) + } + }) + }() + } + // What the buckets held when the indexer started moves the modules once, not once per key. + if watched > 0 { + if err := ix.Recompute(ctx); err != nil { + logger.Printf("the modules could not be written again from the toolchains: %v", err) + } + } + return errs, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/inputs.go b/modules/mesh-delivery-indexer/internal/index/inputs.go index 52f189b..250b78b 100644 --- a/modules/mesh-delivery-indexer/internal/index/inputs.go +++ b/modules/mesh-delivery-indexer/internal/index/inputs.go @@ -2,6 +2,8 @@ package index import ( "fmt" + "maps" + "slices" "sort" "strings" @@ -46,9 +48,12 @@ type baseFP func(module string) (fp, none string) // // The rules are the controller's builder (mesh-controller internal/builder/builder.go): a bundle or a package of a // language with a toolchain (go, typescript) is the artifact that has one; a package, and a bundle of those languages, -// resolves packages when it is built (wantsPackages, :1017), so it has an entry in Locks, with the hash of its -// language's lock file or an empty hash when none was carried (the fingerprint is then none, "no lock"); an archive, -// an image and an upstream resolve nothing and have no toolchain. A base is a build.on entry that names a module. +// resolves packages when it is built (wantsPackages, :1017) only when its language's lock file exists: it then has +// an entry in Locks with that file's hash, and an artifact whose lock file the merge did not carry has no entry (a +// dependency-free or vendored bundle resolves nothing, as mesh-sdk SourceInputs2 says of an artifact not in Locks; +// novox/hq ADR 0469 clause 5, issue 1459). A bundle that needs a lock and lacks one is refused where it is built, not +// here, so a fingerprint is never given for a build that could not run. An archive, an image and an upstream resolve +// nothing and have no toolchain. A base is a build.on entry that names a module. func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) moduleInputs { m, err := catalogue.ParseManifest([]byte(raw)) if err != nil { @@ -61,37 +66,29 @@ func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) mo return moduleInputs{None2: "the merge names no tree for the module's directory"} } kept := delivery.KeptInputs2{Tree: tree} - langs := map[string]string{} contexts := false var missing []string + langs := languagesOf(m) if m.Build != nil { for _, a := range m.Build.Artifacts { if a.Context != nil { contexts = true } - if a.Kind != catalogue.ArtifactBundle && a.Kind != catalogue.ArtifactPackage { - continue - } - name, ok := lockFile[a.Language] - if !ok { - continue - } - langs[a.Name] = a.Language - if kept.Locks == nil { - kept.Locks = map[string]string{} - } - kept.Locks[a.Name] = locks[name] } - seen := map[string]bool{} - for _, on := range m.Build.On { - if on.Module == "" || seen[on.Module] { - continue + for _, name := range slices.Sorted(maps.Keys(langs)) { + // A lock file the merge did not carry leaves no entry: the artifact resolves nothing (ADR 0469 clause 5). + if h := locks[lockFile[langs[name]]]; h != "" { + if kept.Locks == nil { + kept.Locks = map[string]string{} + } + kept.Locks[name] = h } - seen[on.Module] = true - if fp, none := base(on.Module); fp != "" { + } + for _, on := range basesOf(m) { + if fp, none := base(on); fp != "" { kept.Bases = append(kept.Bases, fp) } else { - missing = append(missing, fmt.Sprintf("the base %s has no fingerprint (%s)", on.Module, none)) + missing = append(missing, fmt.Sprintf("the base %s has no fingerprint (%s)", on, none)) } } } @@ -113,3 +110,38 @@ func inputsOf(module, raw, tree string, locks map[string]string, base baseFP) mo } return out } + +// languagesOf is the language of each artifact of the manifest that has a toolchain (a bundle or a package of go or +// typescript), by artifact name: the artifacts a build module's toolchain enters the fingerprint of. +func languagesOf(m catalogue.Manifest) map[string]string { + langs := map[string]string{} + if m.Build == nil { + return langs + } + for _, a := range m.Build.Artifacts { + if a.Kind != catalogue.ArtifactBundle && a.Kind != catalogue.ArtifactPackage { + continue + } + if _, ok := lockFile[a.Language]; ok { + langs[a.Name] = a.Language + } + } + return langs +} + +// basesOf is the modules the manifest names as bases (its build.on entries that name a module), each once, sorted. +func basesOf(m catalogue.Manifest) []string { + if m.Build == nil { + return nil + } + seen := map[string]bool{} + var out []string + for _, on := range m.Build.On { + if on.Module != "" && !seen[on.Module] { + seen[on.Module] = true + out = append(out, on.Module) + } + } + sort.Strings(out) + return out +} diff --git a/modules/mesh-delivery-indexer/internal/index/inputs_test.go b/modules/mesh-delivery-indexer/internal/index/inputs_test.go index faee51b..b395d1e 100644 --- a/modules/mesh-delivery-indexer/internal/index/inputs_test.go +++ b/modules/mesh-delivery-indexer/internal/index/inputs_test.go @@ -43,25 +43,40 @@ func TestAGoModuleKeepsItsTreeAndTheHashOfItsLockFile(t *testing.T) { // 1450/AC } } -func TestAnArtifactThatResolvesPackagesAndHasNoLockFileHasAnEmptyHash(t *testing.T) { // 1450/AC2 +func TestAnArtifactWhoseLockFileTheMergeDidNotCarryHasNoLocksEntry(t *testing.T) { // 1459/AC1, AC3 + // A Go bundle without a go.sum (a dependency-free or vendored module): no entry, so a fingerprint is possible (ADR 0469 clause 5). in := inputsOf("avahi", catalogueManifest(t, "avahi"), "avahitree", nil, noBases) - if in.Kept == nil || !reflect.DeepEqual(in.Kept.Locks, map[string]string{"tools": ""}) { - t.Fatalf("kept = %+v, want Locks {tools: \"\"}", in.Kept) + if in.Kept == nil || len(in.Kept.Locks) != 0 { + t.Fatalf("kept = %+v, want no Locks entry", in.Kept) } - // The fingerprint of such inputs is none, "no lock" (mesh-sdk SourceFingerprint2). - if fp, none := delivery.SourceFingerprint2(in.Kept.With("avahi", nil)); fp != "" || none != "no lock" { - t.Errorf("fingerprint = %q none %q, want none 'no lock'", fp, none) + if !reflect.DeepEqual(in.Languages, map[string]string{"tools": "go"}) { + t.Errorf("languages = %v, want tools: go (the toolchain still applies)", in.Languages) } - // A TypeScript bundle's lock file is package-lock.json, not go.sum. - ts := inputsOf("audit-logger", catalogueManifest(t, "audit-logger"), "t", map[string]string{"go.sum": "wrong", "package-lock.json": avahiGoSum}, noBases) - if ts.Kept == nil || !reflect.DeepEqual(ts.Kept.Locks, map[string]string{"code": avahiGoSum}) || !reflect.DeepEqual(ts.Languages, map[string]string{"code": "typescript"}) { + // With the toolchain known, mesh-sdk gives such inputs a src2: value, never 'no lock'. + if fp, none := delivery.SourceFingerprint2(in.Kept.With("avahi", map[string]delivery.Toolchain2{"tools": {Language: "go", Version: "1.25.0", Recipe: "1"}})); fp == "" || none != "" { + t.Errorf("fingerprint = %q none %q, want a value", fp, none) + } + // A TypeScript bundle without package-lock.json (go.sum of another language does not count). + ts := inputsOf("audit-logger", catalogueManifest(t, "audit-logger"), "t", map[string]string{"go.sum": "wrong"}, noBases) + if ts.Kept == nil || len(ts.Kept.Locks) != 0 || !reflect.DeepEqual(ts.Languages, map[string]string{"code": "typescript"}) { t.Errorf("typescript: kept %+v languages %v", ts.Kept, ts.Languages) } - // netcheck has a Go and a TypeScript bundle: each takes its own language's lock file. +} + +func TestAnArtifactWhoseLockFileTheMergeCarriedHasItsHash(t *testing.T) { // 1459/AC2, AC3 + ts := inputsOf("audit-logger", catalogueManifest(t, "audit-logger"), "t", map[string]string{"go.sum": "wrong", "package-lock.json": avahiGoSum}, noBases) + if ts.Kept == nil || !reflect.DeepEqual(ts.Kept.Locks, map[string]string{"code": avahiGoSum}) { + t.Errorf("typescript: kept %+v", ts.Kept) + } + // netcheck has a Go and a TypeScript bundle: each takes its own language's lock file, and one without it has none. both := inputsOf("netcheck", catalogueManifest(t, "netcheck"), "t", map[string]string{"go.sum": "g", "package-lock.json": "p"}, noBases) if both.Kept == nil || !reflect.DeepEqual(both.Kept.Locks, map[string]string{"tools-go": "g", "tools-typescript": "p"}) { t.Errorf("netcheck: kept %+v", both.Kept) } + goOnly := inputsOf("netcheck", catalogueManifest(t, "netcheck"), "t", map[string]string{"go.sum": "g"}, noBases) + if goOnly.Kept == nil || !reflect.DeepEqual(goOnly.Kept.Locks, map[string]string{"tools-go": "g"}) { + t.Errorf("netcheck without package-lock.json: kept %+v", goOnly.Kept) + } } // aModuleWithABase is avahi's manifest (mesh-catalog a67f0196) with the one build.on entry that names a module as the diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 84d4dfc..24a52cc 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -71,7 +71,7 @@ func sorted(s []string) []string { return s } -func TestManifestDeclarations(t *testing.T) { // 1238/AC1 +func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build modules' toolchains) m, top := load(t) if m.Module != "mesh-delivery-indexer" { t.Errorf("module = %q", m.Module) @@ -79,7 +79,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1 if want := []state{{Name: "candidates", History: 10}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } - if want := []string{"mesh-controller.builds", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { + if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-controller.builds", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } // "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 814948c..3492dc2 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -11,6 +11,8 @@ { "name": "modules", "history": 20 } ], "reads": [ + "mesh-build-go.toolchains", + "mesh-build-node.toolchains", "mesh-controller.builds", "mesh-delivery-gate.source" ],