//go:build integration package main import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "maps" "os" "path/filepath" "slices" "strings" "sync" "sync/atomic" "testing" "time" "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" // The build runner the build modules pull and run asks with: mesh-build-go's own package (novox/hq ADR 0593 §1), // another repository's, by its published version, so the builder is tested against the real pulling code. "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshbuilds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry" sdkstep "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/buildstep" runner "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/steprun" "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/views/viewstest" "mesh-delivery-builder/internal/build" "mesh-delivery-builder/internal/versions" ) // busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the // controller's user and the builder's own principals . (two instances, for the test of one build). const busConf = ` listen: 127.0.0.1:-1 jetstream { store_dir: %q } accounts { MESH { jetstream: enabled users = [ { user: "ctl", password: "pw" } { user: "n1.mesh-delivery-builder", password: "pw" } { user: "n2.mesh-delivery-builder", password: "pw" } ] } } ` const testAssigner = "acme-assigner" // a holder of the seat mesh-module-assigner no code names const ( commit = "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" fp = "src2:3333333333333333333333333333333333333333333333333333333333333333" ) // fakeBuildModules are the build modules on the nodes as the real ones run since R4g (ADR 0525; issue 1887 A4): each pulls // its build seat's asks on its nodes (runner.Puller over the seat's workers) and runs an ask's job with step bodies that // answer as the real tools do, ending the step on . of its steps bucket. type fakeBuildModules struct { js jetstream.JetStream ctx context.Context failBuild atomic.Bool // refusePush makes every bundle and archive build fail as a push the artifact store refused (ADR 0498 clause 2). refusePush atomic.Bool mu sync.Mutex calls map[string]int // by `.` fetches []map[string]any builds []map[string]any // the arguments of every build call // closed, while set, keeps every node's capacity at 0: no ask is pulled until open is called (a test of the asks // waiting in their streams). closed atomic.Bool pullers []*runner.Puller // hang, while it holds a channel, keeps every build step running until the channel is closed: a step that never ends, // so the builder's step bound runs out (design 51's round-2 case). hang atomic.Pointer[chan struct{}] } // open lets the nodes pull again and wakes their pullers. func (f *fakeBuildModules) open() { f.closed.Store(false) f.mu.Lock() ps := append([]*runner.Puller(nil), f.pullers...) f.mu.Unlock() for _, p := range ps { p.Wake() } } func (f *fakeBuildModules) count(tool string) int { f.mu.Lock() defer f.mu.Unlock() return f.calls[tool] } // digestOf is what every node builds an artifact to. func digestOf(artifact string) string { sum := sha256.Sum256([]byte(artifact)) return "sha256:" + hex.EncodeToString(sum[:]) } var prefixOf = map[string]string{"mesh-build-go": "build_go_", "mesh-build-node": "build_node_", "mesh-build-image": "build_image_"} // step is one tool's answer, as the real module's. func (f *fakeBuildModules) step(module, name string, args map[string]any) (any, error) { tool := prefixOf[module] + name f.mu.Lock() f.calls[module+"."+tool]++ if name == "fetch" { f.fetches = append(f.fetches, args) } else { f.builds = append(f.builds, args) } f.mu.Unlock() if h := f.hang.Load(); h != nil && name == "build" { select { case <-*h: case <-f.ctx.Done(): } } switch { case name == "fetch": merge := "" if args["base"] != nil { // a fetch with a base answers the merge commit (mesh-build-go FetchResult) merge = `,"merge":"` + strings.Repeat("9", 40) + `"` } return json.RawMessage(`{"workspace":"w` + sdkstep.NewID() + `","commit":"` + commit + `"` + merge + `}`), nil case f.failBuild.Load() && tool != "build_image_copy": // the build fails wherever it runs return nil, fmt.Errorf("go build of %v: boom", args["artifact"]) case tool == "build_image_copy": // the upstream copy answers the digest and the registry's reference (ADR 0498 clause 4) d := digestOf(fmt.Sprint(args["artifact"])) return json.RawMessage(`{"artifact":"` + fmt.Sprint(args["artifact"]) + `","digest":"` + d + `","reference":"registry.example/` + fmt.Sprint(args["module"]) + `/` + fmt.Sprint(args["artifact"]) + `@` + d + `","pushed":true}`), nil case tool != "build_image_build" && f.refusePush.Load(): return nil, fmt.Errorf("%s%v: the artifact store answered 507 Insufficient Storage", builds.NotStored, args["artifact"]) case tool != "build_image_build": // a bundle or an archive is pushed and its reference answered (ADR 0498 clause 1) d := digestOf(fmt.Sprint(args["artifact"])) return json.RawMessage(`{"digest":"` + d + `","reference":"artifact-store://` + filepath.Base(fmt.Sprint(args["module"])) + `/` + fmt.Sprint(args["artifact"]) + `/blobs/` + d + `"}`), nil case tool == "build_image_build" && args["publish"] == true: // a trunk image is pushed: the digest and the reference d := digestOf(fmt.Sprint(args["artifact"])) return json.RawMessage(`{"digest":"` + d + `","reference":"registry.example/imaged/server@` + d + `"}`), nil default: return json.RawMessage(`{"digest":"` + digestOf(fmt.Sprint(args["artifact"])) + `"}`), nil } } // serve makes module's steps bucket and its build seat's stream and workers, and pulls its asks on nodes. func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) { kv, err := f.js.CreateKeyValue(f.ctx, jetstream.KeyValueConfig{Bucket: buildrun.StepsBucket(module), History: 1}) if err != nil { t.Fatal(err) } language := strings.TrimPrefix(module, "mesh-build-") if err := runner.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil { t.Fatal(err) } // Design 47 G §544, §600: the build modules' helper declares each queue's two workers, main's and the named asks' // (accept..), and the puller takes from both; the bed declares none of its own, since a worker filtering // both forms overlaps the named worker, which a work queue refuses. put := func(key string, value any) (uint64, error) { raw, _ := value.(json.RawMessage) return kv.Put(f.ctx, key, raw) } call := func(name string) runner.StepFunc { return func(ctx context.Context, args map[string]any) (any, error) { return f.step(module, name, args) } } steps := runner.AskSteps{Fetch: call("fetch"), Build: call("build"), Copy: call("copy")} for _, node := range nodes { p := &runner.Puller{Source: &runner.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: node, Put: put, Capacity: func() int { if f.closed.Load() { return 0 } return 2 }, PullWait: 200 * time.Millisecond, Run: func(ctx context.Context, a sdkstep.StepAsk) (any, error) { return runner.RunAsk(ctx, steps, a) }} f.mu.Lock() f.pullers = append(f.pullers, p) f.mu.Unlock() go func() { _ = p.Pull(f.ctx) }() } } // trunkHead is a trunk head of novox/mesh-catalog. const trunkHead = "3333333333333333333333333333333333333333" type bed struct { ctx context.Context js jetstream.JetStream catalog *catalogHolder // the seat mesh-catalog, as its holder answers the branch follower source jetstream.KeyValue builds jetstream.KeyValue heard jetstream.KeyValue // the state toolchains-heard instances jetstream.KeyValue out *saidLog // the builder's process log, every instance's stop func() // stops the builder started last fake *fakeBuildModules srvURL string dir string } // startBed starts the bus, the buckets the controller makes from the builder's and the build modules' manifests, the // build modules' streams and the builder's durable consumers on them, two nodes' toolchains, the indexer's manifests, and // the fake build modules. It runs no builder. func startBed(t *testing.T) *bed { return startBedWith(t, false) } // startBedWith is startBed with the nodes' capacity 0 from the start when closed: a puller that reads its capacity before a // test could close the nodes may be in a pull already and takes the first ask put (issue 1928). func startBedWith(t *testing.T, closed bool) *bed { dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "b") if err != nil { t.Fatal(err) } t.Cleanup(func() { os.RemoveAll(dir) }) conf := filepath.Join(dir, "bus.conf") if err := os.WriteFile(conf, []byte(fmt.Sprintf(busConf, filepath.Join(dir, "js"))), 0o600); err != nil { t.Fatal(err) } opts, err := server.ProcessConfigFile(conf) if err != nil { t.Fatal(err) } opts.NoLog, opts.NoSigs = true, true srv, err := server.NewServer(opts) if err != nil { t.Fatal(err) } go srv.Start() if !srv.ReadyForConnections(10 * time.Second) { t.Fatal("the bus did not come up") } t.Cleanup(srv.Shutdown) ctl, err := nats.Connect(srv.ClientURL(), nats.UserInfo("ctl", "pw")) if err != nil { t.Fatal(err) } t.Cleanup(ctl.Close) js, err := jetstream.New(ctl) if err != nil { t.Fatal(err) } ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) t.Cleanup(cancel) b := &bed{ctx: ctx, js: js, srvURL: srv.ClientURL(), dir: dir} // EVENTS, where the build seats say toolchain-reported (design 47 G §492), with each builder's durable consumer of it, and // the builder's state toolchains-heard (§604). if _, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{Name: eventsStream, Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil { t.Fatal(err) } for _, node := range []string{"n1", "n2"} { if _, err := js.CreateOrUpdateConsumer(ctx, eventsStream, jetstream.ConsumerConfig{Durable: node + "_mesh-delivery-builder", AckPolicy: jetstream.AckExplicitPolicy}); err != nil { t.Fatal(err) } } if kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, buildrun.ToolchainsHeardState), History: 1}); err != nil { t.Fatal(err) } else { b.heard = kv } mk := func(name string, history int) jetstream.KeyValue { kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)}) if err != nil { t.Fatal(err) } return kv } // The seat mesh-catalog's holder and the assigner as the owners of what the builder follows: the catalog half through the seat's // verbs and events (design 47 G §348, §614), the assignments by their read tool and views family (ADR 0615, DLV-M1); n1 and n2 // each have their durable consumer. b.catalog = newCatalogHolder(t, ctx, js, ctl) b.catalog.buildSeats(t, ctl, map[string]string{"node-build-go": testGo, "node-build-node": testNode, "node-build-image": testImage, "git": "mesh-delivery-forge-bridge", "mesh-catalog": "mesh-delivery-indexer", // the floor's sources are the holders of git and mesh-catalog "mesh-builds": builds.BuilderModule}) // the builder's own seat, whose verb reserved it answers once its definition is read (A-115, A-278) readers := []string{"n1_" + builds.BuilderModule, "n2_" + builds.BuilderModule} // Design 47 G §717, A-298: the catalog half names testAssigner the claimant of mesh-module-assigner, and its seat-holders view // names it the seat's holder, so the builder follows the assignments at it; nothing answers the seat's verb assignments. b.catalog.put(t, meshcatalog.CatalogEntry{Module: testAssigner, Repository: "novox/mesh-catalog", Commit: "c0", Fp: "src2:" + strings.Repeat("a", 64), ManifestDigest: "sha256:" + strings.Repeat("b", 64), Manifest: map[string]any{"module": testAssigner, "claims": []any{map[string]any{"name": "mesh-module-assigner"}}}}) seatHolders := viewstest.Serve(t, ctx, js, ctl, testAssigner, "seat-holders", testAssigner+".seat-holders", readers...) if _, err := seatHolders.Put(ctx, "mesh-module-assigner", []byte(`{"seat":"mesh-module-assigner","holders":[{"module":"`+testAssigner+`","node":"n1"}]}`)); err != nil { t.Fatal(err) } viewstest.Serve(t, ctx, js, ctl, testAssigner, "assignments", testAssigner+".assignments", readers...) b.builds = mk(builds.BuildsBucket(builds.BuilderModule), builds.BuilderBuildsHistory) b.instances = mk(builds.StateBucket(builds.BuilderModule, instancesState), 1) // The state bounds, whose keys have each its own time to live: an ask's step bounds (design 47 G A-77, A-118) are kept in it. if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: builds.StateBucket(builds.BuilderModule, versions.BoundsState), History: 1, LimitMarkerTTL: time.Second}); err != nil { t.Fatal(err) } b.out = &saidLog{} // The builder's module log, as the runtime raises it (ADR 0297): its stream on the log's subjects. logStream := buscompose.LogStreamName(builds.BuilderModule, builds.BuilderLog) if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: logStream, Subjects: []string{buscompose.LogSubject(builds.BuilderModule, builds.BuilderLog) + ".>"}, AllowDirect: true}); err != nil { t.Fatal(err) } // The builder's own stream of its findings' events (design 47 G A-30, §383), which its finding writer says into. if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "BUILDER_FINDINGS", Subjects: []string{"mesh.stream.mesh-delivery-builder.event.finding-written.*", "mesh.stream.mesh-delivery-builder.event.finding-cleared.*", "mesh.stream.mesh-delivery-builder.event.findings-stated.*"}}); err != nil { t.Fatal(err) } // The builder's stream of build ends (design 47 G A-96), which it says build.ended into. if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "BUILDER_BUILDS", Subjects: []string{"mesh.stream.mesh-delivery-builder.event.build.ended.>"}}); err != nil { t.Fatal(err) } b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}} b.fake.closed.Store(closed) b.fake.serve(t, "mesh-build-go", "n1", "n2") b.fake.serve(t, "mesh-build-node", "n1", "n2") b.fake.serve(t, "mesh-build-image", "n1") // the image build module runs on n1 only b.report(t, "mesh-build-go", "n1", "go", "go1.27.1", "1") b.report(t, "mesh-build-go", "n2", "go", "go1.27.1", "1") b.report(t, "mesh-build-node", "n1", "typescript", "tsc5", "1") b.report(t, "mesh-build-node", "n2", "typescript", "tsc5", "1") return b } // logged is every entry of module's key in the builder's log, read through process.Conn.ReadLog on the controller's user. func (b *bed) logged(t *testing.T, module string) []builds.BuildLogEntry { t.Helper() conn, err := process.Connect(process.Account{URL: b.srvURL, Module: "mesh-delivery-view", User: "ctl", Password: "pw"}) if err != nil { t.Fatal(err) } defer conn.Close() entries, err := conn.ReadLog(builds.BuilderModule, builds.BuilderLog).Since(time.Unix(0, 0), module) if err != nil { t.Fatal(err) } var out []builds.BuildLogEntry for _, le := range entries { e, err := builds.DecodeBuildLogEntry(le.Value) if err != nil { t.Fatal(err) } out = append(out, e) } return out } // report says a node's toolchain of a language on the build module's seat, as the seat's holder does (design 47 G §492). func (b *bed) report(t *testing.T, module, node, language, version, recipe string) { t.Helper() seat := map[string]string{"mesh-build-go": "node-build-go", "mesh-build-node": "node-build-node"}[module] data, err := json.Marshal(map[string]string{"node": node, "language": language, "version": version, "recipe": recipe, "digest": "sha256:" + strings.Repeat("a", 64)}) if err != nil { t.Fatal(err) } if _, err := b.js.Publish(b.ctx, "mesh.seat."+seat+".event.toolchain-reported."+node, data); err != nil { t.Fatal(err) } } // module writes modules. as the indexer does for a module with a new-form fingerprint, and its manifest by digest. func (b *bed) module(t *testing.T, name, fixture, path string) string { return b.moduleWith(t, name, fixture, path, bothToolchains) } // bothToolchains are the toolchains the bed's nodes report, as a module's fingerprint names them. var bothToolchains = map[string]builds.ToolchainUsed{"go": {Version: "go1.27", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}} // moduleWith is module with the toolchains its fingerprint was computed with. func (b *bed) moduleWith(t *testing.T, name, fixture, path string, toolchains map[string]builds.ToolchainUsed) string { return b.moduleCommit(t, name, fixture, path, toolchains, commit) } func (b *bed) moduleCommit(t *testing.T, name, fixture, path string, toolchains map[string]builds.ToolchainUsed, at string) string { t.Helper() return b.moduleFP(t, name, fixture, path, toolchains, at, fp, 0) } // moduleFP is moduleCommit with the new-form fingerprint fp2 and the source revision src of its merge (0: none). func (b *bed) moduleFP(t *testing.T, name, fixture, path string, toolchains map[string]builds.ToolchainUsed, at, fp2 string, src uint64) string { t.Helper() raw, err := os.ReadFile(filepath.Join("testdata", fixture)) if err != nil { t.Fatal(err) } digest := meshcatalogclient.ManifestDigest(raw) var manifest map[string]any if err := json.Unmarshal(raw, &manifest); err != nil { t.Fatal(err) } b.catalog.put(t, b.entry(name, path, at, fp2, digest, manifest, toolchains, src)) return digest } // manifestOf is a fixture manifest as the entry holds it. func (b *bed) manifestOf(t *testing.T, fixture string) map[string]any { t.Helper() raw, err := os.ReadFile(filepath.Join("testdata", fixture)) if err != nil { t.Fatal(err) } var m map[string]any if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) } return m } // entry is the catalog graph entry of a module as the holder answers it. func (b *bed) entry(name, path, at, fp2, digest string, manifest map[string]any, toolchains map[string]builds.ToolchainUsed, src uint64) meshcatalog.CatalogEntry { return meshcatalog.CatalogEntry{Module: name, Repository: "novox/mesh-catalog", Commit: at, Fp: fp2, Path: path, ManifestDigest: digest, Manifest: manifest, Inputs: &meshcatalog.EntryInputs{Toolchains: build.CatalogToolchains(toolchains), Src: int64(src)}} } // archive writes modules. as the real indexer does for a module of an archive only: its fingerprint names no // toolchain (mesh-delivery-indexer internal/index inputs.go languagesOf gives an archive no language), so Toolchains is empty. func (b *bed) archive(t *testing.T, name, path, at string) string { t.Helper() return b.moduleCommit(t, name, "catalogue-a67f0196-ca-trust.module.json", path, nil, at) } // start runs the program as the builder of a node. func (b *bed) start(t *testing.T, node string, env map[string]string) { t.Helper() broker := filepath.Join(t.TempDir(), "broker") cred, _ := json.Marshal(process.Account{URL: b.srvURL, Node: node, Module: "mesh-delivery-builder", User: node + ".mesh-delivery-builder", Password: "pw"}) if err := os.WriteFile(broker, cred, 0o600); err != nil { t.Fatal(err) } e := map[string]string{"MESH_BROKER_FILE": broker} for k, v := range env { e[k] = v } runCtx, stop := context.WithCancel(b.ctx) done := make(chan error, 1) go func() { done <- run(runCtx, func(k string) string { return e[k] }, b.out) }() var once sync.Once b.stop = func() { once.Do(func() { stop(); <-done }) } t.Cleanup(b.stop) } // saidLog is the builder's process log as its instances write it. type saidLog struct { mu sync.Mutex buf strings.Builder } func (l *saidLog) String() string { l.mu.Lock() defer l.mu.Unlock() return l.buf.String() } func (l *saidLog) Write(p []byte) (int, error) { l.mu.Lock() defer l.mu.Unlock() return l.buf.Write(p) } // awaitSaid reads the process log until it holds want, within 20 s, and answers the line. func (b *bed) awaitSaid(t *testing.T, want string) string { t.Helper() deadline := time.Now().Add(20 * time.Second) for { b.out.mu.Lock() all := b.out.buf.String() b.out.mu.Unlock() for _, line := range strings.Split(all, "\n") { if strings.Contains(line, want) { return line } } if time.Now().After(deadline) { t.Fatalf("the builder never said %q; it said:\n%s", want, all) } select { case <-b.ctx.Done(): t.Fatal(b.ctx.Err()) case <-time.After(50 * time.Millisecond): } } } // await reads the build key until pred holds of it, within 20 s. func (b *bed) await(t *testing.T, module string, pred func(builds.BuilderBuild) bool) builds.BuilderBuild { t.Helper() return b.awaitFP(t, module, fp, pred) } // awaitFP is await of the build key of fingerprint at. func (b *bed) awaitFP(t *testing.T, module, at string, pred func(builds.BuilderBuild) bool) builds.BuilderBuild { t.Helper() key, err := builds.BuilderBuildKey(module, at) if err != nil { t.Fatal(err) } var last builds.BuilderBuild for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { e, err := b.builds.Get(b.ctx, key) if err != nil { continue } if last, err = builds.DecodeBuilderBuild(e.Value()); err == nil && pred(last) { return last } } t.Fatalf("builds.%s = %+v within 20 s", key, last) return last } func isState(state string) func(builds.BuilderBuild) bool { return func(v builds.BuilderBuild) bool { return v.State == state } } // TestBuilderMakesADoneKeyWithTheDigests: modules.netcheck (a Go and a TypeScript bundle) and its manifest are in the // indexer's buckets, and two nodes report each build module's toolchain; the program writes builds.netcheck. done, // the digests of both bundles, both build modules' toolchains, after a fetch and a build on one node (ADR 0478 clause 4). func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 b := startBed(t) // DLV-B10 (design 51, round 2): the builder's versions, findings and the floor's three sources are on the bus; the seat // answers 0.1.4, so netcheck's bundles are numbered from 0.1.5. Two merges close together, the first replaced before // its claim: the second is numbered at once, and the first never. reg := b.registry(t, "n1", "0.1.4") const netcheck = "catalogue-a67f0196-netcheck.module.json" first := "src2:" + strings.Repeat("1", 64) b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, first, 1) digest := b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, fp, 2) b.start(t, "n1", map[string]string{"MESH_BUILDER_STEP_BOUND": "1s"}) v := b.await(t, "netcheck", isState(builds.BuildDone)) for _, a := range []string{"tools-go", "tools-typescript"} { es := reg.of(t, "netcheck/"+a) if len(es) != 1 || es[0].Version != "0.1.5" || es[0].Fingerprint != fp || es[0].State != versions.Numbered || es[0].Digest != digestOf(a) { t.Errorf("versions of netcheck/%s = %+v, want the second merge numbered 0.1.5 at once with its digest, the first never", a, es) } } if v.By != "n1.mesh-delivery-builder" || v.Manifest != digest || len(v.Artifacts) != 2 { t.Fatalf("build = %+v", v) } for _, a := range v.Artifacts { if a.Digest != digestOf(a.Name) || a.Kind != "bundle" || a.Reference != "artifact-store://netcheck/"+a.Name+"/blobs/"+a.Digest { t.Errorf("artifact = %+v, want the digest every node built and the reference its push answered (ADR 0498 clause 1)", a) } } if v.Toolchains["go"].Version != "go1.27" || v.Toolchains["typescript"].Version != "tsc5" { t.Errorf("toolchains = %+v", v.Toolchains) } // A-115: the builder's process answers the seat mesh-builds' verb reserved over the bus; the reservation of a module it never // built is answered in words (here that the bed's state reserved is not on the bus), never left unanswered. if conn, err := process.Connect(process.Account{URL: b.srvURL, Module: "mesh-delivery-view", User: "ctl", Password: "pw"}); err != nil { t.Fatal(err) } else { defer conn.Close() if _, err := conn.Call("seat:mesh-builds.reserved", meshbuilds.ReservedInput{Module: "never-built"}); err == nil || !strings.Contains(err.Error(), "_reserved") { t.Errorf("reserved of a module never built = %v, want the process's answer naming its state reserved", err) } } // ADR 0618 §1: the done key names the reference forms it is written in. if len(v.Needs) != 1 || v.Needs[0] != "form:artifact-store" { t.Errorf("needs = %v, want the first form", v.Needs) } // 1955/L7 (ADR 0571 §1): the build's start and end are in the module log build-log under the module's key, read as a // reader with the log's read reads them, the end naming the manifest and what was made. // DLV-B10: between them the build log holds the build's narrations (build.said), each version numbered, and why. var ended []builds.BuildLogEntry for _, e := range b.logged(t, "netcheck") { if e.Event != builds.BuildSaidEvent { ended = append(ended, e) } else if e.Why != versions.SaidNumbered || !strings.Contains(e.Line, "numbered 0.1.5") || e.FP != fp { t.Errorf("said = %+v, want netcheck's versions numbered", e) } } if len(ended) != 2 || ended[0].Event != builds.BuildStartedEvent || ended[1].Event != builds.BuildEndedEvent || ended[1].Manifest != digest || len(ended[1].Made) != 2 || ended[1].Node != "n1" { t.Errorf("logged = %+v, want the start and the end of netcheck's build", ended) } // Design 47 G §379, A-96: the end is also said as the event build.ended on the builder's stream, its body the module, the // fingerprint and each artifact's digest. if st, err := b.js.Stream(b.ctx, "BUILDER_BUILDS"); err != nil { t.Fatal(err) } else if raw, err := st.GetLastMsgForSubject(b.ctx, "mesh.stream.mesh-delivery-builder.event."+builds.BuildEndedEventName("netcheck", fp)); err != nil { t.Errorf("no build.ended of netcheck's build: %v", err) } else { var ev builds.BuildEnded if err := json.Unmarshal(raw.Data, &ev); err != nil || ev.Module != "netcheck" || ev.Fingerprint != fp || ev.Outcome != builds.BuildDone || len(ev.Made) != 2 { t.Errorf("build.ended = %s (%v), want netcheck at its fingerprint with two artifacts' digests", raw.Data, err) } } for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 1, "mesh-build-go.build_go_build": 1, "mesh-build-node.build_node_fetch": 1, "mesh-build-node.build_node_build": 1} { if got := b.fake.count(tool); got != want { t.Errorf("%s called %d times, want %d (on one node, ADR 0478 clause 4)", tool, got, want) } } // Design 32, design 47 G §348: a package's name the catalog half's entry of the module gives later reaches the builder by the // branch follower's view (never the indexer's state), and looks at the module again; the done bundles are kept, not numbered twice. named := b.entry("netcheck", "modules/netcheck", commit, fp, digest, nil, bothToolchains, 0) named.Manifest = b.manifestOf(t, "catalogue-a67f0196-netcheck.module.json") named.Artifacts = []meshcatalog.Artifact{{Ref: "netcheck:lib", Language: "go", Package: &meshcatalog.ArtifactPackage{Ecosystem: "go", Name: "netcheck"}}} b.catalog.put(t, named) b.awaitSaid(t, "names a package of netcheck") // Design 47 G §349: version.numbered is no longer said, for a number or for a late name. if got := reg.numberedOf("netcheck/tools-go"); len(got) != 0 { t.Errorf("version.numbered of netcheck/tools-go said %v, want none", got) } for _, a := range []string{"tools-go", "tools-typescript"} { reg.published(t, "netcheck/"+a, "0.1.5", digestOf(a)) } reg.awaitOf(t, "netcheck/tools-go", func(es []versions.Entry) bool { return len(es) == 1 && es[0].State == versions.Published }) // A numbered version a holder will refuse below, written to the state as the builder numbers one. refusable, _ := json.Marshal(versions.Entry{Name: "netcheck/tools-go", Kind: versions.KindBundle, Series: "0.1", Version: "0.1.6", Module: "netcheck", Fingerprint: "src2:" + strings.Repeat("6", 64), Seat: ociartifactregistry.Seat, State: versions.Numbered, At: time.Now().UTC()}) if _, err := reg.versions.Put(b.ctx, versions.Key("netcheck/tools-go", "0.1.6"), refusable); err != nil { t.Fatal(err) } // Running-testing note (review round 1): two claims never get one number on the bus's own compare-and-set, as // the builder's state versions keeps them: eight claims of one name at once, eight versions. store := versions.Versions{Store: awaitedVersions{b: process.AwaitBucket(b.ctx, b.js, nil, builds.StateBucket(builds.BuilderModule, versions.State), "", t.Logf)}} numbers := make(chan string, 8) var claims sync.WaitGroup for i := range 8 { claims.Go(func() { d, _, err := store.Number(b.ctx, versions.Claim{Name: "netcheck/race", Kind: versions.KindBundle, Fingerprint: fmt.Sprintf("f%d", i)}) if err != nil { t.Error(err) } numbers <- d.Version }) } claims.Wait() close(numbers) got := map[string]bool{} for v := range numbers { got[v] = true } if len(got) != 8 { t.Errorf("eight claims of one name at once were numbered %v, want eight versions", slices.Sorted(maps.Keys(got))) } if got := reg.numberedOf("netcheck/tools-go"); len(got) != 0 { t.Errorf("version.numbered of netcheck/tools-go said %v, want none", got) } // DLV-B10: a build module's bound running out fails the trunk build, which is the builder's finding build-failed; the // fingerprint built again keeps its number, said with why, and its done build clears the finding. release := make(chan struct{}) b.fake.hang.Store(&release) third := "src2:" + strings.Repeat("4", 64) b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, commit, third, 3) f := b.awaitFP(t, "netcheck", third, isState(builds.BuildFailed)) if !strings.Contains(f.Error, "did not end within") { t.Errorf("the failed build = %+v, want its bound run out", f) } // A-162: the failed end is said as build.ended too, with its error whole. if st, err := b.js.Stream(b.ctx, "BUILDER_BUILDS"); err != nil { t.Fatal(err) } else if raw, err := st.GetLastMsgForSubject(b.ctx, "mesh.stream.mesh-delivery-builder.event."+builds.BuildEndedEventName("netcheck", third)); err != nil { t.Errorf("no build.ended of netcheck's failed build: %v", err) } else if ev := (builds.BuildEnded{}); json.Unmarshal(raw.Data, &ev) != nil || ev.Outcome != builds.BuildFailed || ev.Error != f.Error { t.Errorf("build.ended = %s, want the failed outcome with the error %q", raw.Data, f.Error) } failed := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "netcheck", BuildFailed) reg.awaitFinding(t, failed, true) b.fake.hang.Store(nil) close(release) b.moduleFP(t, "netcheck", netcheck, "modules/netcheck", bothToolchains, "89abcdef0123456789abcdef0123456789abcdef", third, 3) // the entry moves to a later commit, same fingerprint b.awaitFP(t, "netcheck", third, isState(builds.BuildDone)) b.awaitSaid(t, "kept: fingerprint "+third+" is numbered under it") if !slices.ContainsFunc(b.logged(t, "netcheck"), func(e builds.BuildLogEntry) bool { return e.Event == builds.BuildSaidEvent && e.Why == versions.SaidKept && e.FP == third && strings.Contains(e.Line, "kept: fingerprint "+third) }) { t.Errorf("the build log of netcheck says no version kept at %s", third) } reg.awaitFinding(t, failed, false) // DLV-B4: the seats' refused and removed, heard on the builder's durable consumer, move a numbered version: a retention // removal turns the entry removed with its reason, a refusal refused with the holder's reason and the bytes it refused. reg.outcome(t, ociartifactregistry.SubjectRemoved, versions.Outcome{Name: "netcheck/tools-typescript", Version: "0.1.5", Kind: "bundle", Holder: "distribution", Reason: "retention"}) reg.outcome(t, ociartifactregistry.SubjectRefused, versions.Outcome{Name: "netcheck/tools-go", Version: "0.1.6", Digest: digestOf("other"), Kind: "bundle", Holder: "distribution", Reason: "the cap"}) reg.awaitOf(t, "netcheck/tools-typescript", func(es []versions.Entry) bool { return slices.ContainsFunc(es, func(e versions.Entry) bool { return e.Version == "0.1.5" && e.State == versions.Removed && e.Reason == "retention" }) }) reg.awaitOf(t, "netcheck/tools-go", func(es []versions.Entry) bool { return slices.ContainsFunc(es, func(e versions.Entry) bool { return e.Version == "0.1.6" && e.State == versions.Refused && e.Reason == "the cap" && e.RefusedDigest == digestOf("other") }) }) } // TestAnArchiveOnlyModuleAndARootModuleAreBuilt: the indexer gives an archive no language, so modules.ca-trust has no // toolchains; the builder builds it on the build module's nodes all the same (review round 4). A module whose path is "." // is given to the build tools as it is (1615/AC4). Two builders on two nodes start together on the archive's key: the // second takes over the key the first holds building (ADR 0478 clause 6), and the key ends done, written once (1607/AC7). // One table: three tests were merged, to pay for the enrolment's and to bring mesh-delivery's count down (DLV-B8). func TestAnArchiveOnlyModuleAndARootModuleAreBuilt(t *testing.T) { for _, c := range []struct { name, path string two bool }{{"no toolchain", "modules/ca-trust", false}, {"root", meshcatalogclient.RootPath, false}, {"two instances together", "modules/ca-trust", true}} { t.Run(c.name, func(t *testing.T) { b := startBed(t) b.archive(t, "ca-trust", c.path, commit) b.start(t, "n1", nil) if c.two { b.start(t, "n2", nil) } v := b.await(t, "ca-trust", isState(builds.BuildDone)) if c.name == "no toolchain" && (len(v.Toolchains) != 0 || len(v.Artifacts) != 1 || strings.HasPrefix(v.Step, "one node (")) { t.Errorf("build = %+v, want done naming no toolchain", v) } if c.two && (len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "archive" || v.Artifacts[0].Digest != digestOf("files")) { t.Errorf("build = %+v, want the archive's digest", v) } if n := b.fake.count("mesh-build-go.build_go_build"); c.two && n < 1 { // each instance that built, on one node t.Errorf("build_go_build was called %d times, want one for each instance that built", n) } }) } } // TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint: a build step fails, so the key is // failed naming it; the indexer writing modules. again with the same fingerprint wakes the builder, which builds // the key again as a new revision of the same key, now done. func TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint(t *testing.T) { // 1607/AC6, AC7 b := startBed(t) b.fake.failBuild.Store(true) b.archive(t, "ca-trust", "modules/ca-trust", commit) b.start(t, "n1", nil) v := b.await(t, "ca-trust", isState(builds.BuildFailed)) if !strings.Contains(v.Error, " on n") || !strings.Contains(v.Error, "boom") { t.Errorf("build = %+v, want failed naming the node and the step's reason", v) } key, _ := builds.BuilderBuildKey("ca-trust", fp) failed, err := b.builds.Get(b.ctx, key) if err != nil { t.Fatal(err) } b.fake.failBuild.Store(false) // The indexer writes the module again (here at a later commit), with the same fingerprint. b.archive(t, "ca-trust", "modules/ca-trust", "89abcdef0123456789abcdef0123456789abcdef") b.await(t, "ca-trust", isState(builds.BuildDone)) done, err := b.builds.Get(b.ctx, key) if err != nil { t.Fatal(err) } if done.Revision() <= failed.Revision() { t.Errorf("revision %d, was %d: the key was not written as a new revision", done.Revision(), failed.Revision()) } } // TestAToolchainChangeDoesNotBuildTheOldFingerprint: the fingerprint names go1.27.1, the nodes then agree on go1.28.0 (a // report of both nodes), and the builder writes no key for the old fingerprint although it was woken by the report; a // module value naming go1.27.2 (the new fingerprint, computed by the indexer) is built. func TestAToolchainChangeDoesNotBuildTheOldFingerprint(t *testing.T) { // 1615/AC4 b := startBed(t) // The nodes' word is go1.27.2 when the module comes: the bed's go1.27.1 reports are not left before them on EVENTS, where the // builder's new durable would hear them first and agree on go1.27.1 for a moment, building the old fingerprint if its entry // is heard then (the cause of this test's failures, twice on 2026-11-12). st, err := b.js.Stream(b.ctx, eventsStream) if err != nil { t.Fatal(err) } if err := st.Purge(b.ctx, jetstream.WithPurgeSubject("mesh.seat.node-build-go.event.toolchain-reported.>")); err != nil { t.Fatal(err) } b.report(t, "mesh-build-go", "n1", "go", "go1.28.0", "1") b.report(t, "mesh-build-go", "n2", "go", "go1.28.0", "1") b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck") // names go1.27 b.start(t, "n1", nil) // Another module, whose fingerprint names the new toolchain, is built; by then the old one has been looked at. b.moduleCommit(t, "netcheck2", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck", map[string]builds.ToolchainUsed{"go": {Version: "go1.28", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}, commit) b.await(t, "netcheck2", isState(builds.BuildDone)) // The builder never agreed on go1.27, not for a moment: that is what would build the old fingerprint. info, err := st.Info(b.ctx) if err != nil { t.Fatal(err) } for seq := info.State.FirstSeq; seq <= info.State.LastSeq; seq++ { m, err := st.GetMsg(b.ctx, seq) if errors.Is(err, jetstream.ErrMsgNotFound) { continue } if err != nil { t.Fatal(err) } if strings.HasSuffix(m.Subject, ".toolchain-agreed") && strings.Contains(string(m.Data), `"version":"go1.27"`) { t.Errorf("the builder said the nodes agree on go1.27: %s", m.Data) } } if _, err := b.builds.Get(b.ctx, mustKey(t, "netcheck")); err == nil { t.Error("the old fingerprint, naming go1.27, was built with go1.28.0; the log: " + b.out.String()) } if n := b.fake.count("mesh-build-go.build_go_build"); n != 1 { t.Errorf("build_go_build was called %d times, want 1 (the new fingerprint on one node only)", n) } } func mustKey(t *testing.T, module string) string { k, err := builds.BuilderBuildKey(module, fp) if err != nil { t.Fatal(err) } return k } // moduleAt is module with the go toolchain version the fingerprint names and the commit it was computed at. func (b *bed) moduleAt(t *testing.T, name, fixture, path, goVersion, at string) { t.Helper() b.moduleCommit(t, name, fixture, path, map[string]builds.ToolchainUsed{"go": {Version: goVersion, Recipe: "1"}}, at) } // TestAKeyLeftBuildingForAModuleThatIsGoneIsClosedAtStart: the builds bucket holds a key left building by another node for // a module the indexer no longer has; the program, at start, fails it naming why (issue 1660, ADR 0478 clause 6). func TestAKeyLeftBuildingForAModuleThatIsGoneIsClosedAtStart(t *testing.T) { b := startBed(t) data, err := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildBuilding, By: "n2.mesh-delivery-builder", Manifest: strings.Repeat("a", 64)}) if err != nil { t.Fatal(err) } if _, err := b.builds.Put(b.ctx, mustKey(t, "ghost"), data); err != nil { t.Fatal(err) } b.start(t, "n1", nil) v := b.await(t, "ghost", isState(builds.BuildFailed)) if !strings.Contains(v.Error, "no longer has this fingerprint") || !strings.Contains(v.Error, "n2.mesh-delivery-builder") { t.Errorf("build = %+v, want it closed naming who left it", v) } } // TestAnImageIsBuiltOnOneNodeAndEveryFingerprintOfTheCatalogHalfPublishes: the catalog half's entry of imaged (one image // artifact) is built on n1, where mesh-build-image runs: its key is done with the digest and the reference, its build call carrying // publish; the entry's next fingerprint is built again with publish and ends with a reference of its own (ADR 0487 clauses 5 and // 12; a run's candidate is no input of the builder: design 47 G A-84). func TestAnImageIsBuiltOnOneNodeAndEveryFingerprintOfTheCatalogHalfPublishes(t *testing.T) { // 1705 b := startBed(t) other := "src2:" + strings.Repeat("e", 64) digest := b.moduleFP(t, "imaged", "catalogue-a67f0196-imaged.module.json", "modules/imaged", nil, commit, other, 0) b.start(t, "n1", nil) okey, _ := builds.BuilderBuildKey("imaged", other) awaitKey := func(key string, pred func(builds.BuilderBuild) bool) builds.BuilderBuild { var last builds.BuilderBuild for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { if e, err := b.builds.Get(b.ctx, key); err == nil { if last, err = builds.DecodeBuilderBuild(e.Value()); err == nil && pred(last) { return last } } } t.Fatalf("builds.%s = %+v within 20 s; the log: %s", key, last, b.out.String()) return last } v := awaitKey(okey, isState(builds.BuildDone)) if len(v.Artifacts) != 1 || v.Artifacts[0].Digest != digestOf("server") || !strings.HasSuffix(v.Artifacts[0].Reference, "@"+digestOf("server")) { t.Fatalf("build = %+v, want the digest and the reference", v) } // The module's next fingerprint: built once more, published. if got := b.moduleFP(t, "imaged", "catalogue-a67f0196-imaged.module.json", "modules/imaged", nil, commit, fp, 0); got != digest { t.Fatalf("manifest digest = %s, want %s", got, digest) } fkey, _ := builds.BuilderBuildKey("imaged", fp) v = awaitKey(fkey, func(v builds.BuilderBuild) bool { return v.State == builds.BuildDone && len(v.Artifacts) == 1 && v.Artifacts[0].Reference != "" }) if v.Artifacts[0].Digest != digestOf("server") { t.Errorf("second build = %+v", v) } b.fake.mu.Lock() defer b.fake.mu.Unlock() published := 0 for _, a := range b.fake.builds { if a["publish"] == true { published++ } } if len(b.fake.builds) != 2 || published != 2 { t.Errorf("builds = %v, want two, both with publish", b.fake.builds) } } // TestEveryDoneKeyNamesItsReferencesAndARefusedPushIsBuiltAgain: with the artifact store refusing pushes, the key of an // archive module ends failed opening "not stored:"; once it takes them, a new modules value builds the key again, done, its // every artifact naming a reference (issue 1587, B8, ADR 0498 clauses 1, 2 and 8). func TestEveryDoneKeyNamesItsReferencesAndARefusedPushIsBuiltAgain(t *testing.T) { b := startBed(t) b.fake.refusePush.Store(true) b.archive(t, "ca-trust", "modules/ca-trust", commit) b.start(t, "n1", nil) v := b.await(t, "ca-trust", isState(builds.BuildFailed)) if !strings.HasPrefix(v.Error, builds.NotStored) || !strings.Contains(v.Error, "507") { t.Fatalf("build = %+v, want failed opening %q with the store's words", v, builds.NotStored) } b.fake.refusePush.Store(false) b.archive(t, "ca-trust", "modules/ca-trust", "89abcdef0123456789abcdef0123456789abcdef") v = b.await(t, "ca-trust", isState(builds.BuildDone)) if ok, missing := v.Referenced(); !ok { t.Errorf("build = %+v, artifacts naming no reference: %v", v, missing) } } // TestAPinnedUpstreamIsCopiedByTheImageBuildModule: hello-web's manifest (captured from mesh-catalog 49caf17f) names an // upstream image pinned by its digest and an archive; the upstream is copied on n1, where mesh-build-image runs, with no // fetch of the image module, and the key ends done with both references (issue 1587, B8, ADR 0498 clause 4). func TestAPinnedUpstreamIsCopiedByTheImageBuildModule(t *testing.T) { b := startBed(t) b.moduleCommit(t, "hello-web", "catalogue-49caf17f-hello-web.module.json", "modules/hello-web", nil, commit) b.start(t, "n1", nil) v := b.await(t, "hello-web", isState(builds.BuildDone)) if ok, missing := v.Referenced(); !ok { t.Fatalf("build = %+v, artifacts naming no reference: %v", v, missing) } if n := b.fake.count("mesh-build-image.build_image_copy"); n != 1 { t.Errorf("build_image_copy called %d times, want once", n) } if n := b.fake.count("mesh-build-image.build_image_fetch"); n != 0 { t.Errorf("build_image_fetch called %d times, want none for a copy", n) } } // TestAManifestFallingBackToAPredefinedKeyIsBuilt: claude-code's manifest (mesh-catalog 835bf22b) has a config value falling // back to a predefined key; the builder, which holds no published list of them, reads it and builds the key done (issue // 1751, mesh-sdk catalogue.ParseManifestForInputs). func TestAManifestFallingBackToAPredefinedKeyIsBuilt(t *testing.T) { b := startBed(t) b.module(t, "claude-code", "catalogue-835bf22b-claude-code.module.json", "modules/claude-code") b.start(t, "n1", nil) v := b.await(t, "claude-code", func(v builds.BuilderBuild) bool { return v.State == builds.BuildDone || v.State == builds.BuildFailed }) if v.State != builds.BuildDone { t.Errorf("build = %+v, want done: the manifest is read", v) } } // 1886/AC5: a builder on n1 whose Go, Node and image build modules each answered a reference says under n1, in its bucket // of instances, that it writes references, and takes the say away when it stops: the controller delivers from its keys // only while it holds. func TestABuilderWhoseBuildModulesAnsweredReferencesSaysItAndStoppedSaysNothing(t *testing.T) { b := startBed(t) b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck") // a Go and a TypeScript bundle b.moduleCommit(t, "imaged", "catalogue-a67f0196-imaged.module.json", "modules/imaged", nil, commit) // an image, for the trunk b.start(t, "n1", nil) var said string for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && said == ""; time.Sleep(50 * time.Millisecond) { if e, err := b.instances.Get(b.ctx, "n1"); err == nil { said = string(e.Value()) } } if !strings.Contains(said, `"writes":["references"]`) { t.Fatalf("the builder said %q", said) } b.stop() if _, err := b.instances.Get(b.ctx, "n1"); !errors.Is(err, jetstream.ErrKeyNotFound) { t.Fatalf("the stopped builder still says: %v", err) } } // 1887/AC1, AC2 on the real bus: the builder puts the trunk fingerprint's build as asks on the build seats' delivery // streams and calls no node's tool; the build modules that pulled them wrote the steps, and the key is done naming the // references. func TestTheBuilderAsksTheBuildSeatsAndCallsNoNode(t *testing.T) { b := startBed(t) b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck") sub, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw")) if err != nil { t.Fatal(err) } defer sub.Close() var calls atomic.Int64 // A build module's tool, never the owners' read tools the builder's first views ask (DLV-M1). s, err := sub.Subscribe("mesh.mod.*.call.>", func(m *nats.Msg) { if strings.HasPrefix(m.Subject, "mesh.mod.mesh-build-") { calls.Add(1) } }) if err != nil { t.Fatal(err) } defer s.Unsubscribe() asked := b.asksPut(t, sub) b.start(t, "n1", nil) v := b.await(t, "netcheck", isState(builds.BuildDone)) for _, a := range v.Artifacts { if a.Reference == "" { t.Errorf("artifact %s names no reference", a.Name) } } if n := calls.Load(); n != 0 { t.Errorf("%d tool calls reached a build module", n) } // Design 47 G §717, A-298: the assignments are followed at the holder the seat-holders view names, no seat verb asked for it. for deadline := time.Now().Add(10 * time.Second); !strings.Contains(b.out.String(), "holds "+testAssigner+"'s assignments, the holder of the seat mesh-module-assigner"); time.Sleep(10 * time.Millisecond) { if time.Now().After(deadline) { t.Fatalf("the builder did not follow the assignments at the seat's holder:\n%s", b.out.String()) } } // 1887/AC1: one ask per build module the build needs (netcheck: a Go and a TypeScript bundle), each on its seat's // delivery stream. got := asked() if len(got) != 2 { t.Fatalf("asks put = %+v, want one Go and one Node delivery ask", got) } for _, a := range got { if a.Kind != sdkstep.AskDelivery || a.Build == nil || a.Build.Fp != fp { t.Errorf("ask %+v, want a delivery ask of the fingerprint", a) } } } // asksPut hears every ask put on a build seat's accept subjects (a core subscription beside the stream) and answers the // asks heard so far. func (b *bed) asksPut(t *testing.T, nc *nats.Conn) func() []sdkstep.StepAsk { t.Helper() var mu sync.Mutex var asks []sdkstep.StepAsk s, err := nc.Subscribe("mesh.seat.*.accept.>", func(m *nats.Msg) { if a, err := sdkstep.DecodeStepAsk(m.Data); err == nil { mu.Lock() asks = append(asks, a) mu.Unlock() } }) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = s.Unsubscribe() }) return func() []sdkstep.StepAsk { mu.Lock() defer mu.Unlock() return append([]sdkstep.StepAsk(nil), asks...) } } // 1887/AC3: a builder that started again while its asks waited in the streams (no node had a free slot) puts no second // ask: it waits for the asks its building key names, and the key is done once a node took them. func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) { b := startBedWith(t, true) // 1928/AC1: no node can take an ask from the start, so no pull in hand takes the first one b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck") sub, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw")) if err != nil { t.Fatal(err) } defer sub.Close() asked := b.asksPut(t, sub) b.start(t, "n1", nil) v := b.await(t, "netcheck", func(v builds.BuilderBuild) bool { return v.State == builds.BuildBuilding && len(v.Asks) == 2 }) for deadline := time.Now().Add(20 * time.Second); len(asked()) < 2; time.Sleep(20 * time.Millisecond) { if time.Now().After(deadline) { t.Fatalf("asks put = %+v within 20 s, want the two the key names", asked()) } } // 1945 note 62, on the real bus: the stream read as builder_seat_queue reads it shows the Go ask waiting; the ask can // still be answered (nothing was taken since); and a message dropped as an ask asked again is gone from the stream. held := heldOnTheBus{js: b.js, seats: testSeats()} goAsk := sdkstep.StepAsk{ID: v.Asks[sdkstep.AskGo], Language: sdkstep.AskGo, Kind: sdkstep.AskDelivery} stale := sdkstep.StepAsk{ID: "mvo53jcm70b5f7", Language: sdkstep.AskGo, Kind: sdkstep.AskDelivery} subject, _ := sdkstep.AskSubjectBy(sdkstep.AskGo, sdkstep.AskDelivery, meshbuilds.Seat) if _, err := b.js.Publish(b.ctx, subject, []byte("{}"), jetstream.WithMsgID(stale.ID)); err != nil { t.Fatal(err) } queued := func() []build.QueuedState { q, err := held.Queue(b.ctx, sdkstep.AskGo, 10) if err != nil || len(q.Kinds) != 1 || q.Kinds[0].Kind != sdkstep.AskDelivery { t.Fatalf("the Go seat's queue = %+v, %v; want its delivery asks", q, err) } return q.Kinds[0].Messages } if m := queued(); len(m) != 2 || m[0].Ask != goAsk.ID || !m[0].Waits || m[1].Ask != stale.ID { t.Fatalf("the Go delivery queue = %+v, want the key's ask waiting, then the stale one", m) } if why, err := held.Gone(b.ctx, goAsk); why != "" || err != nil { t.Errorf("Gone(the waiting ask) = %q, %v; want it answerable", why, err) } if err := held.Drop(b.ctx, stale); err != nil { t.Fatal(err) } if m := queued(); len(m) != 1 || m[0].Ask != goAsk.ID { t.Fatalf("after the drop the Go delivery queue = %+v, want the key's ask alone", m) } b.stop() b.start(t, "n1", nil) // the same builder, started again, while both asks wait b.fake.open() done := b.await(t, "netcheck", isState(builds.BuildDone)) if got := asked(); len(got) != 2 { t.Fatalf("asks put = %d (%+v), want the two the key named and none again", len(got), got) } for lang, id := range v.Asks { found := false for _, a := range asked() { found = found || a.ID == id } if !found { t.Errorf("the %s ask %s the key named was never put", lang, id) } } if len(done.Artifacts) != 2 { t.Errorf("done = %+v", done) } } // registry is the part of the bed design 51's builder cases need: the builder's states versions, findings, floor-waits and // reasks, EVENTS with the seat oci-artifact-registry's outcomes and the builder's durable consumer of it, the builder's // own events kept, and the floor's three sources served on the bus as their holders answer through the tool runner // ({result}): the seat's versions, the seat git's tags and the indexer's indexer_requirements. type registry struct { b *bed versions, findings, reasks jetstream.KeyValue mu sync.Mutex numbered []string // the bodies said on the subject version.numbered was said on before design 47 G §349 } func (b *bed) registry(t *testing.T, node string, seatVersions ...string) *registry { t.Helper() r := ®istry{b: b} mk := func(name string) jetstream.KeyValue { kv, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: name, History: 1}) if err != nil { t.Fatal(err) } return kv } r.versions, r.findings, r.reasks = mk(builds.StateBucket(builds.BuilderModule, versions.State)), mk(builds.StateBucket(builds.BuilderModule, mcseat.StateFindings)), mk(builds.StateBucket(builds.BuilderModule, build.ReAsksState)) mk(builds.StateBucket(builds.BuilderModule, versions.FloorWaitsState)) // EVENTS with two seats' outcomes, and the forge bridge's stream (a branch's end): each with the builder's durable consumer. for stream, subjects := range map[string][]string{ eventsStream: {"mesh.mod.*.event.>", "mesh.seat.*.event.>"}, } { if _, err := b.js.CreateOrUpdateStream(b.ctx, jetstream.StreamConfig{Name: stream, Subjects: subjects}); err != nil { t.Fatal(err) } if _, err := b.js.CreateOrUpdateConsumer(b.ctx, stream, jetstream.ConsumerConfig{Durable: node + "_" + builds.BuilderModule, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { t.Fatal(err) } } nc := b.js.Conn() // Nothing says version.numbered any more (§349): its old subject is watched, to say so if something does. sub, err := nc.Subscribe("mesh.mod."+builds.BuilderModule+".event.version.numbered", func(m *nats.Msg) { r.mu.Lock() r.numbered = append(r.numbered, string(m.Data)) r.mu.Unlock() }) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = sub.Unsubscribe() }) var listed []map[string]string for _, v := range seatVersions { listed = append(listed, map[string]string{"version": v}) } for subject, answer := range map[string]any{ "mesh.seat." + ociartifactregistry.Seat + ".call.versions.>": map[string]any{"versions": listed}, "mesh.seat.git.call.tags.>": map[string]any{"tags": []string{}}, } { reply, _ := json.Marshal(map[string]any{"result": answer, "node": "ctl"}) s, err := nc.Subscribe(subject, func(m *nats.Msg) { _ = m.Respond(reply) }) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = s.Unsubscribe() }) } return r } // of is every entry versions holds of name. func (r *registry) of(t *testing.T, name string) []versions.Entry { t.Helper() l, err := r.versions.ListKeysFiltered(r.b.ctx, versions.NameFilter(name)) if err != nil { t.Fatal(err) } defer func() { _ = l.Stop() }() var out []versions.Entry for k := range l.Keys() { e, err := r.versions.Get(r.b.ctx, k) if err != nil { t.Fatal(err) } var v versions.Entry if err := json.Unmarshal(e.Value(), &v); err != nil { t.Fatal(err) } out = append(out, v) } return out } // awaitOf reads name's entries until pred holds of them, within 20 s. func (r *registry) awaitOf(t *testing.T, name string, pred func([]versions.Entry) bool) []versions.Entry { t.Helper() var es []versions.Entry for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { if es = r.of(t, name); pred(es) { return es } } t.Fatalf("versions of %s = %+v within 20 s", name, es) return es } // published says the artifact seat's published for name@version, as its holder does. func (r *registry) published(t *testing.T, name, version, digest string) { t.Helper() r.outcome(t, ociartifactregistry.SubjectPublished, versions.Outcome{Name: name, Version: version, Digest: digest, Kind: "bundle", Holder: "distribution"}) } // outcome says a seat's outcome on subject, as its holder does. func (r *registry) outcome(t *testing.T, subject string, o versions.Outcome) { t.Helper() data, _ := json.Marshal(o) msg := nats.NewMsg(subject) msg.Header.Set("x-event-id", subject+"-"+o.Name+"-"+o.Version) msg.Data = data if _, err := r.b.js.PublishMsg(r.b.ctx, msg); err != nil { t.Fatal(err) } } // answered writes a reasks value and reads it until the builder answered it, within 20 s. func (r *registry) answered(t *testing.T, key string, value []byte) build.ReAsk { t.Helper() if _, err := r.reasks.Put(r.b.ctx, key, value); err != nil { t.Fatal(err) } var a build.ReAsk for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { if e, err := r.reasks.Get(r.b.ctx, key); err == nil && json.Unmarshal(e.Value(), &a) == nil && !a.Answered.IsZero() { return a } } t.Fatalf("reasks.%s = %+v, not answered within 20 s", key, a) return a } // awaitFinding reads the builder's finding key until it is there (raised) or gone, within 20 s. func (r *registry) awaitFinding(t *testing.T, key string, raised bool) { t.Helper() for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { if _, err := r.findings.Get(r.b.ctx, key); (err == nil) == raised { return } } t.Fatalf("the finding %s raised = %v within 20 s, want %v", key, !raised, raised) } // numberedOf are the bodies said as version.numbered for name: none since design 47 G §349. func (r *registry) numberedOf(name string) []string { r.mu.Lock() defer r.mu.Unlock() var out []string for _, body := range r.numbered { if strings.Contains(body, `"`+name+`"`) { out = append(out, body) } } return out }