package build import ( "context" "encoding/json" "errors" "fmt" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "strconv" "strings" "sync" "sync/atomic" "testing" "time" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" ) // buildModules simulates the two build modules on the nodes: a long tool answers a step and key at once and writes the // step's end into the steps (as mesh-build-go and mesh-build-node do), with a digest by node and artifact, or a failure. type buildModules struct { mu sync.Mutex n int calls []call steps *fakeSteps digests map[string]string // `/` -> digest; the default is `digest` fails map[string]string // `/` -> the reason the step fails with // noRef names `/` whose bundle or archive build answers no reference (its push not answered). noRef map[string]bool // packages are the package artifacts by name, each with the `@` its trunk build publishes (ADR 0498 clause 5). packages map[string]string // hold, when set before the builds start, blocks every call until it is closed; entered is sent to as a call arrives. // conflicts, when set, is a file a fetch with a base reports in conflict; noMerge makes it answer neither a merge nor a conflict. conflicts string noMerge bool hold chan struct{} entered chan struct{} } func newBuildModules() *buildModules { return &buildModules{steps: &fakeSteps{steps: map[string]delivery.Step{}}, digests: map[string]string{}, fails: map[string]string{}, noRef: map[string]bool{}, packages: map[string]string{}} } func (b *buildModules) CallOn(node, key string, body any) (json.RawMessage, error) { if b.hold != nil { select { case b.entered <- struct{}{}: default: } <-b.hold } b.mu.Lock() defer b.mu.Unlock() b.calls = append(b.calls, call{node, key, body}) b.n++ id := fmt.Sprintf("s%d", b.n) module, tool, _ := strings.Cut(key, ".") args, _ := body.(map[string]any) step := delivery.Step{ID: id, Tool: tool, Node: node, State: delivery.StepPassed} if reason, ok := b.fails[node+"/"+tool]; ok { step.State, step.Reason = delivery.StepFailed, reason } else if strings.HasSuffix(tool, "_fetch") { step.Result = json.RawMessage(`{"workspace":"w-` + node + `"}`) if args["base"] != nil { // a fetch with a base answers the merge, or the files in conflict (mesh-build-go FetchResult) if b.conflicts != "" { step.Result = json.RawMessage(`{"workspace":"w-` + node + `","base":"` + fmt.Sprint(args["base"]) + `","conflicts":["` + b.conflicts + `"]}`) } else if !b.noMerge { step.Result = json.RawMessage(`{"workspace":"w-` + node + `","base":"` + fmt.Sprint(args["base"]) + `","merge":"` + strings.Repeat("9", 40) + `"}`) } } } else { d := digest if v, ok := b.digests[node+"/"+fmt.Sprint(args["artifact"])]; ok { d = v } if tool == "build_image_copy" { // mesh-build-image's copy answers the digest and where it copied to (ADR 0498 clause 4) step.Result = json.RawMessage(`{"artifact":"` + fmt.Sprint(args["artifact"]) + `","digest":"` + d + `","reference":"registry.example/` + fmt.Sprint(args["module"]) + `/` + fmt.Sprint(args["artifact"]) + `@` + d + `","pushed":true}`) } else if pkg, ok := b.packages[fmt.Sprint(args["artifact"])]; ok { // a package names its version only when published step.Result = json.RawMessage(`{"digest":"` + d + `"}`) if args["publish"] == true { step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"` + pkg + `"}`) } } else if tool == "build_image_build" && args["publish"] == true { step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"registry.example/m/` + fmt.Sprint(args["artifact"]) + `@` + d + `"}`) } else if tool != "build_image_build" && !b.noRef[node+"/"+fmt.Sprint(args["artifact"])] { // A bundle or an archive is pushed to the artifact store and its reference answered (ADR 0498 clause 1). step.Result = json.RawMessage(`{"digest":"` + d + `","reference":"artifact-store://m/` + fmt.Sprint(args["artifact"]) + `/blobs/` + d + `"}`) } else { step.Result = json.RawMessage(`{"digest":"` + d + `"}`) } } sk, _ := delivery.StepKey(node, id) b.steps.mu.Lock() b.steps.steps[module+"/"+sk] = step b.steps.mu.Unlock() return json.RawMessage(`{"step":"` + id + `","key":"` + sk + `","state":"running"}`), nil } func (b *buildModules) callsOf(tool string) (out []call) { b.mu.Lock() defer b.mu.Unlock() for _, c := range b.calls { if strings.HasSuffix(c.key, "."+tool) { out = append(out, c) } } return out } type fakeManifests struct { byDigest map[string]catalogue.Manifest } func (f *fakeManifests) Manifest(_ context.Context, d string) (catalogue.Manifest, error) { m, ok := f.byDigest[d] if !ok { return catalogue.Manifest{}, fmt.Errorf("manifests.%s is not there", d) } return m, nil } func mdigest(c byte) string { return hex64(c) } type rig struct { b *Builder st *memStore bm *buildModules tools *buildrun.ToolchainSet mf *fakeManifests key string // asks are the build seats' streams; puller is the node that pulls every ask ("ace" when empty). asks *buildrun.FakeAsks puller string } // toolPrefix is the start of each build module's tool names, as the fake build modules answer them. var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_", ImageBuildModule: "build_image_"} // answer runs an ask as the build module that pulls it does (buildstep.RunAsk), with the fake build modules' tools on the // rig's pulling node, and answers the step that node writes. func (r *rig) answer(a delivery.StepAsk) delivery.Step { node := r.puller if node == "" { node = "ace" } module, _ := delivery.BuildModuleOf(a.Language) runner := buildrun.Runner{Caller: r.bm, Steps: r.bm.steps, Bound: time.Second} tool := func(name string) buildstep.StepFunc { return func(ctx context.Context, args map[string]any) (any, error) { raw, err := runner.Run(ctx, module, node, toolPrefix[module]+name, args) if err != nil { return nil, err } return raw, nil } } steps := buildstep.AskSteps{Fetch: tool("fetch"), Build: tool("build"), Copy: tool("copy")} out, err := buildstep.RunAsk(context.Background(), steps, a) step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed} if err != nil { step.State, step.Reason = delivery.StepFailed, err.Error() return step } step.Result, _ = json.Marshal(out) return step } func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig { t.Helper() r := &rig{st: newMem(), bm: newBuildModules(), tools: &buildrun.ToolchainSet{}, mf: &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): {Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}}}} for _, n := range []string{"ace", "novox"} { r.tools.Put(buildrun.GoBuildModule, n+".go", tcValue(t, "go", "go1.27.1"), false) r.tools.Put(buildrun.NodeBuildModule, n+".typescript", tcValue(t, "typescript", "tsc5"), false) } r.asks = &buildrun.FakeAsks{Answer: r.answer} r.b = &Builder{ Claimer: Claimer{Store: r.st, By: "novox.builder"}, Asks: buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second}, Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf, Trunk: func(context.Context, uint64, string) (string, error) { return trunkHead, nil }, } var err error if r.key, err = delivery.BuilderBuildKey("m", fp1); err != nil { t.Fatal(err) } return r } // agreedToolchains are what the rig's nodes report, and so what a fingerprint computed now names. var agreedToolchains = map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}} func moduleValue(t *testing.T, path string) []byte { return moduleValueWith(t, path, agreedToolchains) } func moduleValueWith(t *testing.T, path string, toolchains map[string]delivery.ToolchainUsed) []byte { t.Helper() data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP: "src1:" + hex64('f'), FP2: fp1, Manifest: mdigest('a'), Path: path, Toolchains: toolchains}) if err != nil { t.Fatal(err) } return data } func (r *rig) consider(t *testing.T, path string) delivery.BuilderBuild { t.Helper() r.b.Module(context.Background(), "m", moduleValue(t, path)) r.b.Wait() if _, ok := r.st.vals[r.key]; !ok { t.Fatal("no build key was written") } return valueOf(t, r.st, r.key) } var ( goBundle = catalogue.Artifact{Name: "tool", Kind: "bundle", Language: "go"} tsBundle = catalogue.Artifact{Name: "web", Kind: "bundle", Language: "typescript"} archiveA = catalogue.Artifact{Name: "files", Kind: "archive"} imageArt = catalogue.Artifact{Name: "img", Kind: "image"} manifestD = mdigest('a') ) func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4 r := newRig(t, goBundle, archiveA, tsBundle) v := r.consider(t, "modules/m") if v.State != delivery.BuildDone || v.By != "novox.builder" || v.Manifest != manifestD || len(v.Artifacts) != 3 { t.Fatalf("build = %+v, want done by novox.builder with three artifacts", v) } for i, want := range []string{"files", "tool", "web"} { if a := v.Artifacts[i]; a.Name != want || a.Digest != digest || a.Reference != "artifact-store://m/"+want+"/blobs/"+digest { t.Errorf("artifact %d = %+v, want %s with the digest and its artifact-store reference (ADR 0498 clause 1)", i, a, want) } } if v.Artifacts[0].Kind != "archive" || v.Artifacts[1].Kind != "bundle" { t.Errorf("kinds = %+v", v.Artifacts) } if got := v.Toolchains["go"]; got.Version != "go1.27.1" || got.Recipe != "1" { t.Errorf("toolchains = %+v, want go and typescript named", v.Toolchains) } if got := v.Toolchains["typescript"]; got.Version != "tsc5" { t.Errorf("toolchains = %+v", v.Toolchains) } // One fetch per build module, on one node (ADR 0478 clause 4); every build names the module's directory. if n := len(r.bm.callsOf("build_go_fetch")); n != 1 { t.Errorf("%d go fetches, want one", n) } if n := len(r.bm.callsOf("build_go_build")); n != 2 { // tool and files t.Errorf("%d go builds, want 2", n) } if n := len(r.bm.callsOf("build_node_build")); n != 1 { t.Errorf("%d node builds, want 1", n) } c := r.bm.callsOf("build_go_build")[0] if args := c.body.(map[string]any); args["module"] != "modules/m" || !strings.HasPrefix(fmt.Sprint(args["workspace"]), "w-") { t.Errorf("build args = %+v, want the module's path and the fetch's workspace", args) } f := r.bm.callsOf("build_go_fetch")[0].body.(map[string]any) if f["repository"] != "novox/mesh-catalog" || f["commit"] != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" { t.Errorf("fetch args = %+v", f) } } // 1887/AC1, AC2: a trunk fingerprint with no key is one ask on the Go seat's delivery stream, named in the building key // before it is put, and the builder calls no node; the step a node wrote for it makes the key done naming its reference. func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) { r := newRig(t, goBundle) r.asks.Answer = nil // no node has pulled it yet r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) var asks []delivery.StepAsk for deadline := time.Now().Add(2 * time.Second); len(asks) == 0; time.Sleep(time.Millisecond) { if time.Now().After(deadline) { t.Fatal("no ask was put") } asks = r.asks.Asks() } a := asks[0] if len(asks) != 1 || a.Kind != delivery.AskDelivery || a.Language != delivery.AskGo || a.Build == nil || a.Build.FP != fp1 { t.Fatalf("asks = %+v, want one Go delivery ask of the fingerprint", asks) } if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding || v.Asks[delivery.AskGo] != a.ID { t.Fatalf("the key = %+v, want building naming the ask %s", v, a.ID) } if n := len(r.bm.callsOf("build_go_build")) + len(r.bm.callsOf("build_go_fetch")); n != 0 { t.Fatalf("the builder called a node %d times", n) } r.asks.End(r.answer(a)) // a node pulled it and ran it r.b.Wait() if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || !strings.Contains(v.Step, "on ace") || v.Artifacts[0].Reference == "" || len(v.Asks) != 0 { t.Fatalf("the key = %+v, want done on ace naming the reference", v) } } // 1887/AC3: a builder that starts again while its ask waits puts no second ask: it waits for the step of the ask the // building key names. func TestARestartedBuilderWaitsForTheAskTheKeyNames(t *testing.T) { r := newRig(t, goBundle) const id = "mvl0frv18d1957" data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD, Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: id}}) r.st.vals[r.key] = data r.asks.Answer = nil r.asks.End(delivery.Step{ID: id, Node: "shanks", State: delivery.StepPassed, Result: []byte(`{"artifacts":[{"artifact":"tool","digest":"` + digest + `","reference":"artifact-store://m/tool/blobs/` + digest + `"}]}`)}) v := r.consider(t, "modules/m") if n := len(r.asks.Asks()); n != 0 { t.Fatalf("%d asks put again", n) } if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on shanks") { t.Fatalf("the key = %+v, want done from the ask's step on shanks", v) } } func TestAFailingStepFailsTheBuildWithItsReasonAndNodeAndIsTriedAgainAsANewRevision(t *testing.T) { // 1607/AC7, ADR 0429 clause 18 r := newRig(t, goBundle) r.bm.fails["ace/build_go_build"] = "go build of tool: undefined: x" v := r.consider(t, "modules/m") if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "ace") || !strings.Contains(v.Error, "undefined: x") { t.Fatalf("build = %+v, want failed with the node and the reason", v) } rev := r.st.revs[r.key] delete(r.bm.fails, "ace/build_go_build") v = r.consider(t, "modules/m") if v.State != delivery.BuildDone || r.st.revs[r.key] <= rev { t.Errorf("build = %+v at revision %d (was %d), want done as a new revision of the same key", v, r.st.revs[r.key], rev) } } func TestAnImageGoesToOneNodeAndGetsOneDoneKeyWithItsReference(t *testing.T) { // 1705a, ADR 0487 clause 5 r := newRig(t, imageArt) r.puller = "ace" // two nodes qualify; an image still builds on one v := r.consider(t, "modules/m") if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "image" || v.Artifacts[0].Digest != digest { t.Fatalf("build = %+v, want done with one image artifact and its digest", v) } if v.Artifacts[0].Reference == "" { t.Errorf("a trunk image has no reference: %+v", v.Artifacts[0]) } if n := len(r.bm.callsOf("build_image_fetch")); n != 1 { t.Errorf("%d image fetches, want one", n) } writes := r.st.writes r.consider(t, "modules/m") if r.st.writes != writes || len(r.bm.callsOf("build_image_build")) != 1 { t.Errorf("a done image key was built again") } } func TestABundleAndAnImageOfOneModuleBuildOnOneNodeEach(t *testing.T) { // 1705a r := newRig(t, goBundle, imageArt) r.puller = "ace" v := r.consider(t, "modules/m") if v.State != delivery.BuildDone || len(v.Artifacts) != 2 { t.Fatalf("build = %+v", v) } if n := len(r.bm.callsOf("build_go_build")); n != 1 { t.Errorf("%d go builds, want 1 (one node per fingerprint, ADR 0478 clause 4)", n) } if n := len(r.bm.callsOf("build_image_build")); n != 1 { t.Errorf("%d image builds, want 1", n) } } func TestACandidatesImageCallCarriesNoPublishAndItsKeyNoReference(t *testing.T) { // 1705b, lead's decision r := newRig(t, imageArt) r.puller = "ace" trunkModule(t, r) r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) r.b.Wait() v := valueOf(t, r.st, r.key) if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Reference != "" { t.Fatalf("build = %+v, want done with no reference", v) } calls := r.bm.callsOf("build_image_build") if len(calls) != 1 { t.Fatalf("%d image builds, want 1", len(calls)) } if _, has := calls[0].body.(map[string]any)["publish"]; has { t.Errorf("a candidate's image call carries publish: %+v", calls[0].body) } } func TestATrunkImageCallCarriesPublish(t *testing.T) { // 1705b r := newRig(t, imageArt) r.puller = "ace" r.consider(t, "modules/m") calls := r.bm.callsOf("build_image_build") if len(calls) != 1 || calls[0].body.(map[string]any)["publish"] != true { t.Errorf("calls = %+v, want publish true", calls) } } func TestAnUnpinnedUpstreamFailsNotPinnedAndIsNeverBuiltAgain(t *testing.T) { // B3, ADR 0498 clause 4 r := newRig(t, archiveA, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine"}) v := r.consider(t, "modules/m") if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotPinned) || !strings.Contains(v.Error, "up") { t.Fatalf("build = %+v, want failed 'not pinned:' naming the artifact", v) } if len(r.bm.calls) != 0 { t.Errorf("%d calls, want none for an unpinned upstream", len(r.bm.calls)) } writes := r.st.writes r.consider(t, "modules/m") if r.st.writes != writes { t.Errorf("an unpinned upstream's key was written again (%d writes, was %d)", r.st.writes, writes) } } func TestAModuleKeyWithNoPathFailsNamingThatReason(t *testing.T) { // 1607/AC8, ADR 0478 clause 8 r := newRig(t, goBundle) v := r.consider(t, "") if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "path") || len(r.bm.calls) != 0 { t.Errorf("build = %+v with %d calls, want failed naming the missing path and nothing called", v, len(r.bm.calls)) } } func TestAnUnreadableManifestFailsTheBuild(t *testing.T) { r := newRig(t, goBundle) delete(r.mf.byDigest, mdigest('a')) v := r.consider(t, "modules/m") if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "manifest") { t.Errorf("build = %+v, want failed naming the manifest", v) } } func TestAModuleWithNoFingerprintIsNotBuilt(t *testing.T) { // 1607/AC2 r := newRig(t) data, _ := delivery.EncodeModule(delivery.Module{Repository: "novox/r", Commit: "c", FP: "src1:" + hex64('f'), None2: "toolchain not yet reported"}) r.b.Module(context.Background(), "n", data) r.b.Wait() if len(r.st.vals) != 0 { t.Errorf("keys %v were written: a module with no new-form fingerprint has nothing to build", r.st.vals) } } func TestAModuleWithNoArtifactGetsADoneKeyWithNoArtifactAndNoBuildCall(t *testing.T) { // issue 1613, ADR 0478 clause 1 for name, manifest := range map[string]catalogue.Manifest{"no build": {Module: "m"}, "no artifact": {Module: "m", Build: &catalogue.Build{}}} { r := newRig(t) r.mf.byDigest[mdigest('a')] = manifest v := r.consider(t, "") // a module with nothing to build needs no directory if v.State != delivery.BuildDone || len(v.Artifacts) != 0 || v.Manifest != manifestD || v.By != "novox.builder" { t.Errorf("%s: build = %+v, want done with no artifact", name, v) } if len(r.bm.calls) != 0 { t.Errorf("%s: %d build module calls, want none", name, len(r.bm.calls)) } writes := r.st.writes r.b.tried = map[string]bool{} r.consider(t, "") if r.st.writes != writes { t.Errorf("%s: the done key was written again", name) } } } func TestAKeyTheBuilderLeftBuildingIsBuiltAtStartAndDoneKeysAreNotBuiltTwice(t *testing.T) { // 1607/AC3, AC7 r := newRig(t, goBundle) seed(t, r.st, delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD}) if v := r.consider(t, "modules/m"); v.State != delivery.BuildDone { t.Fatalf("build = %+v, want the builder's own building key built", v) } calls := len(r.bm.calls) r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) r.b.Reconsider(context.Background()) r.b.Wait() if len(r.bm.calls) != calls { t.Errorf("a done key was built again (%d calls, was %d)", len(r.bm.calls), calls) } } // Two instances that start together on one key: the second finds the key building and takes it over (ADR 0478 clause 6: // a key found building at start is taken over, whoever holds it), so both may build; the key is written once, by the one // whose compare-and-set comes first, and the other's write is refused, never overwriting it. func TestTwoInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { // 1607/AC7, review round 4 r := newRig(t, goBundle) other := &Builder{Claimer: Claimer{Store: r.st, By: "ace.builder"}, Asks: r.b.Asks, Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf} value := moduleValue(t, "modules/m") var wg sync.WaitGroup for _, b := range []*Builder{r.b, other} { wg.Add(1) go func() { defer wg.Done(); b.Module(context.Background(), "m", value); b.Wait() }() } wg.Wait() if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(v.Artifacts) != 1 { t.Errorf("build = %+v, want one done key", v) } if n := len(r.bm.callsOf("build_go_build")); n < 1 { t.Errorf("%d builds, want one for each instance that built", n) } } func TestAModulesValueThatArrivesWhileItsKeyIsBeingBuiltIsLookedAtAgainWhenItEnds(t *testing.T) { // 1607/AC6 (no event is dropped, no timer) r := newRig(t, goBundle) r.bm.fails["ace/build_go_build"] = "go build of tool: boom" r.bm.hold, r.bm.entered = make(chan struct{}), make(chan struct{}, 8) value := moduleValue(t, "modules/m") r.b.Module(context.Background(), "m", value) <-r.bm.entered // the first build is under way r.b.Module(context.Background(), "m", value) r.b.Reconsider(context.Background()) delete(r.bm.fails, "ace/build_go_build") // what the event is about: the build can succeed now close(r.bm.hold) r.b.Wait() if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone { t.Errorf("build = %+v, want the failed key built again as a new revision after the first build ended", v) } } func TestAKeyIsBuiltOnlyWithTheToolchainItsFingerprintNamesAndNothingIsWrittenOtherwise(t *testing.T) { // 1615/AC4 r := newRig(t, goBundle) old := map[string]delivery.ToolchainUsed{"go": {Version: "go1.26.0", Recipe: "1"}} r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", old)) r.b.Wait() if len(r.st.vals) != 0 || len(r.bm.calls) != 0 { t.Errorf("keys %v and %d calls for a fingerprint of go1.26.0 while the nodes agree on go1.27.1, want neither", r.st.vals, len(r.bm.calls)) } // A recipe that differs is a different toolchain too, and a value that names none (written before) is not built. r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "0"}})) r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", nil)) r.b.Wait() if len(r.st.vals) != 0 || len(r.bm.calls) != 0 { t.Errorf("keys %v and %d calls for a recipe that differs or a value that names no toolchain", r.st.vals, len(r.bm.calls)) } // The toolchain moves under a key that is not built yet: the old fingerprint stays unbuilt, the new one builds. r.tools.Put(buildrun.GoBuildModule, "ace.go", tcValue(t, "go", "go1.27.2"), false) r.tools.Put(buildrun.GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.2"), false) r.b.Reconsider(context.Background()) r.b.Wait() if len(r.st.vals) != 0 { t.Errorf("keys %v: the old fingerprint was built with the new toolchain", r.st.vals) } r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.2", Recipe: "1"}})) r.b.Wait() if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || v.Toolchains["go"].Version != "go1.27.2" { t.Errorf("build = %+v, want done with go1.27.2", v) } } func TestNodesTheyDisagreeOnLeaveTheKeyUnwritten(t *testing.T) { // 1615/AC4 r := newRig(t, goBundle) r.tools.Put(buildrun.GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false) r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) r.b.Wait() if len(r.st.vals) != 0 || len(r.bm.calls) != 0 { t.Errorf("keys %v and %d calls while the nodes disagree", r.st.vals, len(r.bm.calls)) } } func TestARootModulesPathIsDotAndIsGivenToTheBuildTools(t *testing.T) { // 1615/AC4 r := newRig(t, goBundle) if v := r.consider(t, delivery.RootPath); v.State != delivery.BuildDone { t.Fatalf("build = %+v, want done for a root module", v) } if args := r.bm.callsOf("build_go_build")[0].body.(map[string]any); args["module"] != "." { t.Errorf("build args = %+v, want module \".\"", args) } } func TestACandidatesImageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testing.T) { // review r1, mesh-delivery 1; ADR 0487 clause 5 r := newRig(t, imageArt) r.puller = "ace" trunkModule(t, r) // the trunk carries another fingerprint; fp1 is the candidate's r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) r.b.Wait() v := valueOf(t, r.st, r.key) if v.State != delivery.BuildDone || v.Artifacts[0].Reference != "" { t.Fatalf("candidate build = %+v, want done with no reference", v) } if n := len(r.bm.callsOf("build_image_build")); n != 1 { t.Fatalf("%d image builds after the candidate, want 1", n) } // The pull request is merged: the trunk's record names the candidate's fingerprint. r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) r.b.Wait() v = valueOf(t, r.st, r.key) if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Reference == "" { t.Fatalf("trunk build = %+v, want done with a reference", v) } calls := r.bm.callsOf("build_image_build") published := 0 for _, c := range calls { if c.body.(map[string]any)["publish"] == true { published++ } } if len(calls) != 2 || published != 1 { t.Errorf("%d image builds, %d with publish, want 2 and exactly 1", len(calls), published) } // Written once more only: looking again builds nothing. rev := r.st.revs[r.key] r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) r.b.Wait() if r.st.revs[r.key] != rev || len(r.bm.callsOf("build_image_build")) != 2 { t.Error("a published key was built again") } } func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // review r2, mesh-delivery 1; ADR 0487 clause 5 r := newRig(t, imageArt) r.puller = "ace" trunkModule(t, r) r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) r.b.Wait() if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone { t.Fatalf("candidate build = %+v", v) } r.bm.fails["ace/build_image_build"] = "push to the registry: connection refused" r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the merge: the trunk names the fingerprint r.b.Wait() v := valueOf(t, r.st, r.key) if v.State != delivery.BuildFailed || !strings.Contains(v.Error, "connection refused") || !strings.Contains(v.Error, "publish") { t.Fatalf("build = %+v, want failed naming the publish and its reason", v) } // The failed key is retaken as any failed key is, and ends published once the registry answers. delete(r.bm.fails, "ace/build_image_build") r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) r.b.Wait() if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || v.Artifacts[0].Reference == "" { t.Errorf("build = %+v, want done with a reference", v) } } // goneAsks answers Gone from a function of the ask, and counts how often it was asked; it keeps the ids it was asked to // drop from their stream (Dropper). type goneAsks struct { mu sync.Mutex n int why func(a delivery.StepAsk) string dropped []string } func (g *goneAsks) Drop(_ context.Context, a delivery.StepAsk) error { g.mu.Lock() defer g.mu.Unlock() g.dropped = append(g.dropped, a.ID) return nil } func (g *goneAsks) Gone(_ context.Context, a delivery.StepAsk) (string, error) { g.mu.Lock() g.n++ g.mu.Unlock() return g.why(a), nil } func (g *goneAsks) asked() int { g.mu.Lock() defer g.mu.Unlock() return g.n } // waitForAsks waits until n asks were put. func waitForAsks(t *testing.T, r *rig, n int) []delivery.StepAsk { t.Helper() for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) < n; time.Sleep(time.Millisecond) { if time.Now().After(deadline) { t.Fatalf("%d asks put, want %d", len(r.asks.Asks()), n) } } return r.asks.Asks() } // Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked // again; the key is done when a node takes it. Issue 1945: an ask that can no longer be answered (its stream gave it up) // is asked again under a new id, said, at most Askings times; a person asks it again by hand the same way. func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { t.Run("waits in its stream: never asked again", func(t *testing.T) { r := newRig(t, goBundle) r.asks.Answer = nil r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond} gone := &goneAsks{why: func(delivery.StepAsk) string { return "" }} r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) waitForAsks(t, r, 1) time.Sleep(100 * time.Millisecond) if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding { t.Fatalf("the key = %+v while its ask waits, want building", v) } if gone.asked() == 0 { t.Fatal("whether the ask can still be answered was never read at the check bound") // 1945 } r.asks.End(r.answer(r.asks.Asks()[0])) r.b.Wait() if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 { t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks())) } }) for _, c := range []struct { name string askings int // step is the key's step words at start when it names an ask (a key taken over), "" for a fresh build. step string byHand bool always bool // Gone says every ask cannot be answered, not only the first puts int done bool asking int // the asking the event says, 0 when none is said }{ {name: "given up by its stream: asked again under a new id and done from it", puts: 2, done: true, asking: 2}, {name: "asked again by hand: the same, with no check bound", byHand: true, puts: 2, done: true, asking: 2}, {name: "never answerable: failed after its askings", askings: 2, always: true, puts: 2}, {name: "taken over at its last asking: failed, not asked again", askings: 3, always: true, step: "asked mesh-build-go again (asking 3): x", puts: 0}, } { t.Run(c.name, func(t *testing.T) { // 1945 r := newRig(t, goBundle) r.asks.Answer = nil r.b.Askings = c.askings var said []AskedAgain var mu sync.Mutex r.b.Said = func(e AskedAgain) { mu.Lock(); said = append(said, e); mu.Unlock() } first := "" if c.step != "" { first = "mvl0frv18d1960" data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD, Step: c.step, Asks: map[string]string{delivery.AskGo: first}}) r.st.vals[r.key] = data } gone := &goneAsks{} if !c.byHand { gone.why = func(a delivery.StepAsk) string { if c.always || a.ID == first || first == "" { return "it waits first in line while a node took a later ask" } return "" } r.b.Gone, r.b.CheckAfter = gone, 5*time.Millisecond } r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) if c.byHand { waitForAsks(t, r, 1) for deadline := time.Now().Add(2 * time.Second); len(r.b.AskAgain(r.key, "stuck since 19:06")) == 0; time.Sleep(time.Millisecond) { if time.Now().After(deadline) { t.Fatal("no lane of the key waits") } } } if c.done { asks := waitForAsks(t, r, c.puts) if first == "" && !c.byHand { first = asks[0].ID } last := asks[len(asks)-1] if v := valueOf(t, r.st, r.key); v.Asks[delivery.AskGo] != last.ID || !strings.Contains(v.Step, "again (asking 2)") { t.Fatalf("the key = %+v, want building naming the ask asked again %s", v, last.ID) } r.asks.End(r.answer(last)) } r.b.Wait() v := valueOf(t, r.st, r.key) if n := len(r.asks.Asks()); n != c.puts { t.Errorf("%d asks put, want %d", n, c.puts) } if c.done && v.State != delivery.BuildDone { t.Errorf("the key = %+v, want done from the ask asked again", v) } if !c.done && (v.State != delivery.BuildFailed || !strings.Contains(v.Error, "could not be answered and was asked")) { t.Errorf("the key = %+v, want failed saying the ask could not be answered", v) } mu.Lock() defer mu.Unlock() // Note 62: the old message of an ask asked again because its stream gave it up is dropped from the stream. if gone.mu.Lock(); c.done && !c.byHand && (len(gone.dropped) != 1 || gone.dropped[0] != first) { t.Errorf("dropped %v from the stream, want the old ask %s", gone.dropped, first) } gone.mu.Unlock() if c.asking > 0 && (len(said) != 1 || said[0].Asking != c.asking || said[0].New == said[0].Old || said[0].Key != r.key || said[0].Why == "") { t.Errorf("said %+v, want once at asking %d", said, c.asking) } }) } t.Run("what the bus says of an ask", func(t *testing.T) { // 1945 put := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC) taken := func(kind string, at time.Time) *Taken { return &Taken{Ask: "later", Kind: kind, At: at, Node: "g14", Started: at} } later := taken(delivery.AskDelivery, put.Add(time.Minute)) // ahead is n messages put before the ask, a minute apart, each with step (its state, "" for none). ahead := func(n int, step string) []Queued { var qs []Queued for i := range n { at := put.Add(-time.Duration(n-i) * time.Minute) qs = append(qs, Queued{Seq: uint64(i + 1), Ask: fmt.Sprint("a", i), Kind: delivery.AskDelivery, At: at, PutAt: at, Step: step}) } return qs } seen := func(qs []Queued, last *Taken) AskSeen { return AskSeen{Stream: "S", Kind: delivery.AskDelivery, At: put, OnStream: true, PutAt: put, Ahead: qs, Last: last} } for _, c := range []struct { name string seen AskSeen want string // a part of the why, "" when it can be answered waiting int }{ {"a node took it", AskSeen{Stream: "S", Step: true}, "", 0}, {"no longer on its stream", AskSeen{Stream: "S"}, "no longer holds it", 0}, {"first in line, a later ask taken", seen(nil, later), "no longer delivers it", 0}, {"first in line, nothing taken since", seen(nil, nil), "", 0}, {"first in line, the ask taken last was asked before it", seen(nil, taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0}, {"two wait ahead, nothing taken since", seen(ahead(2, ""), nil), "", 2}, {"two ahead passed over, the ask not yet", seen(ahead(2, ""), taken(delivery.AskDelivery, put.Add(-time.Second))), "", 0}, // Note 62: seven messages SEAT_NODE_BUILD_GO gave up on sat ahead of a Go ask for five hours. {"seven given up on ahead, a later ask taken", seen(ahead(7, ""), later), "behind 7 given up on or taken", 0}, {"seven taken ahead (failed as poison), a later ask taken", seen(ahead(7, delivery.StepFailed), later), "no longer delivers it", 0}, {"a check ask taken later proves a delivery ask passed", seen(ahead(7, ""), taken(delivery.AskCheck, put.Add(time.Minute))), "no longer delivers it", 0}, {"a delivery ask taken later does not prove a check ask passed", func() AskSeen { s := seen(ahead(3, ""), later); s.Kind = delivery.AskCheck; return s }(), "", 3}, } { got := c.seen.Why() if (c.want == "") != (got == "") || !strings.Contains(got, c.want) || c.seen.Waiting() != c.waiting { t.Errorf("%s: why %q and %d waiting, want %q and %d", c.name, got, c.seen.Waiting(), c.want, c.waiting) } } if at := time.UnixMilli(1793120000123); !AskedAt(strconv.FormatInt(at.UnixMilli(), 36)+"0a1b2c").Equal(at) || !AskedAt("x").IsZero() { t.Errorf("AskedAt does not read an ask id's time") } }) t.Run("the tool builder_stuck", func(t *testing.T) { // 1945 at := time.Date(2026, 10, 25, 17, 6, 0, 0, time.UTC) state := func(vals map[string]string) StateRead { return StateRead{Keys: func() ([]string, error) { var ks []string for k := range vals { ks = append(ks, k) } return ks, nil }, Get: func(k string) (json.RawMessage, time.Time, error) { if v, ok := vals[k]; ok { return json.RawMessage(v), at, nil } return nil, time.Time{}, nil }} } key, _ := delivery.BuilderBuildKey("m", fp1) done, _ := delivery.BuilderBuildKey("d", fp1) building, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD, Step: "asked mesh-build-go again (asking 2): x", Asks: map[string]string{delivery.AskGo: "mvo53jcm70b5f7", delivery.AskImage: "mvo53jcm70b5f8"}}) finished, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: manifestD, Step: "built"}) running, _ := json.Marshal(delivery.Step{V: delivery.Version, ID: "mvo53jcm70b5f8", Tool: "delivery ask", Node: "g14", State: delivery.StepRunning, Started: at}) _, reask, _ := NewReAsk("m", fp1, "stuck", "shanks.lead", at) r := StuckReader{Builds: state(map[string]string{key: string(building), done: string(finished)}), ReAsks: state(map[string]string{key: string(reask)}), Steps: map[string]StateRead{buildrun.GoBuildModule: state(map[string]string{}), ImageBuildModule: state(map[string]string{"g14.mvo53jcm70b5f8": string(running)})}, Now: func() time.Time { return at.Add(time.Hour) }} out, err := r.Stuck("") if err != nil { t.Fatal(err) } stuck := out["stuck"].([]Stuck) if len(stuck) != 1 || stuck[0].Key != key || stuck[0].Age != "1h0m0s" || stuck[0].NoneStep != 1 || stuck[0].ReAsk == nil || stuck[0].ReAsk.Why != "stuck" { t.Fatalf("stuck = %+v, want the building key alone, an hour old, one ask with no step, with its ask again", stuck) } a := stuck[0].Asks if len(a) != 2 || a[0].Language != delivery.AskGo || a[0].Asking != 2 || a[0].Step != "none" || a[0].Asked.IsZero() || a[1].Asking != 1 || !strings.HasPrefix(a[1].Step, "running on g14") { t.Fatalf("asks = %+v, want go at asking 2 with no step and image at asking 1 running on g14", a) } // 1827: on the live shape (3000 build keys over 100 modules, 1 ms a Get, three keys of each module building) a module reads only its // own keys, the whole answer reads MaxReads values in parallel and says it was cut, and the history (Written) is asked // only for the rows kept. var gets, written, inFlight, maxFlight atomic.Int64 vals := map[string]string{} for i := 0; i < 3000; i++ { k, _ := delivery.BuilderBuildKey(fmt.Sprintf("mod%02d", i%100), fmt.Sprintf("src2:%064x", i)) vals[k] = string(finished) if (i/100)%10 == 0 { vals[k] = string(building) } } slow := state(vals) get := slow.Get slow.Get = func(k string) (json.RawMessage, time.Time, error) { gets.Add(1) n := inFlight.Add(1) for m := maxFlight.Load(); n > m && !maxFlight.CompareAndSwap(m, n); m = maxFlight.Load() { } time.Sleep(time.Millisecond) inFlight.Add(-1) raw, _, err := get(k) return raw, time.Time{}, err } slow.Written = func(string) time.Time { written.Add(1); return at } for _, row := range []struct { module string stuck int gets int64 cut bool }{{"mod10", 3, 30, false}, {"", 100, MaxReads, true}} { gets.Store(0) written.Store(0) out, err := StuckReader{Builds: slow, ReAsks: state(map[string]string{}), Steps: map[string]StateRead{}, Now: func() time.Time { return at }}.Stuck(row.module) if err != nil { t.Fatal(err) } _, cut := out["cut"] if n := len(out["stuck"].([]Stuck)); n != row.stuck || cut != row.cut || written.Load() != int64(n) || gets.Load() != row.gets { t.Errorf("%q: %d stuck, %d gets, %d written, cut %v; want %d stuck and gets, written for each, cut %v", row.module, n, gets.Load(), written.Load(), cut, row.stuck, row.cut) } } if maxFlight.Load() < 2 { t.Errorf("the values were read one at a time") } }) } // heldAsks says an ask is held when its id is among held. type heldAsks map[string]bool func (h heldAsks) Held(_ context.Context, a delivery.StepAsk) (bool, error) { return h[a.ID], nil } // 1910: a builder that stopped between naming the ask in its key and putting it left a key naming an ask neither the // stream holds nor any node ran: started again, the builder puts that ask, under the same id, once, and the key is // built from its step; an ask the stream holds is not put again. func TestAnAskTheKeyNamesAndNobodyPutIsPutAtStart(t *testing.T) { for _, held := range []bool{false, true} { r := newRig(t, goBundle) const id = "mvl0frv18d1958" data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD, Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: id}}) r.st.vals[r.key] = data r.b.Held = heldAsks{id: held} if held { r.asks.Answer = nil r.asks.End(delivery.Step{ID: id, Node: "shanks", State: delivery.StepPassed, Result: []byte(`{"artifacts":[{"artifact":"tool","digest":"` + digest + `","reference":"artifact-store://m/tool/blobs/` + digest + `"}]}`)}) } v := r.consider(t, "modules/m") asks := r.asks.Asks() switch { case held && len(asks) != 0: t.Errorf("an ask the stream holds was put again: %+v", asks) case !held && (len(asks) != 1 || asks[0].ID != id): t.Errorf("the ask nobody put: asks %+v, want one under %s", asks, id) } if v.State != delivery.BuildDone { t.Errorf("held %v: the key = %+v, want done", held, v) } } } // unreadableHeld cannot tell whether an ask is held. type unreadableHeld struct{} func (unreadableHeld) Held(context.Context, delivery.StepAsk) (bool, error) { return false, errors.New("permissions violation for publish to $JS.API.STREAM.INFO.SEAT_NODE_BUILD_GO") } // 1910: a take-over that cannot read whether the ask its key names is on its stream says so and waits for the ask's step: // the key is not failed, and the build finishes when a node ran the ask. func TestATakeOverThatCannotReadTheStreamWaitsForTheAsk(t *testing.T) { r := newRig(t, goBundle) const id = "mvl0frv18d1959" data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD, Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: id}}) r.st.vals[r.key] = data r.b.Held = unreadableHeld{} r.asks.Answer = nil r.asks.End(delivery.Step{ID: id, Node: "shanks", State: delivery.StepPassed, Result: []byte(`{"artifacts":[{"artifact":"tool","digest":"` + digest + `","reference":"artifact-store://m/tool/blobs/` + digest + `"}]}`)}) v := r.consider(t, "modules/m") if v.State != delivery.BuildDone || len(r.asks.Asks()) != 0 { t.Fatalf("the key = %+v after %d asks, want done from the ask's step, none put", v, len(r.asks.Asks())) } }