package build import ( "context" "errors" "fmt" "slices" "strings" "sync" "testing" "git.novox.be/novox/mesh-sdk/go/delivery" ) // A done key whose blob the artifact store no longer holds is built again (novox/hq issue 1937): on 2026-10-24 the // store's 03:30 collection deleted the bare blobs of node-tools', mesh-host's and the forge bridge's done keys, which the // controller had never taken in, and the builder kept them done. // fakeHeld answers the controller's verb held: the references in gone are gone. type fakeHeld struct { mu sync.Mutex gone map[string]bool err error asked [][]string } func (f *fakeHeld) stored(_ context.Context, refs []string) ([]string, error) { f.mu.Lock() defer f.mu.Unlock() f.asked = append(f.asked, slices.Clone(refs)) if f.err != nil { return nil, f.err } var out []string for _, r := range refs { if f.gone[r] { out = append(out, r) } } return out, nil } func (f *fakeHeld) asks() int { f.mu.Lock() defer f.mu.Unlock() return len(f.asked) } // logged keeps what the builder says. type logged struct { mu sync.Mutex lines []string } func (l *logged) logf(format string, args ...any) { l.mu.Lock() defer l.mu.Unlock() l.lines = append(l.lines, fmt.Sprintf(format, args...)) } func (l *logged) has(words ...string) bool { l.mu.Lock() defer l.mu.Unlock() for _, line := range l.lines { all := true for _, w := range words { all = all && strings.Contains(line, w) } if all { return true } } return false } const goneRef = "artifact-store://m/tool/blobs/" + digest func doneKeyNaming(ref string) delivery.BuilderBuild { return delivery.BuilderBuild{State: delivery.BuildDone, By: "shanks.builder", Manifest: manifestD, Step: "built mesh-build-go on shanks", Artifacts: []delivery.BuiltArtifact{{Name: "tool", Kind: "bundle", Digest: digest, Reference: ref}}} } // 1937/AC3 (unit, the claim): a done key of the trunk's fingerprint whose blob is gone is claimed building as a new // revision, its step naming the gone reference; a candidate's done key is not asked about. func TestADoneTrunkKeyWhoseBlobIsGoneIsClaimedAgain(t *testing.T) { st := newMem() c := Claimer{Store: st, By: "novox.builder"} key, _ := delivery.BuilderBuildKey("m", fp1) rev := putBuild(t, st, key, doneKeyNaming(goneRef)) held := &fakeHeld{gone: map[string]bool{goneRef: true}} cl, err := c.Claim(t.Context(), "m", fp1, manifestD, ClaimOptions{Stored: held.stored}) if err != nil || cl != nil || held.asks() != 0 { t.Fatalf("a candidate's done key: claim %+v, %v, %d asks; want nothing and no ask", cl, err, held.asks()) } cl, err = c.Claim(t.Context(), "m", fp1, manifestD, ClaimOptions{Trunk: true, Stored: held.stored}) if err != nil || cl == nil { t.Fatalf("claim = %+v, %v; want the key claimed again", cl, err) } if cl.Rev <= rev || cl.Build.State != delivery.BuildBuilding || !slices.Equal(cl.Gone, []string{goneRef}) { t.Errorf("claim = %+v (revision was %d); want building as a new revision naming the gone reference", cl, rev) } v := valueOf(t, st, key) if v.State != delivery.BuildBuilding || !strings.Contains(v.Step, "no longer holds") || !strings.Contains(v.Step, goneRef) { t.Errorf("the key holds %+v; want building, its step naming the gone reference", v) } } // 1937/AC5 (unit, the claim): a done trunk key whose blobs are all held stays done; a store that cannot be asked leaves // it done and says why; images and packages are not asked about. func TestADoneTrunkKeyStaysDoneWhileTheStoreHoldsItsBlobs(t *testing.T) { st := newMem() c := Claimer{Store: st, By: "novox.builder"} key, _ := delivery.BuilderBuildKey("m", fp1) b := doneKeyNaming(goneRef) b.Artifacts = append(b.Artifacts, delivery.BuiltArtifact{Name: "img", Kind: "image", Digest: digest, Reference: "registry.example/m/img@" + digest}) rev := putBuild(t, st, key, b) held := &fakeHeld{gone: map[string]bool{}} if cl, err := c.Claim(t.Context(), "m", fp1, manifestD, ClaimOptions{Trunk: true, Stored: held.stored}); err != nil || cl != nil { t.Fatalf("claim = %+v, %v; want nothing: the store holds the blob", cl, err) } if len(held.asked) != 1 || !slices.Equal(held.asked[0], []string{goneRef}) { t.Errorf("asked %v; want the bundle's reference alone", held.asked) } held.err = errors.New("the controller did not answer") cl, err := c.Claim(t.Context(), "m", fp1, manifestD, ClaimOptions{Trunk: true, Stored: held.stored}) if cl != nil || err == nil || !strings.Contains(err.Error(), "cannot be asked") || !strings.Contains(err.Error(), "left done") { t.Errorf("claim = %+v, %v; want the key left done and said", cl, err) } if st.revs[key] != rev { t.Errorf("the key moved to revision %d (was %d)", st.revs[key], rev) } } // 1937/AC3 (unit, the builder): the builder's look at a module whose done key's blob is gone builds it again, says so // in a log line and in the done key's step, and asks the store once per process while it answers held. func TestTheBuilderBuildsAgainADoneKeyWhoseBlobIsGone(t *testing.T) { r := newRig(t, goBundle) log := &logged{} r.b.Logf = log.logf held := &fakeHeld{gone: map[string]bool{goneRef: true}} r.b.Stored = held.stored rev := putBuild(t, r.st, r.key, doneKeyNaming(goneRef)) v := r.consider(t, "modules/m") if v.State != delivery.BuildDone || r.st.revs[r.key] <= rev || len(r.bm.callsOf("build_go_build")) == 0 { t.Fatalf("build = %+v at revision %d (was %d); want built again", v, r.st.revs[r.key], rev) } if !strings.Contains(v.Step, "built again: the artifact store no longer held "+goneRef) { t.Errorf("the done key's step %q does not say it was built again for the gone blob", v.Step) } if !log.has(r.key, "built again", goneRef) { t.Errorf("no log line says the key was built again: %v", log.lines) } // The new revision's blob is held: asked once, then not again in this process. held.mu.Lock() held.gone = map[string]bool{} held.mu.Unlock() rev = r.st.revs[r.key] asks := held.asks() r.consider(t, "modules/m") r.consider(t, "modules/m") if r.st.revs[r.key] != rev { t.Errorf("a key whose blob is held was built again") } if held.asks() != asks+1 { t.Errorf("asked the store %d times for two looks at a held key; want once", held.asks()-asks) } } // 1937/AC4 (unit, the builder): artifacts-gone for the module's fingerprint makes the builder ask again, even for a key // it found held before, and build it again; one for another fingerprint builds nothing. func TestArtifactsGoneMakesTheBuilderAskAndBuildAgain(t *testing.T) { r := newRig(t, goBundle) log := &logged{} r.b.Logf = log.logf held := &fakeHeld{gone: map[string]bool{}} r.b.Stored = held.stored putBuild(t, r.st, r.key, doneKeyNaming(goneRef)) if v := r.consider(t, "modules/m"); v.State != delivery.BuildDone || len(r.bm.callsOf("build_go_build")) != 0 { t.Fatalf("a key whose blob is held was built: %+v", v) } rev := r.st.revs[r.key] held.mu.Lock() held.gone[goneRef] = true // collected after the builder's look; the controller's take-in found it held.mu.Unlock() r.b.ArtifactsGone(context.Background(), delivery.ArtifactsGone{Event: delivery.ArtifactsGoneEvent, Module: "m", Fingerprint: "src2:" + strings.Repeat("9", 64), Gone: []string{goneRef}}) r.b.Wait() if r.st.revs[r.key] != rev || !log.has("a fingerprint the module does not have now") { t.Fatalf("an event for another fingerprint moved the key (revision %d, was %d): %v", r.st.revs[r.key], rev, log.lines) } r.b.ArtifactsGone(context.Background(), delivery.ArtifactsGone{Event: delivery.ArtifactsGoneEvent, Module: "m", Key: r.key, Fingerprint: fp1, Gone: []string{goneRef}}) r.b.Wait() v := valueOf(t, r.st, r.key) if v.State != delivery.BuildDone || r.st.revs[r.key] <= rev || len(r.bm.callsOf("build_go_build")) == 0 { t.Fatalf("build = %+v at revision %d (was %d); want built again on the event", v, r.st.revs[r.key], rev) } if !log.has("heard artifacts-gone") || !log.has(r.key, "built again", goneRef) { t.Errorf("the builder did not say what it heard and did: %v", log.lines) } }