mesh-delivery/modules/mesh-delivery-builder/internal/build/gone_test.go

package build

import (
	"context"
	"errors"
	"fmt"
	"slices"
	"strings"
	"sync"
	"testing"

	"git.novox.be/novox/mesh-sdk/go/delivery"
)

// A done key whose blob the artifact store no longer holds is built again (novox/hq issue 1937): on 2026-10-24 the
// store's 03:30 collection deleted the bare blobs of node-tools', mesh-host's and the forge bridge's done keys, which the
// controller had never taken in, and the builder kept them done.

// fakeHeld answers the controller's verb held: the references in gone are gone.
type fakeHeld struct {
	mu    sync.Mutex
	gone  map[string]bool
	err   error
	asked [][]string
}

func (f *fakeHeld) stored(_ context.Context, refs []string) ([]string, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.asked = append(f.asked, slices.Clone(refs))
	if f.err != nil {
		return nil, f.err
	}
	var out []string
	for _, r := range refs {
		if f.gone[r] {
			out = append(out, r)
		}
	}
	return out, nil
}

func (f *fakeHeld) asks() int {
	f.mu.Lock()
	defer f.mu.Unlock()
	return len(f.asked)
}

// logged keeps what the builder says.
type logged struct {
	mu    sync.Mutex
	lines []string
}

func (l *logged) logf(format string, args ...any) {
	l.mu.Lock()
	defer l.mu.Unlock()
	l.lines = append(l.lines, fmt.Sprintf(format, args...))
}

func (l *logged) has(words ...string) bool {
	l.mu.Lock()
	defer l.mu.Unlock()
	for _, line := range l.lines {
		all := true
		for _, w := range words {
			all = all && strings.Contains(line, w)
		}
		if all {
			return true
		}
	}
	return false
}

const goneRef = "artifact-store://m/tool/blobs/" + digest

func doneKeyNaming(ref string) delivery.BuilderBuild {
	return delivery.BuilderBuild{State: delivery.BuildDone, By: "shanks.builder", Manifest: manifestD, Step: "built mesh-build-go on shanks",
		Artifacts: []delivery.BuiltArtifact{{Name: "tool", Kind: "bundle", Digest: digest, Reference: ref}}}
}

// 1937/AC3 (unit, the claim): a done key of the trunk's fingerprint whose blob is gone is claimed building as a new
// revision, its step naming the gone reference; a candidate's done key is not asked about.
func TestADoneTrunkKeyWhoseBlobIsGoneIsClaimedAgain(t *testing.T) {
	st := newMem()
	c := Claimer{Store: st, By: "novox.builder"}
	key, _ := delivery.BuilderBuildKey("m", fp1)
	rev := putBuild(t, st, key, doneKeyNaming(goneRef))
	held := &fakeHeld{gone: map[string]bool{goneRef: true}}

	cl, err := c.Claim(t.Context(), "m", fp1, manifestD, ClaimOptions{Stored: held.stored})
	if err != nil || cl != nil || held.asks() != 0 {
		t.Fatalf("a candidate's done key: claim %+v, %v, %d asks; want nothing and no ask", cl, err, held.asks())
	}
	cl, err = c.Claim(t.Context(), "m", fp1, manifestD, ClaimOptions{Trunk: true, Stored: held.stored})
	if err != nil || cl == nil {
		t.Fatalf("claim = %+v, %v; want the key claimed again", cl, err)
	}
	if cl.Rev <= rev || cl.Build.State != delivery.BuildBuilding || !slices.Equal(cl.Gone, []string{goneRef}) {
		t.Errorf("claim = %+v (revision was %d); want building as a new revision naming the gone reference", cl, rev)
	}
	v := valueOf(t, st, key)
	if v.State != delivery.BuildBuilding || !strings.Contains(v.Step, "no longer holds") || !strings.Contains(v.Step, goneRef) {
		t.Errorf("the key holds %+v; want building, its step naming the gone reference", v)
	}
}

// 1937/AC5 (unit, the claim): a done trunk key whose blobs are all held stays done; a store that cannot be asked leaves
// it done and says why; images and packages are not asked about.
func TestADoneTrunkKeyStaysDoneWhileTheStoreHoldsItsBlobs(t *testing.T) {
	st := newMem()
	c := Claimer{Store: st, By: "novox.builder"}
	key, _ := delivery.BuilderBuildKey("m", fp1)
	b := doneKeyNaming(goneRef)
	b.Artifacts = append(b.Artifacts, delivery.BuiltArtifact{Name: "img", Kind: "image", Digest: digest, Reference: "registry.example/m/img@" + digest})
	rev := putBuild(t, st, key, b)
	held := &fakeHeld{gone: map[string]bool{}}
	if cl, err := c.Claim(t.Context(), "m", fp1, manifestD, ClaimOptions{Trunk: true, Stored: held.stored}); err != nil || cl != nil {
		t.Fatalf("claim = %+v, %v; want nothing: the store holds the blob", cl, err)
	}
	if len(held.asked) != 1 || !slices.Equal(held.asked[0], []string{goneRef}) {
		t.Errorf("asked %v; want the bundle's reference alone", held.asked)
	}
	held.err = errors.New("the controller did not answer")
	cl, err := c.Claim(t.Context(), "m", fp1, manifestD, ClaimOptions{Trunk: true, Stored: held.stored})
	if cl != nil || err == nil || !strings.Contains(err.Error(), "cannot be asked") || !strings.Contains(err.Error(), "left done") {
		t.Errorf("claim = %+v, %v; want the key left done and said", cl, err)
	}
	if st.revs[key] != rev {
		t.Errorf("the key moved to revision %d (was %d)", st.revs[key], rev)
	}
}

// 1937/AC3 (unit, the builder): the builder's look at a module whose done key's blob is gone builds it again, says so
// in a log line and in the done key's step, and asks the store once per process while it answers held.
func TestTheBuilderBuildsAgainADoneKeyWhoseBlobIsGone(t *testing.T) {
	r := newRig(t, goBundle)
	log := &logged{}
	r.b.Logf = log.logf
	held := &fakeHeld{gone: map[string]bool{goneRef: true}}
	r.b.Stored = held.stored
	rev := putBuild(t, r.st, r.key, doneKeyNaming(goneRef))
	v := r.consider(t, "modules/m")
	if v.State != delivery.BuildDone || r.st.revs[r.key] <= rev || len(r.bm.callsOf("build_go_build")) == 0 {
		t.Fatalf("build = %+v at revision %d (was %d); want built again", v, r.st.revs[r.key], rev)
	}
	if !strings.Contains(v.Step, "built again: the artifact store no longer held "+goneRef) {
		t.Errorf("the done key's step %q does not say it was built again for the gone blob", v.Step)
	}
	if !log.has(r.key, "built again", goneRef) {
		t.Errorf("no log line says the key was built again: %v", log.lines)
	}
	// The new revision's blob is held: asked once, then not again in this process.
	held.mu.Lock()
	held.gone = map[string]bool{}
	held.mu.Unlock()
	rev = r.st.revs[r.key]
	asks := held.asks()
	r.consider(t, "modules/m")
	r.consider(t, "modules/m")
	if r.st.revs[r.key] != rev {
		t.Errorf("a key whose blob is held was built again")
	}
	if held.asks() != asks+1 {
		t.Errorf("asked the store %d times for two looks at a held key; want once", held.asks()-asks)
	}
}

// 1937/AC4 (unit, the builder): artifacts-gone for the module's fingerprint makes the builder ask again, even for a key
// it found held before, and build it again; one for another fingerprint builds nothing.
func TestArtifactsGoneMakesTheBuilderAskAndBuildAgain(t *testing.T) {
	r := newRig(t, goBundle)
	log := &logged{}
	r.b.Logf = log.logf
	held := &fakeHeld{gone: map[string]bool{}}
	r.b.Stored = held.stored
	putBuild(t, r.st, r.key, doneKeyNaming(goneRef))
	if v := r.consider(t, "modules/m"); v.State != delivery.BuildDone || len(r.bm.callsOf("build_go_build")) != 0 {
		t.Fatalf("a key whose blob is held was built: %+v", v)
	}
	rev := r.st.revs[r.key]

	held.mu.Lock()
	held.gone[goneRef] = true // collected after the builder's look; the controller's take-in found it
	held.mu.Unlock()
	r.b.ArtifactsGone(context.Background(), delivery.ArtifactsGone{Event: delivery.ArtifactsGoneEvent, Module: "m",
		Fingerprint: "src2:" + strings.Repeat("9", 64), Gone: []string{goneRef}})
	r.b.Wait()
	if r.st.revs[r.key] != rev || !log.has("a fingerprint the module does not have now") {
		t.Fatalf("an event for another fingerprint moved the key (revision %d, was %d): %v", r.st.revs[r.key], rev, log.lines)
	}

	r.b.ArtifactsGone(context.Background(), delivery.ArtifactsGone{Event: delivery.ArtifactsGoneEvent, Module: "m",
		Key: r.key, Fingerprint: fp1, Gone: []string{goneRef}})
	r.b.Wait()
	v := valueOf(t, r.st, r.key)
	if v.State != delivery.BuildDone || r.st.revs[r.key] <= rev || len(r.bm.callsOf("build_go_build")) == 0 {
		t.Fatalf("build = %+v at revision %d (was %d); want built again on the event", v, r.st.revs[r.key], rev)
	}
	if !log.has("heard artifacts-gone") || !log.has(r.key, "built again", goneRef) {
		t.Errorf("the builder did not say what it heard and did: %v", log.lines)
	}
}