diff --git a/cmd/mesh-controller/asker_retry_integration_test.go b/cmd/mesh-controller/asker_retry_integration_test.go index 5e323b58..2ea3177f 100644 --- a/cmd/mesh-controller/asker_retry_integration_test.go +++ b/cmd/mesh-controller/asker_retry_integration_test.go @@ -23,7 +23,7 @@ import ( func TestAWarrantForRetryRetriesAFailedPlanInTheStoreAndEndsItsCondition(t *testing.T) { open := aMesh(t) ctx := t.Context() - asked := asksRecorded(t) + asksRecorded(t) planMovedHeard(t) // the bus the verb says its moves on, and the hand-act log twoTiers(t, open) before := time.Now().UTC().Add(-time.Hour) @@ -123,12 +123,9 @@ func TestAWarrantForRetryRetriesAFailedPlanInTheStoreAndEndsItsCondition(t *test t.Fatal(err) } m := p.Modules["a"] - if p.State != inventory.PlanBuilding || p.Tier != 0 || m.State != "asked" || m.Build == "" || m.Build == "build-1" { + if p.State != inventory.PlanBuilding || p.Tier != 0 || m.State != "asked" || !m.Key || m.Why != "waits for the builder's key" || m.Build == "build-1" { t.Fatalf("after the warrant the plan is %s at tier %d and a is %+v", p.State, p.Tier, m) } - if len(*asked) != 1 { - t.Fatalf("the build was asked %d times: %v", len(*asked), *asked) - } if got := store[w.Ask]; got.Acted != "done" { t.Fatalf("the ask ended %q", got.Acted) } diff --git a/cmd/mesh-controller/away_integration_test.go b/cmd/mesh-controller/away_integration_test.go index c1d37e74..45156cdd 100644 --- a/cmd/mesh-controller/away_integration_test.go +++ b/cmd/mesh-controller/away_integration_test.go @@ -8,6 +8,7 @@ import ( "fmt" "strings" "sync" + "sync/atomic" "testing" "time" @@ -15,6 +16,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/builderkeys" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/testbus" @@ -68,6 +70,16 @@ func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) { var mu sync.Mutex var sends [][]string everyKeyDone(t) // the builds of the tiers: each module is taken from a done key as soon as the walk reaches it + // Except tier 1's module b, whose key is not built until the test says so: a walk that takes it in the same look + // goes on through tier 1's send and overwrites the note of the move before the test reads it (the old build ask + // was where the walk paused). + wholeKeys := openPlanKeys + var holdB atomic.Bool + holdB.Store(true) + openPlanKeys = func() (planKeyReads, func(), error) { + all, done, err := wholeKeys() + return keysHolding{all, "b", &holdB, ""}, done, err + } digests := 0 wasSend := sendRollout sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { @@ -189,7 +201,9 @@ func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) { if !strings.Contains(moved, "laptop was away: sent when it is heard again") || strings.Contains(moved, "anchor") { t.Errorf("1161/AC2: the plan says %q; want laptop named away and not anchor", moved) } - // Tier 1's module is taken from its key, which the walk's look does by itself. + // Tier 1's module is taken from its key once that is built, which the walk's look does by itself. + holdB.Store(false) + askToLook() eventually(t, "tier 1 sent to anchor alone", func() bool { mu.Lock() defer mu.Unlock() @@ -211,3 +225,39 @@ func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) { t.Errorf("1161/AC2: the plan keeps %v away; want laptop", got) } } + +// keysHolding is the stand-in builder's keys but for one module, whose key is not built while held; only the key of +// the fingerprint fp when that is given. +type keysHolding struct { + planKeyReads + module string + held *atomic.Bool + fp string +} + +func (k keysHolding) Build(ctx context.Context, module, fp string) (builderkeys.Key, error) { + if module == k.module && k.held.Load() && (k.fp == "" || fp == k.fp) { + return builderkeys.Key{Module: module, Fingerprint: fp}, nil + } + return k.planKeyReads.Build(ctx, module, fp) +} + +// holdKeys has the stand-in builder's keys of these modules (at any commit) unbuilt until the returned flag is cleared, +// so a walk that took them stays on its tier as it did while its build ran. It restores what it replaced at the end of +// the test, and the flag is set until the test lowers it. +func holdKeys(t *testing.T, modules ...string) *atomic.Bool { + t.Helper() + held := &atomic.Bool{} + held.Store(true) + was := openPlanKeys + openPlanKeys = func() (planKeyReads, func(), error) { + all, done, err := was() + var keys planKeyReads = all + for _, m := range modules { + keys = keysHolding{keys, m, held, ""} + } + return keys, done, err + } + t.Cleanup(func() { openPlanKeys = was }) + return held +} diff --git a/cmd/mesh-controller/backups_before_send_integration_test.go b/cmd/mesh-controller/backups_before_send_integration_test.go index 2baca74b..33e30492 100644 --- a/cmd/mesh-controller/backups_before_send_integration_test.go +++ b/cmd/mesh-controller/backups_before_send_integration_test.go @@ -211,8 +211,12 @@ func TestADeliveryThatMovesABackedUpModuleWaitsForItsBackupAndAFailedOneHoldsItA } seats["laptop"].answers("notes", true, "") advancePlans(ctx, open) - if got := read("plan-two"); got.Backups.State != inventory.BackupDone || len(*asked) != 2 { - t.Fatalf("the plan did not go on: %+v builds=%v", got.Backups, *asked) + // One key taken, not two: plan-one and plan-two carry the same module at the same commit, so they share one key + // build id (key_takein.go keyBuildID), and a walk finds a key already taken (keyTakes.taken, plan_keys.go) and takes + // it in no more. So that the smaller count hides no plan that stopped, plan-two's module must be built from that key. + if got := read("plan-two"); got.Backups.State != inventory.BackupDone || len(*asked) != 1 || + got.Modules["notes"].Build == "" || got.Modules["notes"].State == "failed" || got.Modules["notes"].Key && got.Modules["notes"].State == "asked" { + t.Fatalf("the plan did not go on: %+v module=%+v builds=%v", got.Backups, got.Modules["notes"], *asked) } if said, _ := probeBackupHeld(ctx, &doctor{open: open}); len(said) != 0 { t.Fatalf("the condition stays: %+v", said) diff --git a/cmd/mesh-controller/batches_integration_test.go b/cmd/mesh-controller/batches_integration_test.go index d7afe0af..38c83a2b 100644 --- a/cmd/mesh-controller/batches_integration_test.go +++ b/cmd/mesh-controller/batches_integration_test.go @@ -8,6 +8,7 @@ import ( "os" "slices" "strings" + "sync/atomic" "testing" "time" @@ -65,7 +66,7 @@ func cutAt(t *testing.T, open *stores, now time.Time) { func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) { open := windowed(t) everyHeadWaits(t) - asked := asksWithPaths(t) + asksWithPaths(t) claude := catalogueMerge("553b7191claude", "app", t0) dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second)) hear(t, open, claude, t0.Add(time.Second)) @@ -96,8 +97,12 @@ func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) { if w.Delivery == nil || !slices.EqualFunc(w.Delivery.Merges, want, sameMerge) { t.Fatalf("the walk answers %+v, want %+v", w.Delivery, want) } - if len(*asked) != 2 || (*asked)[0][2] != "main" || (*asked)[1][2] != "main" { - t.Fatalf("the walk did not ask its modules at the branch, which holds the commit it carries: %v", *asked) + // The cut takes its first tier from the builder's keys (a look takes them in later; none runs here), and says at + // which commit: the one the branch holds, which is the later merge the walk carries. + for _, m := range w.Tiers[w.Tier] { + if s := w.Modules[m]; s == nil || !s.Key || s.State != "asked" || s.KeyAt != dunst.Commit { + t.Fatalf("the walk did not take %s from its key at the commit it carries (%s): %+v", m, dunst.Commit, s) + } } // Heard again, as the bus may hand it over twice: never a second merge, nor a second walk. hear(t, open, claude, t0.Add(5*time.Minute)) @@ -131,11 +136,10 @@ func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) { t.Fatalf("the walk does not build %s: %v", r, w.Modules) } } - // Each module's ask is recorded at its own repository's commit. - for _, r := range []string{"one", "two", "three"} { - q, found, err := open.inventory.BuildRequestByID(t.Context(), w.Modules[r].Build) - if err != nil || !found || q.Commit != "c-"+r { - t.Fatalf("%s's build was not recorded at its own commit: %+v %v %v", r, q, found, err) + // Each module of the first tier is taken from its key at its own repository's commit (the cut asks no build). + for _, r := range w.Tiers[w.Tier] { + if s := w.Modules[r]; s == nil || !s.Key || s.State != "asked" || s.KeyAt != "c-"+r { + t.Fatalf("%s was not taken from its key at its own commit: %+v", r, s) } } } @@ -340,7 +344,7 @@ func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) { func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) { open := windowed(t) everyHeadWaits(t) - asked := asksWithPaths(t) + asksWithPaths(t) ctx := t.Context() // Made before the cut marks their modules seen, as merges are: walked again, they are news all the same. oldest := catalogueMerge("aaaa0001", "app", t0.Add(-3*time.Minute)) @@ -367,9 +371,10 @@ func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) { !sameMerge(alone.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) { t.Fatalf("the newest earlier merge was not walked alone on its own commit: %+v", alone) } - if _, in := alone.Modules["app"]; !in || (*asked)[len(*asked)-1] != [3]string{"novox/mesh-catalog", "modules/app", - middle.Commit} { - t.Fatalf("the merge walked alone does not build app at its own commit: %v, asked %v", alone.Modules, *asked) + // The cut takes the first tier from the builder's keys and asks no build, so nothing is recorded as asked: the + // module is taken from its key at the commit the merge was walked on. + if app := alone.Modules["app"]; app == nil || !app.Key || app.State != "asked" || app.KeyAt != middle.Commit { + t.Fatalf("the merge walked alone does not build app at its own commit: %v", alone.Modules) } // Retried, the failed walk would name merges the search answers now. if _, err := retryPlan(ctx, open, failed.ID); err == nil || !strings.Contains(err.Error(), "walked alone") { @@ -508,6 +513,14 @@ func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) { func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) { open := windowed(t) asked := asksWithPaths(t) + // The started walk's key (one at c1) is not built, so that walk stays started as it did while its build ran: with + // the stand-in builder's key done at once, one step would carry it to done and nothing would be beside the other. + wholeKeys, held := openPlanKeys, &atomic.Bool{} + held.Store(true) + openPlanKeys = func() (planKeyReads, func(), error) { + all, done, err := wholeKeys() + return keysHolding{all, "one", held, fpOfModuleAt("one", "c1")}, done, err + } ctx := t.Context() hear(t, open, repoMerge("one", "c1", t0), t0) cutAt(t, open, t0.Add(2*time.Minute)) @@ -541,7 +554,11 @@ func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) { t.Fatalf("a deferred walk is watched as a tier running late: %+v", f) } } - done := started[0] + // Read again: the steps since wrote the started walk (what its key waits for), so the copy read at its cut is stale. + done, err := open.inventory.PlanByID(ctx, started[0].ID) + if err != nil { + t.Fatal(err) + } done.State = inventory.PlanDone if err := open.inventory.SavePlan(ctx, &done); err != nil { t.Fatal(err) @@ -624,6 +641,14 @@ func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) { // A catalogue walk waiting for its word when an own-path batch is cut keeps waiting beside the own-path // walk, is not folded into it, and its word is taken meanwhile: it starts once the own-path walk ended. catalogueWalk := ws[0] + // The controller's second walk keeps its key unbuilt, so it stays started as it did while its build ran; with the + // stand-in builder's key done at once one step would carry it to done and nothing would be beside the catalogue walk. + wholeKeys, held := openPlanKeys, &atomic.Bool{} + held.Store(true) + openPlanKeys = func() (planKeyReads, func(), error) { + all, done, err := wholeKeys() + return keysHolding{all, "mesh-controller", held, fpOfModuleAt("mesh-controller", "k2")}, done, err + } hear(t, open, repoMerge("mesh-controller", "k2", t0.Add(4*time.Minute)), t0.Add(4*time.Minute)) cutAt(t, open, t0.Add(6*time.Minute)) ws, _ = walks(t, open) @@ -642,7 +667,11 @@ func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) { if len(*asked) != before { t.Fatalf("the catalogue walk started beside the controller's: asked %v", (*asked)[before:]) } - own2 := ws[0] + // Read again: the step above wrote the walk (what its key waits for), so the copy read at its cut is stale. + own2, err := open.inventory.PlanByID(ctx, ws[0].ID) + if err != nil { + t.Fatal(err) + } own2.State = inventory.PlanDone if err := open.inventory.SavePlan(ctx, &own2); err != nil { t.Fatal(err) diff --git a/cmd/mesh-controller/beside_test.go b/cmd/mesh-controller/beside_test.go index 0d417830..d3626c82 100644 --- a/cmd/mesh-controller/beside_test.go +++ b/cmd/mesh-controller/beside_test.go @@ -4,6 +4,7 @@ package main import ( "strings" + "sync/atomic" "testing" "time" @@ -15,6 +16,14 @@ import ( func TestAnUnrelatedBatchWalksBesideAnOpenWalk(t *testing.T) { open := windowed(t) asksWithPaths(t) + // Both modules' keys are not built, so each walk stays open as it did while its build ran: with the stand-in + // builder's key done at once, one step would carry a one-module walk to done. + wholeKeys, held := openPlanKeys, &atomic.Bool{} + held.Store(true) + openPlanKeys = func() (planKeyReads, func(), error) { + all, done, err := wholeKeys() + return keysHolding{keysHolding{all, "one", held, ""}, "two", held, ""}, done, err + } ctx := t.Context() hear(t, open, repoMerge("one", "c1", t0), t0) cutAt(t, open, t0.Add(90*time.Second)) diff --git a/cmd/mesh-controller/busobjects_test.go b/cmd/mesh-controller/busobjects_test.go index c40758f6..53a9145e 100644 --- a/cmd/mesh-controller/busobjects_test.go +++ b/cmd/mesh-controller/busobjects_test.go @@ -46,25 +46,33 @@ func aCarriedConsumer(t *testing.T, open *stores) broker.Consumer { return broker.Consumer{} } -// aLateHolder is a module holding the build agent's seat on anchor, assigned after the controller -// started, and the worker its seat's queue should have for it. +// aLateHolder is a module holding a build seat (node-build-go) on anchor, assigned after the controller started, and +// the worker its seat's queue should have for it: a build seat is pulled by kind, so the worker is the one for the +// seat's first accepted kind (the build agent's seat, which had one worker for all, is gone with the builder). func aLateHolder(t *testing.T, open *stores) broker.Consumer { t.Helper() + // With an account to read, as a holder has: a module with none is no bus user, so its seat's workers are not derived. register(t, open, catalogue.Manifest{Module: "late-builder", Version: "1", - Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}}) + OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh/late-builder/broker"}}, + Claims: []catalogue.Claim{{Name: "node-build-go", Scope: catalogue.ScopeNode}}}) if _, err := open.inventory.Assign(t.Context(), "anchor", "late-builder"); err != nil { t.Fatal(err) } + _, workers, err := seatTrafficObjects(t.Context(), open.inventory) + if err != nil { + t.Fatal(err) + } for _, s := range inventory.MeshSeats() { - if s.Name == "node-build-agent" { - c, needed := broker.HolderConsumerFor("anchor", "late-builder", s) - if !needed { - t.Fatal("the build agent's seat needs no worker") + if s.Name != "node-build-go" || len(s.Accepts) == 0 { + continue + } + for _, w := range workers { + if w.Name == broker.WorkerName(s.Name, s.Accepts[0]) { + return w } - return c } } - t.Fatal("the mesh declares no build agent's seat") + t.Fatal("the build seat's holder is derived no worker") return broker.Consumer{} } diff --git a/cmd/mesh-controller/delivery_integration_test.go b/cmd/mesh-controller/delivery_integration_test.go index a50bf4ec..fa0c04cf 100644 --- a/cmd/mesh-controller/delivery_integration_test.go +++ b/cmd/mesh-controller/delivery_integration_test.go @@ -72,7 +72,10 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin } // No holder on record: the merge starts its walk, as before. + // A cut takes its first tier from the builder's keys and a step of the walk records the take (the walk asks no + // build), so each walk that starts is stepped once before its takes are counted. p := merge("c1aaaaaaaa", "modules/app/index.ts") + advanceHeld(ctx, open) if p.Waiting() || len(*asked) != 1 { t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked) } @@ -116,6 +119,7 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin // ADR 0276): the merge's batch is cut once it ended. finish(p.ID) p = merge("c3cccccccc", "modules/mesh-delivery/main.go") + advanceHeld(ctx, open) if p.Waiting() || len(*asked) != 3 { t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked) } @@ -157,7 +161,9 @@ func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testin t.Fatal(err) } asks := len(*asked) - if p = merge("c6ffffffff", "modules/app/index.ts"); p.Waiting() || len(*asked) != asks+1 { + p = merge("c6ffffffff", "modules/app/index.ts") + advanceHeld(ctx, open) + if p.Waiting() || len(*asked) != asks+1 { t.Fatalf("with the holder gone the walk waited: %v", p.Waiting()) } } diff --git a/cmd/mesh-controller/held.go b/cmd/mesh-controller/held.go new file mode 100644 index 00000000..20e8bb97 --- /dev/null +++ b/cmd/mesh-controller/held.go @@ -0,0 +1,128 @@ +package main + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + + "github.com/novox/mesh-controller/internal/artifacts" + "github.com/novox/mesh-controller/internal/builderkeys" + "github.com/novox/mesh-controller/internal/link" +) + +// The verb `held` and the event `artifacts-gone` (novox/hq issue 1937). +// +// **A done key can name a blob the store no longer holds.** A build module pushes a bundle or an archive as a bare blob +// (ADR 0498 §1); the store's nightly collector deletes every blob no manifest names; and only a take-in or a sweep holds +// one by its manifest. A key the controller has not taken in by the next collection names a blob that is gone. On +// 2026-10-24 that was node-tools', mesh-host's and the forge bridge's. Since issue 1933 the take-in refuses such a key; +// these two let the builder build it again: `held` before it reuses a done key, `artifacts-gone` when a take-in found it. + +// heldMost is the most references one ask of held judges: a builder asks for one key's artifacts at a time. +const heldMost = 64 + +// heldAsk asks the store whether it holds one reference: nil, artifacts.Gone or another error. A variable so a test +// needs no store. +var heldAsk = func(ctx context.Context, address, reference string) error { + return artifacts.Store{Address: address}.Holds(ctx, reference) +} + +// heldAddress is the artifact store's address the serving controller asks. A variable so a test needs no records. +var heldAddress = func(ctx context.Context) (string, error) { + d := doctorFrom + if d == nil || d.open == nil || d.open.inventory == nil { + return "", errors.New("this controller is not serving, so it asks the artifact store nothing: ask again, and " + + "the serving controller answers") + } + shelf, err := d.open.inventory.Catalogue(ctx) + if err != nil { + return "", err + } + address, err := artifactStoreAddress(ctx, d.open.inventory, shelf, "") + if err != nil { + return "", err + } + if address == "" { + return "", errors.New("the artifact store is on no network the controller reaches") + } + return address, nil +} + +// heldAnswer is the verb `held`: each reference given, held, gone (the store answered 404) or unknown with why. It +// holds and writes nothing: a reference the store holds and no record names is left for the take-in to hold. +func heldAnswer(ctx context.Context, given string) (any, error) { + var refs []string + for _, r := range strings.Split(given, ",") { + if r = strings.TrimSpace(r); r != "" && !slices.Contains(refs, r) { + refs = append(refs, r) + } + } + if len(refs) == 0 { + return nil, errors.New("held judges the references given, and none was") + } + if len(refs) > heldMost { + return nil, fmt.Errorf("held judges at most %d references at once, and %d were given", heldMost, len(refs)) + } + for _, r := range refs { + if _, ours := catalogue.InArtifactStore(r); !ours { + return nil, fmt.Errorf("%s is not a reference into the mesh's artifact store (artifact-store://...): nothing was asked", r) + } + } + address, err := heldAddress(ctx) + if err != nil { + return nil, err + } + asking, stop := context.WithTimeout(ctx, 20*time.Second) + defer stop() + a := sdkdelivery.HeldAnswer{Store: address, Held: []string{}, Gone: []string{}} + for _, r := range refs { + switch err := heldAsk(asking, address, r); { + case err == nil: + a.Held = append(a.Held, r) + case errors.Is(err, artifacts.Gone): + a.Gone = append(a.Gone, r) + default: + if a.Unknown == nil { + a.Unknown = map[string]string{} + } + a.Unknown[r] = err.Error() + } + } + out := fmt.Sprintf("the artifact store at %s holds %d of the %d reference(s) asked", address, len(a.Held), len(refs)) + if len(a.Gone) > 0 { + out += "; gone: " + strings.Join(a.Gone, ", ") + } + if len(a.Unknown) > 0 { + out += fmt.Sprintf("; %d could not be asked", len(a.Unknown)) + } + return verbAnswer{Output: out + "\n", OK: true, Answer: a}, nil +} + +// sayArtifactsGone says `artifacts-gone` for a builder's key the take-in refused because the store does not hold what +// it names (issue 1937): the builder hears it and builds the key again. Said from the serving controller only, and never +// in the take-in's way: a bus that cannot carry it is said in the journal. +var sayArtifactsGone = func(ctx context.Context, k builderkeys.Key, module string, gone []string) { + bus := checkEvents + if bus == nil || len(gone) == 0 { + return + } + key, _ := sdkdelivery.BuilderBuildKey(module, k.Fingerprint) + raw, err := sdkdelivery.EncodeArtifactsGone(sdkdelivery.ArtifactsGone{Module: module, Key: key, + Fingerprint: k.Fingerprint, Gone: gone, At: time.Now().UTC()}) + if err != nil { + fmt.Printf("%s: its key names archives the store does not hold, and that could not be said: %v\n", module, err) + return + } + saying, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer stop() + if err := bus.PublishSeatEvent(saying, link.MeshControllerSeat, link.KeyArtifactsGone, raw); err != nil { + fmt.Printf("%s: its key names archives the store does not hold, and %s could not be said: %v\n", + module, link.KeyArtifactsGone, err) + } +} diff --git a/cmd/mesh-controller/held_test.go b/cmd/mesh-controller/held_test.go new file mode 100644 index 00000000..68f129f9 --- /dev/null +++ b/cmd/mesh-controller/held_test.go @@ -0,0 +1,132 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "slices" + "strings" + "sync" + "testing" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + + "github.com/novox/mesh-controller/internal/artifacts" + "github.com/novox/mesh-controller/internal/link" +) + +// 1937/AC1: held answers each reference held, gone (the store's 404) or unknown, asks only HEADs, and its answer is what +// the builder reads (mesh-sdk delivery.DecodeHeldAnswer). +func TestHeldAnswersEachReferenceHeldOrGone(t *testing.T) { + present := "artifact-store://node-tools/runtime/blobs/" + shaOf("held") + missing := "artifact-store://node-tools/runtime/blobs/" + shaOf("gone") + broken := "artifact-store://mesh-host/host-arch/blobs/" + shaOf("broken") + address, ask := heldAddress, heldAsk + t.Cleanup(func() { heldAddress, heldAsk = address, ask }) + heldAddress = func(context.Context) (string, error) { return "store.test:5100", nil } + var asked []string + var mu sync.Mutex + heldAsk = func(_ context.Context, at, reference string) error { + mu.Lock() + asked = append(asked, at+" "+reference) + mu.Unlock() + switch reference { + case missing: + return artifacts.Gone + case broken: + return errors.New("the artifact store answered 500") + } + return nil + } + got, err := heldAnswer(t.Context(), present+", "+missing+","+broken+","+present) + if err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(got) + if err != nil { + t.Fatal(err) + } + a, err := sdkdelivery.DecodeHeldAnswer(raw, []string{present, missing, broken}) + if err != nil { + t.Fatalf("the builder cannot read held's answer: %v\n%s", err, raw) + } + if !slices.Equal(a.Held, []string{present}) || !slices.Equal(a.Gone, []string{missing}) || a.Unknown[broken] == "" { + t.Errorf("held answered %+v", a) + } + if len(asked) != 3 { + t.Errorf("held asked the store %d times for three distinct references: %v", len(asked), asked) + } + if !strings.Contains(got.(verbAnswer).Output, missing) { + t.Errorf("held's words do not name the gone reference: %s", got.(verbAnswer).Output) + } + + for _, refused := range []string{"", "registry.example/x@sha256:00"} { + if _, err := heldAnswer(t.Context(), refused); err == nil { + t.Errorf("held took %q", refused) + } + } + // A verb answered in the serving process, never by a command line. + if !inProcess["held"] { + t.Error("held is not answered by the serving controller") + } + if _, err := argvFor("held", map[string]any{"references": present}); err == nil { + t.Error("held composed a command line") + } +} + +// saidGone keeps the artifacts-gone events said. +type saidGone struct { + link.Bus // only PublishSeatEvent is said on + mu sync.Mutex + said []sdkdelivery.ArtifactsGone +} + +func (s *saidGone) PublishSeatEvent(_ context.Context, seat, event string, body []byte) error { + if seat != link.MeshControllerSeat || event != link.KeyArtifactsGone { + return errors.New("said under the wrong seat or name: " + seat + " " + event) + } + e, err := sdkdelivery.DecodeArtifactsGone(body) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + s.said = append(s.said, e) + return nil +} + +// 1937/AC2: a take-in refused for a gone archive names every gone one and says artifacts-gone with the module, the key, +// its fingerprint and the gone references, so the builder builds the key again; a hand registration says nothing. +func TestARefusedTakeInSaysArtifactsGone(t *testing.T) { + program := "artifact-store://x/tools/blobs/" + shaOf("t") + image := "artifact-store://x/img@" + shaOf("i") + keys, trunk := aKeyOf(tsBundle, 3, + sdkdelivery.BuiltArtifact{Name: "tools", Kind: "bundle", Digest: shaOf("t"), Reference: program}, + sdkdelivery.BuiltArtifact{Name: "img", Kind: "image", Digest: shaOf("i"), Reference: image}) + bus := &saidGone{} + was := checkEvents + checkEvents = bus + t.Cleanup(func() { checkEvents = was }) + gone := func(context.Context, string) error { return artifacts.Gone } + + _, _, err := buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk}, keyContext{hold: gone}) + if err == nil || !strings.Contains(err.Error(), program) || !strings.Contains(err.Error(), image) || + !strings.Contains(err.Error(), "not in the artifact store") { + t.Fatalf("a take-in of a key whose archives are gone: %v", err) + } + if len(bus.said) != 1 { + t.Fatalf("said %d artifacts-gone events; want one", len(bus.said)) + } + e := bus.said[0] + wantKey, _ := sdkdelivery.BuilderBuildKey("x", keyFP) + if e.Module != "x" || e.Fingerprint != keyFP || e.Key != wantKey || !slices.Equal(e.Gone, []string{image, program}) { + t.Errorf("said %+v", e) + } + + bus.said = nil + _, _, _ = buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk, Hand: map[string]string{"tools": program}}, + keyContext{hold: gone}) + if len(bus.said) != 0 { + t.Errorf("a registration by hand said %v: no builder's key was refused", bus.said) + } +} diff --git a/cmd/mesh-controller/key_takein.go b/cmd/mesh-controller/key_takein.go index c958d6b7..3f00bbc2 100644 --- a/cmd/mesh-controller/key_takein.go +++ b/cmd/mesh-controller/key_takein.go @@ -351,6 +351,9 @@ func buildOfKey(ctx context.Context, keys keyReads, t keyTake, kc keyContext) ( // archive the store does not hold is sent, and every machine assigned the module fails its apply on the 404. An // archive is held by its manifest, so the store's own collector keeps it too. if kc.hold != nil { + // **Every gone archive named, and said** (issue 1937): the builder hears `artifacts-gone` and builds the key + // again, so the refusal is the last word on this key, not a plan that fails at every retry. + var gone, goneNames []string for _, a := range k.Build.Artifacts { reference := catalogue.Recorded(a.Reference) if _, ours := catalogue.InArtifactStore(reference); !ours { @@ -360,13 +363,21 @@ func buildOfKey(ctx context.Context, keys keyReads, t keyTake, kc keyContext) ( switch { case err == nil, errors.Is(err, artifacts.ErrNotOurs): case errors.Is(err, artifacts.Gone): - return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder's key of %s names %s for its "+ - "artifact %q, which is not in the artifact store: nothing was recorded or registered, and the key "+ - "must be built again", t.Module, reference, a.Name) + gone = append(gone, reference) + goneNames = append(goneNames, fmt.Sprintf("%s for its artifact %q", reference, a.Name)) default: return waits("the artifact store could not hold %s's artifact %q (%s): %v", t.Module, a.Name, reference, err) } } + if len(gone) > 0 { + if t.Hand == nil { + sayArtifactsGone(ctx, k, t.Module, gone) + } + return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder's key of %s names %s, not in "+ + "the artifact store: nothing was recorded or registered, and the key must be built "+ + "again (said as %s, which the builder builds it again on)", t.Module, strings.Join(goneNames, " and "), + link.KeyArtifactsGone) + } } built, err := builtOfKey(ctx, kc.archive, manifest, k) if err != nil { diff --git a/cmd/mesh-controller/one_group_one_plan_test.go b/cmd/mesh-controller/one_group_one_plan_test.go index 60fd5201..f37d07b0 100644 --- a/cmd/mesh-controller/one_group_one_plan_test.go +++ b/cmd/mesh-controller/one_group_one_plan_test.go @@ -94,8 +94,10 @@ func TestAFeatureWithANewSeatVerbIsOneBatchAndOnePlanInOrderWithNoAfterLine(t *t if plan.Delivery != nil && (plan.Delivery.Awaits != "" || plan.Waiting()) { t.Fatalf("a plan carrying the controller waits for %q's word", plan.Delivery.Awaits) } - if plan.Modules["mesh-controller"].Build == "" || len(*asked) != 1 { - t.Fatalf("the first tier is asked once: build %q, %d ask(s) %v", plan.Modules["mesh-controller"].Build, len(*asked), *asked) + // The cut takes the first tier from the controller's key at its commit and asks no build; the take-in itself is a + // later look's, which this test does not run, so nothing is recorded as asked. + if c := plan.Modules["mesh-controller"]; !c.Key || c.State != "asked" || c.KeyAt != "k1" || len(*asked) != 0 { + t.Fatalf("the first tier is not taken from its key once: %+v, %d ask(s) %v", c, len(*asked), *asked) } // Every merge is answered by that one plan. for _, repository := range []string{"novox/mesh-controller", "novox/mesh-catalog"} { diff --git a/cmd/mesh-controller/pending_integration_test.go b/cmd/mesh-controller/pending_integration_test.go index 1f2b246a..306ae903 100644 --- a/cmd/mesh-controller/pending_integration_test.go +++ b/cmd/mesh-controller/pending_integration_test.go @@ -151,7 +151,9 @@ func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) { if len(pending) != 1 || pending[0].State != inventory.PendingExpired { t.Fatalf("pending after a failed build: %+v", pending) } - for _, want := range []string{"expired", "build b-modules/sensors of modules/sensors failed", + // Its words since R4g (whyNotBuilt): the build, when it was heard, and that it failed (issue 1938: kept from an older + // run until the inventory changed beneath it). + for _, want := range []string{"expired", "modules/sensors in novox/mesh-catalog; its last build, b-modules/sensors at", ", failed: go build", "undefined: sensorsRead", "nothing was assigned", "build \"true\""} { if !strings.Contains(pending[0].Note, want) { t.Fatalf("the expiry does not say %q: %s", want, pending[0].Note) @@ -266,9 +268,18 @@ func TestAPlansBuildRequestIsThePlans(t *testing.T) { if err := (following{open: open}).SourceMoved(ctx, m); err != nil { t.Fatal(err) } + // A walk asks no build, so it leaves no build request (askModule takes the module from its key): the plan's app is + // taken from the key at the commit the plan carries. requests, err := open.inventory.RequestsNamed(ctx, "app") - if err != nil || len(requests) != 1 || requests[0].For != "plan" { - t.Fatalf("the plan's request: %+v %v", requests, err) + if err != nil || len(requests) != 0 { + t.Fatalf("a request was recorded for the plan: %+v %v", requests, err) + } + recent, err := open.inventory.RecentPlans(ctx, 5) + if err != nil || len(recent) != 1 { + t.Fatalf("the plans: %+v %v", recent, err) + } + if app := recent[0].Modules["app"]; app == nil || !app.Key || app.KeyAt != "c1aaaaaaaa" { + t.Fatalf("the plan's app is not taken from its key at the commit it carries: %+v", app) } } diff --git a/cmd/mesh-controller/plan_clock_integration_test.go b/cmd/mesh-controller/plan_clock_integration_test.go index 460db64f..51e968d8 100644 --- a/cmd/mesh-controller/plan_clock_integration_test.go +++ b/cmd/mesh-controller/plan_clock_integration_test.go @@ -15,6 +15,9 @@ func TestAPlanStandingStillIsNotSavedAgain(t *testing.T) { ctx := t.Context() inv := open.inventory asked := asksRecorded(t) + // app's key is not built, so the plan stands on its tier as it did while the build ran; with the stand-in builder's + // key done at once, one step would carry it to done. + holdKeys(t, "app") if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { @@ -27,11 +30,12 @@ func TestAPlanStandingStillIsNotSavedAgain(t *testing.T) { } advanceHeld(ctx, open) recent, err := inv.RecentPlans(ctx, 1) - if err != nil || len(recent) != 1 || len(*asked) != 1 { + if err != nil || len(recent) != 1 || len(*asked) != 0 { t.Fatalf("no plan building: %v %v, asked %v", recent, err, *asked) } before := recent[0] - if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" { + if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" || + !before.Modules["app"].Key || before.Modules["app"].KeyAt != "c1aaaaaaaa" { t.Fatalf("the plan is not waiting on its build: %+v", before) } @@ -46,7 +50,7 @@ func TestAPlanStandingStillIsNotSavedAgain(t *testing.T) { t.Fatalf("a plan that did not move was saved again: revision %d → %d, saved %s → %s", before.Revision, after.Revision, before.Updated, after.Updated) } - if len(*asked) != 1 { - t.Fatalf("advancing asked again: %v", *asked) + if len(*asked) != 0 { + t.Fatalf("advancing took the key in: %v", *asked) } } diff --git a/cmd/mesh-controller/reconnects_integration_test.go b/cmd/mesh-controller/reconnects_integration_test.go index 6cf7e7d3..672a9c13 100644 --- a/cmd/mesh-controller/reconnects_integration_test.go +++ b/cmd/mesh-controller/reconnects_integration_test.go @@ -52,7 +52,9 @@ func TestTheServingControllerLendsItsOwnConnection(t *testing.T) { if a := answer.(verbAnswer); !a.OK || a.Answer == nil { t.Fatalf("hand-acts answered %+v", a) } - _, _ = handlers["queue"](context.Background(), json.RawMessage(`{}`)) + // A read-only verb that runs in the serving process, as a verb that reads the bus does: it uses the + // serving connection and opens none. (A verb run as a command re-executes this test binary, which never ends.) + _, _ = handlers["dead-letters"](context.Background(), json.RawMessage(`{}`)) after, _ := bus.Varz(nil) if opened := after.TotalConnections - before.TotalConnections; opened != 0 { t.Fatalf("the serving controller opened %d connection(s) of its own", opened) diff --git a/cmd/mesh-controller/replay614_test.go b/cmd/mesh-controller/replay614_test.go index ffb8015f..4c2f9d9a 100644 --- a/cmd/mesh-controller/replay614_test.go +++ b/cmd/mesh-controller/replay614_test.go @@ -63,11 +63,14 @@ func TestReplay614ANewSDKPlansTheToolchainAndThenItsBundles(t *testing.T) { m := orderMember{ID: "novox/mesh-sdk@new", Repository: "novox/mesh-sdk", Head: "new", Paths: []string{"src/provisioner/index.ts", "package.json"}} s := pullScope(m.pull(), entries, read, edges) - want := [][]string{{"mesh-sdk"}, {"mesh-tools"}, {"minio", "umami"}} + // Since R4g no bundle is compiled in a toolchain image (526eed99, issue 1851 C1b: the toolchain edge is gone), so a + // new SDK plans itself and what stands on its package; the TypeScript bundles move by their own build keys, whose + // fingerprints name what they are built from (issue 1938: this test was kept from older runs until then). + want := [][]string{{"mesh-sdk"}, {"mesh-tools"}} if !reflect.DeepEqual(s.Reach.Plan.Tiers, want) { t.Fatalf("a new SDK plans %v, want %v (edges %v)", s.Reach.Plan.Tiers, want, edges) } - if !reflect.DeepEqual(s.Dependents, []string{"mesh-tools", "minio", "umami"}) { + if !reflect.DeepEqual(s.Dependents, []string{"mesh-tools"}) { t.Errorf("dependents %v", s.Dependents) } } diff --git a/cmd/mesh-controller/replays_integration_test.go b/cmd/mesh-controller/replays_integration_test.go index 37a4531d..76f28db8 100644 --- a/cmd/mesh-controller/replays_integration_test.go +++ b/cmd/mesh-controller/replays_integration_test.go @@ -316,6 +316,8 @@ func TestReplay296APlanStandingStillInItsTierIsNotSavedAgain(t *testing.T) { ctx := t.Context() inv := open.inventory asked := asksRecorded(t) + // app's key is not built, so the plan stands on its tier as it did while the build ran (see plan_clock_integration_test.go). + holdKeys(t, "app") if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { @@ -328,11 +330,12 @@ func TestReplay296APlanStandingStillInItsTierIsNotSavedAgain(t *testing.T) { } advanceHeld(ctx, open) recent, err := inv.RecentPlans(ctx, 1) - if err != nil || len(recent) != 1 || len(*asked) != 1 { + if err != nil || len(recent) != 1 || len(*asked) != 0 { t.Fatalf("no plan waiting on its build: %v %v, asked %v", recent, err, *asked) } before := recent[0] - if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" { + if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" || + !before.Modules["app"].Key || before.Modules["app"].KeyAt != "3ad22356aa" { t.Fatalf("the plan is not waiting on its build: %+v", before) } for range 3 { diff --git a/cmd/mesh-controller/rollout_events_test.go b/cmd/mesh-controller/rollout_events_test.go index c86e12a4..5818cff8 100644 --- a/cmd/mesh-controller/rollout_events_test.go +++ b/cmd/mesh-controller/rollout_events_test.go @@ -31,7 +31,9 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { // A quarter of an hour of silence is two seconds here. wasSilent := applySilentAfter - applySilentAfter = 2 * time.Second + // Ten seconds here: long enough that laptop's key outlives the controller's replacement under a loaded run. anchor's + // report never races it: anchor answers tier 1 inside its send (reportOnSend below). + applySilentAfter = 10 * time.Second t.Cleanup(func() { applySilentAfter = wasSilent }) js, err := broker.Dial(testbus.URL(t)) @@ -70,6 +72,8 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { var sends [][]string asked := asksRecorded(t) // each module taken from a done key as the walk reaches it, said as repository#build id digests := 0 + var reportOnSend func(node string) // set below, once a report can be made + anchorReported := make(chan error, 1) wasSend := sendRollout sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { mu.Lock() @@ -89,6 +93,15 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { return nil, err } } + // anchor answers tier 1 as soon as it is sent (issue 1938): its report is in before the silence bound can run, by + // order and not by a race with the bound, which twice named it silent under a loaded run. laptop never reports. + if len(sends) == 2 && reportOnSend != nil { + for _, node := range names { + if node == "anchor" { + go reportOnSend(node) + } + } + } return names, nil } t.Cleanup(func() { sendRollout = wasSend }) @@ -147,6 +160,26 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { t.Fatal(err) } } + // tell is a report heard as the controller hears it, for a goroutine: its error is handed back, not fatal. + tell := func(node string) error { + last, found, err := inv.SentTo(ctx, node) + if err != nil || !found { + return fmt.Errorf("%s was sent nothing: %v", node, err) + } + mu.Lock() + seq++ + n := seq + mu.Unlock() + body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest, + "report_sequence": n}) + var report link.Report + if err := json.Unmarshal(body, &report); err != nil { + return err + } + _, err = listener.Heard(ctx, report) + return err + } + reportOnSend = func(node string) { anchorReported <- tell(node) } heldKeys := func() map[silentWait]time.Time { c := silentClockOf() if c == nil { @@ -167,6 +200,9 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { return p } + // Tier 1's module b has its key unbuilt while the first controller runs, so the plan stands at tier 1 as it did while + // its build ran; with the key done at once, the same look would take it and send tier 1 as well. + heldB := holdKeys(t, "b") // The first controller. Its plan has tier 0 built and tier 1 not asked: nothing but the save asks anything. first, stopFirst := context.WithCancel(ctx) defer stopFirst() @@ -181,10 +217,14 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { if err := inv.SavePlan(ctx, &p); err != nil { t.Fatal(err) } - eventually(t, "tier 0 sent to both machines and tier 1 asked, on the save alone", func() bool { + eventually(t, "tier 0 sent to both machines and tier 1 taken from its key, on the save alone", func() bool { mu.Lock() defer mu.Unlock() - return len(sends) == 1 && len(*asked) == 1 && plan().Tier == 1 + // b is taken from its key; the walk carries a commit of novox/mesh-catalog only, and b is registered from + // novox/b, so the key is read at b's own trunk value and KeyAt stays empty (Plan.CommitOn names no other + // repository's commit). + b := plan().Modules["b"] + return len(sends) == 1 && len(*asked) == 0 && plan().Tier == 1 && b != nil && b.Key && b.KeyAt == "" }) eventually(t, "a key on the bus for each machine tier 0 was sent to", func() bool { return len(heldKeys()) == 2 }) @@ -194,6 +234,7 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { time.Sleep(300 * time.Millisecond) reports("anchor") + heldB.Store(false) // b's key is built by the time the new controller looks second, stopSecond := context.WithCancel(ctx) defer stopSecond() waitSecond := startRolloutEvents(second, open, js) @@ -212,7 +253,15 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { defer mu.Unlock() return len(sends) == 2 }) - reports("anchor") + // anchor reported on tier 1 inside its send. + select { + case err := <-anchorReported: + if err != nil { + t.Fatalf("anchor's report on tier 1: %v", err) + } + case <-time.After(15 * time.Second): + t.Fatal("anchor never reported on tier 1") + } // laptop never reports: its key expires and the new controller names it silent, which ends the rollout. eventually(t, "the rollout ended, naming laptop silent and not anchor", func() bool { @@ -228,7 +277,7 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { // No tier asked twice and none skipped: one ask for tier 1 (tier 0 was built when the plan was kept), each tier sent // once to both machines, and said as sent once for each. mu.Lock() - if len(*asked) != 1 || !strings.HasPrefix((*asked)[0], "novox/mesh-catalog#key-") { + if len(*asked) != 1 || !strings.HasPrefix((*asked)[0], "novox/b#key-") { // b is registered from novox/b t.Errorf("taken from keys = %v; want tier 1's module taken once", *asked) } if len(sends) != 2 || strings.Join(sends[0], ",") != "anchor,laptop" || strings.Join(sends[1], ",") != "anchor,laptop" { diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index a4bf2f30..87350b89 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -318,6 +318,8 @@ func (a *verbArguments) commandLine() ([]string, error) { return nil, errors.New("dead-letters is answered by the serving controller, on its own connection, not by a command") case "root-free": return nil, errors.New("root-free is judged by the serving controller, on its own connection, not by a command") + case "held": + return nil, errors.New("held is asked by the serving controller, of the artifact store, not by a command") case "status": return []string{"status", "--json"}, nil case "nodes": @@ -1253,6 +1255,9 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) { if verb == "dead-letters" { return deadLettersAnswer(ctx, a) } + if verb == "held" { + return heldAnswer(ctx, a.given["references"]) + } if verb == "root-free" { return rootFreeAnswer(ctx, a.given["machines"], rootClock()) } @@ -1376,7 +1381,10 @@ var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true, "dead-letters": true, // Whether a machine is root-free, judged live on the serving controller's store and connection (novox/hq ADR // 0259 §8): the router asks it before an approval. - "root-free": true} + "root-free": true, + // Whether the artifact store holds what a builder's key names, asked by the builder before it reuses the key (novox/hq + // issue 1937). + "held": true} // pushFileVar names the file the controller's push settings are written to (module.json, `push-wait`). const pushFileVar = "MESH_PUSH_FILE" diff --git a/cmd/mesh-controller/silent_mid_tier_integration_test.go b/cmd/mesh-controller/silent_mid_tier_integration_test.go index 2593d8b1..4d403b24 100644 --- a/cmd/mesh-controller/silent_mid_tier_integration_test.go +++ b/cmd/mesh-controller/silent_mid_tier_integration_test.go @@ -65,6 +65,9 @@ func TestAPlanPassesATierWhoseMachineNeverReportsAfterTheBoundAndSendsTheNextToT var mu sync.Mutex var sends [][]string asksRecorded(t) // each module is taken from a done key as the walk reaches it + // Except tier 1's module b, whose key is not built until the test says so: a walk that takes it in the same look goes on + // through tier 1's send and overwrites the note of the move before the test reads it (see away_integration_test.go). + heldB := holdKeys(t, "b") digests := 0 wasSend := sendRollout sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { @@ -190,7 +193,12 @@ func TestAPlanPassesATierWhoseMachineNeverReportsAfterTheBoundAndSendsTheNextToT if !strings.Contains(moved, "laptop") || strings.Contains(moved, "anchor") { t.Errorf("1160/AC5: the plan says %q; want laptop named silent and not anchor", moved) } - // Tier 1's module is taken from its key, which the walk's look does by itself. + if b := plan().Modules["b"]; b == nil || !b.Key { + t.Errorf("tier 1's module is not taken from its key: %+v", b) + } + // Tier 1's module is taken from its key once that is built, which the walk's look does by itself. + heldB.Store(false) + askToLook() eventually(t, "tier 1 sent to anchor alone", func() bool { mu.Lock() defer mu.Unlock() diff --git a/cmd/mesh-controller/walk_said_integration_test.go b/cmd/mesh-controller/walk_said_integration_test.go index 25aba361..52f09231 100644 --- a/cmd/mesh-controller/walk_said_integration_test.go +++ b/cmd/mesh-controller/walk_said_integration_test.go @@ -3,14 +3,17 @@ package main import ( + "context" "encoding/json" "os" "testing" "time" "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/nats-io/nats.go" "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/builderkeys" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/testbus" @@ -205,11 +208,17 @@ func TestAWalkStoppedByHandAndRetriedEndsFailedWhenItFailsAgain(t *testing.T) { if p.State != inventory.PlanBuilding || (p.Delivery != nil && p.Delivery.Stopped != "") { t.Fatalf("%s: the walk is %s and keeps its stop: %+v", again[0], p.State, p.Delivery) } - // The build asked fails again: its record says so, and the look settles the walk from it. - if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: p.Modules["a"].Build, Module: "a", Commit: "c0ffee", - Repository: "novox/a", At: time.Now().UTC(), Asked: before, Failed: "it broke again"}); err != nil { - t.Fatal(err) + // The retry takes a from its key (it asks no build), at the commit the walk carries. + if a := p.Modules["a"]; a == nil || !a.Key || a.KeyAt != "c0ffee" { + t.Fatalf("%s: the retried walk does not take a from its key: %+v", again[0], a) } + // The key fails again: the builder's record says so, and the look settles the walk from it. + wholeKeys := openPlanKeys + openPlanKeys = func() (planKeyReads, func(), error) { + all, done, err := wholeKeys() + return keysFailing{all, "a", "it broke again"}, done, err + } + t.Cleanup(func() { openPlanKeys = wholeKeys }) advancePlans(ctx, open) if p, _ = open.inventory.PlanByID(ctx, walk.ID); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed { t.Fatalf("%s: failing again, the walk is %s and ends %s", again[0], p.State, p.Outcome()) @@ -259,3 +268,18 @@ func TestAWalksStartIsSaidAgainWithAMachineWhoseDeclarationChanged(t *testing.T) t.Fatalf("said again: %v", again) } } + +// keysFailing is the stand-in builder's keys but for one module, whose key is failed in these words. +type keysFailing struct { + planKeyReads + module string + words string +} + +func (k keysFailing) Build(ctx context.Context, module, fp string) (builderkeys.Key, error) { + if module == k.module { + return builderkeys.Key{Module: module, Fingerprint: fp, Found: true, Revision: 1, + Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildFailed, Error: k.words}}, nil + } + return k.planKeyReads.Build(ctx, module, fp) +} diff --git a/go.mod b/go.mod index f8f3f062..4e0a9bd9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-controller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 github.com/jackc/pgx/v5 v5.10.0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/go.sum b/go.sum index 5ffa1878..6b35a21b 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,8 @@ git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYg git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 01cf591d..e55ba60f 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -406,6 +406,12 @@ var DegradableGrants = []DegradableGrant{ Reason: "carried only at genesis: the controller composes its own permissions and pushes them to the bus at its " + "first push, and keeps the packages each node declares only after that, holding the lease (ADR 0507 §5, issue 1840)", }, + { + Subject: "mesh.seat." + ControllerSeat + ".event.artifacts-gone", + Reason: "refused: the take-in is refused all the same and says why; only the builder's rebuild waits for the " + + "controller's first push, and the builder also asks held when a module's value or its own start comes (novox/hq " + + "issue 1937)", + }, { Subject: "mesh.seat." + ControllerSeat + ".event.sent", Reason: "refused: the controller says a tier sent to a machine only for outsiders, and a send that could not " + diff --git a/internal/broker/streams.go b/internal/broker/streams.go index f694c83f..d3eec6d9 100644 --- a/internal/broker/streams.go +++ b/internal/broker/streams.go @@ -379,7 +379,9 @@ var ControllerStates = []string{"applied", "refused", "built-before", "sent", // And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3): the assignments and // each registered module's declared config (design 57 F3). - "assignment-changed", "declared-config-changed", "predefined-changed"} + "assignment-changed", "declared-config-changed", "predefined-changed", + // And a builder's key not taken in because the artifact store does not hold what it names (novox/hq issue 1937). + "artifacts-gone"} // BusAdvisories are what the bus server says about the mesh's own account that the controller // reads (novox/hq to-be 45 §3, S9): a durable consumer that handed a message over as often as it diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index 5d4ce16e..a6a80775 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -29,7 +29,7 @@ accounts { ] users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_declared-packages.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_grants.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$KV.mesh-controller_states-announced.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.mod.mesh-controller.event.states", "mesh.node.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-controller.tool.push", "mesh.seat.mesh-controller.tool.upgrade", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_declared-packages.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_grants.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$KV.mesh-controller_states-announced.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.mod.mesh-controller.event.states", "mesh.node.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.artifacts-gone", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-controller.tool.push", "mesh.seat.mesh-controller.tool.upgrade", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] } allow_responses: { max: 1, ttl: "1m" } } } diff --git a/internal/inventory/chain_builds.go b/internal/inventory/chain_builds.go index 0ae1b7a3..03517eac 100644 --- a/internal/inventory/chain_builds.go +++ b/internal/inventory/chain_builds.go @@ -658,9 +658,11 @@ func (i *Inventory) chainBuildByID(ctx context.Context, c *chain.Chain, id strin if err != nil || !ok { return Build{}, false, err } + // **The fingerprint read back with the rest** (novox/hq issue 1938): the controller's grants are keyed by it + // (grants.go currentFingerprints), and a build read without it wrote no grant at all from 2026-10-18 on. b := fromChainBuild(rec) return Build{ID: b.ID, Repository: b.Repository, Ref: b.Ref, Module: b.Module, Commit: b.Commit, On: b.On, - Failed: b.Failed, Made: b.Made, Path: b.Path, Asked: b.Asked, At: b.At}, true, nil + Failed: b.Failed, Made: b.Made, Path: b.Path, Asked: b.Asked, At: b.At, SourceFingerprint: b.SourceFingerprint}, true, nil } func (i *Inventory) chainBuiltAgainst(ctx context.Context, c *chain.Chain) (map[string][]string, error) { diff --git a/internal/inventory/chain_store_test.go b/internal/inventory/chain_store_test.go index 6661dfba..d05be0d2 100644 --- a/internal/inventory/chain_store_test.go +++ b/internal/inventory/chain_store_test.go @@ -26,6 +26,7 @@ func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) { first.ReuseKey = "rk-1" second := aBuild("build-2", "notes", "") second.Commit = "c2" + second.SourceFingerprint = "src2:" + strings.Repeat("2", 64) failed := aBuild("build-3", "notes", "compile error") other := aBuild("build-4", "wiki", "") for _, b := range []Build{first, second, failed, other, first} { @@ -48,6 +49,10 @@ func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) { if err != nil || !found || one.Commit != "c2" || one.Module != "notes" { t.Fatalf("build-2 reads as %+v %v %v", one, found, err) } + // 1938: the fingerprint is read back, which the controller's grants are keyed by. + if one.SourceFingerprint != second.SourceFingerprint { + t.Errorf("build-2 reads back the fingerprint %q; want %q", one.SourceFingerprint, second.SourceFingerprint) + } if _, found, _ := inv.BuildByID(ctx, "build-nope"); found { t.Fatal("a build nobody recorded was found") } diff --git a/internal/link/events.go b/internal/link/events.go index eb86e4c6..551a9039 100644 --- a/internal/link/events.go +++ b/internal/link/events.go @@ -89,6 +89,9 @@ const ( KeyDeclaredConfigChanged = "declared-config-changed" // KeyPredefinedChanged: the predefined-keys mirror's one key was written (design 57 K3): a newer list kept. KeyPredefinedChanged = "predefined-changed" + // KeyArtifactsGone: a builder's key was not taken in because the artifact store does not hold an artifact it names + // (novox/hq issue 1937): the module, the key, its fingerprint and every gone reference. The builder builds the key again. + KeyArtifactsGone = "artifacts-gone" ) // Applied is what a machine now runs, as the mesh states it. diff --git a/module.json b/module.json index 169d08cf..d41ce99e 100644 --- a/module.json +++ b/module.json @@ -102,6 +102,7 @@ "data", "build", "artifacts", + "held", "collect", "images", "mirrors", diff --git a/testdata/beside/CAPTURED b/testdata/beside/CAPTURED index e8220a86..c4cff530 100644 --- a/testdata/beside/CAPTURED +++ b/testdata/beside/CAPTURED @@ -9,7 +9,7 @@ a second place to clean besides the catalogue itself. mesh-catalog 2246ccc492e59579a140752e6f574b2121d296af modules/*/module.json, modules/*/files, modules/*/templates, predefined.json mesh-host 9d9e3a46c4f5943cfffb413d962ce48b75e2fd39 examples/foundation-first-node.lock -mesh-sdk a2c813a6b468899bb5a95666b8f6f305be52f200 conformance/events/module-event.json +mesh-sdk 34f210e5eba1d16abe06b526de1026c282692b83 conformance/events/module-event.json To move them, from this repository's root, with the two repositories checked out beside it: