diff --git a/cmd/mesh-controller/ask_bus_wait_test.go b/cmd/mesh-controller/ask_bus_wait_test.go deleted file mode 100644 index 3df88f5a..00000000 --- a/cmd/mesh-controller/ask_bus_wait_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// A retry of a plan failed past the bus's bound, whose module the bus still does not answer, names that module as waiting: -// read while the plan still says failed, it was named nowhere, and the retry said "asked" with nothing after it (issue -// 1403, met again once the hand rebuild no longer resumed the plan first). -func TestARetryNamesTheModulesStillWaitingForTheBus(t *testing.T) { - since := time.Now().UTC() - p := inventory.Plan{ID: "plan-bus", State: inventory.PlanFailed, Tiers: [][]string{{"a"}}, - Modules: map[string]*inventory.PlanModule{"a": {BusWaitFrom: &since}}} - waiting := busWaitingOnRetry(p) - _, said := retryWords(p, []string{"a"}, nil, waiting) - if !strings.Contains(said, "a not asked yet: the bus did not answer") || strings.Contains(said, "asked ;") { - t.Fatalf("a retry left for the bus is said as %q", said) - } -} diff --git a/cmd/mesh-controller/asker_retry_integration_test.go b/cmd/mesh-controller/asker_retry_integration_test.go deleted file mode 100644 index 2ea3177f..00000000 --- a/cmd/mesh-controller/asker_retry_integration_test.go +++ /dev/null @@ -1,138 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/asks" - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -// 1315/AC3: the operator's Retry on a failed plan, acted on through the real callAction and the controller's own -// `plans` verb, sets the plan building again at its tier and ends its plan-failed condition (novox/hq ADR 0449 §1). -func TestAWarrantForRetryRetriesAFailedPlanInTheStoreAndEndsItsCondition(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asksRecorded(t) - planMovedHeard(t) // the bus the verb says its moves on, and the hand-act log - twoTiers(t, open) - before := time.Now().UTC().Add(-time.Hour) - ended := before.Add(time.Minute) - failed := inventory.Plan{ID: "plan-1315-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee", - Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}}, - Note: "a failed to build in tier 0", - Delivery: &inventory.PlanDelivery{}, - Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", - Why: "killed by hand"}}, - Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(ctx, &failed); err != nil { - t.Fatal(err) - } - - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - // The controller's seat answers `plans` as it does: by running the verb. - sub, err := conn.Subscribe(link.SeatToolSubject("mesh-controller", "plans"), func(m *nats.Msg) { - var args map[string]string - _ = json.Unmarshal(m.Data, &args) - answer := link.Answer{Result: json.RawMessage(`"ok"`)} - if len(args) != 1 || args["retry"] == "" { - answer = link.Answer{Error: "the retry was asked with more than the plan"} - } else if err := plansCommand(context.Background(), []string{"retry", args["retry"]}); err != nil { - answer = link.Answer{Error: err.Error()} - } - body, _ := json.Marshal(answer) - _ = m.Respond(body) - }) - if err != nil { - t.Fatal(err) - } - defer func() { _ = sub.Unsubscribe() }() - - // What S23 raises for it, read from the store as the watchdog reads it. - raised := func(context.Context) ([]conditions.Condition, error) { - facts, err := gatherFailedPlans(ctx, open.inventory, time.Now()) - if err != nil { - return nil, err - } - var out []conditions.Condition - for _, o := range watchPlansFailed(&signalFacts{now: time.Now(), failed: facts}) { - out = append(out, conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline, - Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions}) - } - return out, nil - } - now := time.Now() - store := memAskedStore{} - var sent []published - var acts []link.HandAct - a := &asker{open: raised, store: store, - silence: func(context.Context, string, time.Duration, string, string) error { return nil }, - publish: func(_ context.Context, subject string, body []byte, id string) error { - sent = append(sent, published{subject, id, body}) - return nil - }, - call: callAction(conn), - record: func(_ context.Context, act link.HandAct) error { acts = append(acts, act); return nil }, - now: func() time.Time { return now }, - logf: t.Logf} - cs, _ := raised(ctx) - if len(cs) != 1 || len(cs[0].Actions) != 1 { - t.Fatalf("the failed plan raised %+v", cs) - } - if err := a.reconcile(ctx); err != nil { - t.Fatal(err) - } - var w asks.Warrant - for _, r := range store { - for _, o := range r.Ask.Options { - if o.Label == "Retry" { - w = asks.Warrant{Ask: r.ID, Asker: "mesh-controller", About: r.Condition, Outcome: asks.OutcomeChosen, - Option: o.ID, Label: o.Label, Level: o.Level, Channel: "telegram", Proofs: []string{"P1"}, At: now, - AskDigest: r.Ask.Digest(), - By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "42", Verified: "user id verified"}} - } - } - } - if w.Ask == "" { - t.Fatalf("no ask offers Retry: %+v", store) - } - body, _ := json.Marshal(w) - if err := a.Decided(ctx, body); err != nil { - t.Fatal(err) - } - if err := a.Decided(ctx, body); err != nil { // heard again: nothing more - t.Fatal(err) - } - - p, err := open.inventory.PlanByID(ctx, failed.ID) - if err != nil { - t.Fatal(err) - } - m := p.Modules["a"] - if p.State != inventory.PlanBuilding || p.Tier != 0 || m.State != "asked" || !m.Key || m.Why != "waits for the builder's key" || m.Build == "build-1" { - t.Fatalf("after the warrant the plan is %s at tier %d and a is %+v", p.State, p.Tier, m) - } - if got := store[w.Ask]; got.Acted != "done" { - t.Fatalf("the ask ended %q", got.Acted) - } - if len(acts) != 1 || acts[0].Verb != handActWarrant || !strings.Contains(strings.Join(acts[0].Args, " "), "retry="+failed.ID) { - t.Fatalf("the warrant was recorded as %+v", acts) - } - if cs, _ = raised(ctx); len(cs) != 0 { - t.Fatalf("the plan-failed condition did not end: %+v", cs) - } -} diff --git a/cmd/mesh-controller/asker_test.go b/cmd/mesh-controller/asker_test.go index 40604d7e..3acbec19 100644 --- a/cmd/mesh-controller/asker_test.go +++ b/cmd/mesh-controller/asker_test.go @@ -125,8 +125,15 @@ func newAskerRig(t *testing.T) *askerRig { } func heldCondition() conditions.Condition { - o := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s", - Bound: "24h0m0s", H2: "none: the state is the operator's"}})[0] + release := conditions.Action{Label: "Release", Verb: "mesh-delivery.release", Level: conditions.LevelApprove, + Arguments: map[string]string{"id": "novox/hq@055550802096", "why": ""}} + stop := conditions.Action{Label: "Stop", Verb: "mesh-delivery.stop", Level: conditions.LevelApprove, + Arguments: map[string]string{"id": "novox/hq@055550802096", "why": ""}} + o := conditions.Observation{Scope: conditions.ScopeDelivery, ID: "novox/hq@055550802096", Kind: "delivery-stalled", + Severity: conditions.Warning, Resolver: conditions.ResolverOperator, + Headline: "Delivery of hq held for 36 hours", + Explanation: "A delivery of hq has been held for 36 hours, past its limit.", + Needs: "release it, or stop it, " + FromMeshMCPServer, Actions: []conditions.Action{release, stop}} return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: conditions.Warning, Headline: o.Headline, Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions} } @@ -1026,215 +1033,3 @@ func TestTheServingAskerJudgesTheGrantForReal(t *testing.T) { t.Errorf("held %v (%q, %v) with nothing to read", held, why, err) } } - -// failedPlanCondition is the condition S23 raises for a failed delivery plan, as the keeper holds it. -func failedPlanCondition(plan string) conditions.Condition { - p := aFailedPlan(plan, nil) - f := calm(time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC)) - f.failed = []failedPlanFacts{failedFactsFor(p)} - o := watchPlansFailed(f)[0] - return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline, - Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions} -} - -// 1315/AC1: a failed plan's Retry is asked at the approve level, in the words "the failed builds are asked again". -func TestAFailedPlansRetryIsAskedAtApproveInItsOwnWords(t *testing.T) { - r := newAskerRig(t) - c := failedPlanCondition("plan-1792433168676870614") - r.open = []conditions.Condition{c} - if err := r.a.reconcile(context.Background()); err != nil { - t.Fatal(err) - } - sent := r.asksSent(t) - if len(sent) != 1 || len(sent[0].Options) != 1 { - t.Fatalf("asked %+v", sent) - } - o := sent[0].Options[0] - if o.Label != "Retry" || o.Level != asks.Approve || o.Does != "the failed builds are asked again" { - t.Errorf("the option is %+v", o) - } - if err := sent[0].Check(r.now); err != nil { - t.Errorf("the ask is refused as made: %v", err) - } - if strings.Contains(sent[0].Explanation, "mesh MCP server") { - t.Errorf("the ask still says where else to answer: %q", sent[0].Explanation) - } -} - -// 1315/AC2: the warrant calls plans with retry= and nothing else, once, and is recorded as the hand act; -// the same warrant heard again does nothing. -func TestAWarrantForRetryCallsPlansRetryOnceAndIsRecorded(t *testing.T) { - plan := "plan-1792433168676870614" - r := newAskerRig(t) - c := failedPlanCondition(plan) - r.open = []conditions.Condition{c} - _ = r.a.reconcile(context.Background()) - w := r.warrantFor(t, c.Key, "Retry") - answerWith(t, r, w) - answerWith(t, r, w) - if want := `mesh-controller.plans@ {"retry":"` + plan + `"}`; len(r.called) != 1 || r.called[0] != want { - t.Fatalf("called %v, want exactly %s", r.called, want) - } - if len(r.acts) != 1 { - t.Fatalf("hand-acts %+v", r.acts) - } - act := r.acts[0] - if act.Verb != handActWarrant || strings.Join(act.Args, " ") != "mesh-controller.plans retry="+plan || - act.Ask != w.Ask || act.Condition != c.Key || act.Outcome != "done" || act.Cause != conditions.CauseOperatorAnswer { - t.Errorf("the hand-act is %+v", act) - } - if got := r.store[w.Ask]; got.Acted != "done" { - t.Errorf("kept %+v", got) - } -} - -// 1316/AC1 and AC2: a Retry or a Stop on a plan that moved on is refused by `plans`; the operator who pressed it is -// told once, with the ask's headline and the refusal's words, and the hand act records the failure as before. -func TestARefusedAnswerToAPlansAskIsToldOnceAndStillRecorded(t *testing.T) { - plan := "plan-1792433168676870614" - waiting := conditions.Condition{Key: "plan." + plan + ".waiting", Severity: conditions.Urgent, - Headline: "openrazer delivery waiting to start", Needs: "start it, or stop it.", - Explanation: "Needs you: start it, or stop it.", Actions: waitingActions(plan, conditions.Urgent)} - for _, tc := range []struct { - name string - c conditions.Condition - label string - }{ - {"retry", failedPlanCondition(plan), "Retry"}, - {"stop", waiting, "Stop"}, - } { - t.Run(tc.name, func(t *testing.T) { - r := newAskerRig(t) - r.refuses = verbRefused{verb: "mesh-controller.plans", words: plan + " is done; there is nothing to retry"} - r.open = []conditions.Condition{tc.c} - _ = r.a.reconcile(context.Background()) - w := r.warrantFor(t, tc.c.Key, tc.label) - answerWith(t, r, w) - answerWith(t, r, w) // heard again: told once - if len(r.told) != 1 { - t.Fatalf("told %+v", r.told) - } - got := r.told[0] - if got.headline != tc.c.Headline { - t.Errorf("the headline is %q, not the ask's %q", got.headline, tc.c.Headline) - } - said := strings.Join(got.lines, " ") - if !strings.Contains(said, "the plan is done; there is nothing to retry") || !strings.Contains(said, tc.label) { - t.Errorf("the lines do not carry the answer and the refusal's words: %q", said) - } - if why, ok := conditions.Plain(said); !ok { - t.Errorf("the notice is not plain, so the channel would withhold it: %s: %q", why, said) - } - if len(r.acts) != 1 || !strings.HasPrefix(r.acts[0].Outcome, "failed: ") { - t.Errorf("the failure is not recorded as before: %+v", r.acts) - } - if kept := r.store[w.Ask]; !strings.HasPrefix(kept.Acted, "failed: ") { - t.Errorf("kept %+v", kept) - } - }) - } -} - -// 1316/AC3: an answer that is done tells nothing; and a refusal of an answer that is no plan's is not told here. -func TestADoneAnswerTellsNothingAndOnlyAPlansRefusalIsTold(t *testing.T) { - r := newAskerRig(t) - c := failedPlanCondition("plan-1792433168676870614") - r.open = []conditions.Condition{c} - _ = r.a.reconcile(context.Background()) - answerWith(t, r, r.warrantFor(t, c.Key, "Retry")) - if len(r.told) != 0 { - t.Fatalf("a retry that was done told %+v", r.told) - } - - r = newAskerRig(t) - r.refuses = verbRefused{verb: "node-service-manager.restart", words: "unit not found"} - module := conditions.Condition{Key: "module.openrazer.g14.unhealthy", Severity: conditions.Warning, - Headline: "openrazer not working on g14", Needs: "restart its service openrazer-daemon on g14.", - Explanation: "Needs you: restart it.", Actions: []conditions.Action{{Label: "Restart", - Verb: "node-service-manager.restart", Machine: "g14", Level: conditions.LevelApprove, - Arguments: map[string]string{"unit": "openrazer-daemon.service", "scope": "user"}}}} - r.open = []conditions.Condition{module} - _ = r.a.reconcile(context.Background()) - answerWith(t, r, r.warrantFor(t, module.Key, "Restart")) - if len(r.told) != 0 || len(r.acts) != 1 { - t.Fatalf("a refused restart told %+v, acts %+v", r.told, r.acts) - } -} - -// 1316, review round 1: a call that got no answer may have run, so it is told as not known, never as "nothing was -// changed"; and the sentence a refusal falls back to fits the answer given. -func TestAnAnswerThatGotNoAnswerIsToldAsNotKnownNotAsRefused(t *testing.T) { - plan := "plan-1792433168676870614" - r := newAskerRig(t) - r.refuses = errors.New("mesh-controller did not answer plans within 30s") - c := failedPlanCondition(plan) - r.open = []conditions.Condition{c} - _ = r.a.reconcile(context.Background()) - answerWith(t, r, r.warrantFor(t, c.Key, "Retry")) - if len(r.told) != 1 { - t.Fatalf("told %+v", r.told) - } - said := strings.Join(r.told[0].lines, " ") - if !strings.Contains(said, "not known whether") || strings.Contains(said, "Nothing was changed") { - t.Errorf("a call that may have run is told as %q", said) - } - if why, ok := conditions.Plain(said); !ok { - t.Errorf("not plain: %s: %q", why, said) - } -} - -func TestEachAnswersRefusalFallsBackToItsOwnSentence(t *testing.T) { - plan := "plan-1792433168676870614" - for _, tc := range []struct { - act conditions.Action - want string - }{ - {retryAction(plan), "a retry would not run now"}, - {conditions.Action{Label: "Start", Verb: "mesh-controller.plans", Arguments: map[string]string{"go": plan}}, "could not be started now"}, - {conditions.Action{Label: "Stop", Verb: "mesh-controller.plans", Arguments: map[string]string{"stop": plan}}, "could not be stopped now"}, - } { - // Words the operator's channel withholds: a hash and a dotted name. - got := refusalWords("build 48bda475dunst at mesh.seat.x refused", tc.act) - if !strings.Contains(got, tc.want) { - t.Errorf("%s: %q, want %q", tc.act.Label, got, tc.want) - } - if _, ok := conditions.Plain(got); !ok { - t.Errorf("%s: the fallback is not plain: %q", tc.act.Label, got) - } - } -} - -// 1314/AC6, ADR 0449 §2: a retried plan that fails again within the hour of the answer is asked again, because the -// condition that comes back is a new one, raised after the answered ask ended; the same condition still open is not. -func TestARetriedPlanThatFailsAgainWithinTheHourIsAskedAgain(t *testing.T) { - r := newAskerRig(t) - c := failedPlanCondition("plan-1792433168676870614") - c.Raised = r.now.Add(-time.Minute) - r.open = []conditions.Condition{c} - _ = r.a.reconcile(context.Background()) - answerWith(t, r, r.warrantFor(t, c.Key, "Retry")) - // Still open right after the answer: not asked again at once. - r.now = r.now.Add(time.Minute) - _ = r.a.reconcile(context.Background()) - if n := len(r.asksSent(t)); n != 1 { - t.Fatalf("asked again while the same condition stayed open: %d", n) - } - // The retry ran, the plan left failed and the condition ended. - r.now = r.now.Add(time.Minute) - r.open = nil - _ = r.a.reconcile(context.Background()) - // It fails again, ten minutes after the answer: a condition raised after the answered ask ended. - r.now = r.now.Add(8 * time.Minute) - again := failedPlanCondition("plan-1792433168676870614") - again.Raised = r.now - r.open = []conditions.Condition{again} - if err := r.a.reconcile(context.Background()); err != nil { - t.Fatal(err) - } - if n := len(r.asksSent(t)); n != 2 { - t.Fatalf("the second failure was asked %d times in all, want 2", n) - } - if len(r.called) != 1 { - t.Fatalf("something retried without an answer: %v", r.called) - } -} diff --git a/cmd/mesh-controller/away_integration_test.go b/cmd/mesh-controller/away_integration_test.go deleted file mode 100644 index 45156cdd..00000000 --- a/cmd/mesh-controller/away_integration_test.go +++ /dev/null @@ -1,263 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "fmt" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/nats-io/nats.go" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/builderkeys" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -// 1161/AC5, with the store and a real bus and no ticker: a plan of two tiers whose first tier runs on a machine away when it -// is sent finishes that tier without waiting for it and without its bound, sends the second tier to the machine heard -// alone, and says the machine was away (issue 1161). The machine away keeps its want: it was sent nothing of the plan's -// builds, so the send it gets when it is heard again carries them (AC3). -func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - - // laptop is away as every tier is sent; anchor is heard. - wasAway := judgeAway - judgeAway = func(context.Context, *stores, time.Time) (map[string]bool, error) { - return map[string]bool{"laptop": true}, nil - } - t.Cleanup(func() { judgeAway = wasAway }) - - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - legacy, err := js.Conn().JetStream() - if err != nil { - t.Fatal(err) - } - checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy} - t.Cleanup(func() { checkEvents = nil }) - sentHeard := make(chan *nats.Msg, 32) - sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeySent), sentHeard) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - - wasSaved := inventory.PlanSaved - inventory.PlanSaved = servingPlanSaved(ctx, checkEvents, open) - t.Cleanup(func() { inventory.PlanSaved = wasSaved }) - - // The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept. - var mu sync.Mutex - var sends [][]string - everyKeyDone(t) // the builds of the tiers: each module is taken from a done key as soon as the walk reaches it - // Except tier 1's module b, whose key is not built until the test says so: a walk that takes it in the same look - // goes on through tier 1's send and overwrites the note of the move before the test reads it (the old build ask - // was where the walk paused). - wholeKeys := openPlanKeys - var holdB atomic.Bool - holdB.Store(true) - openPlanKeys = func() (planKeyReads, func(), error) { - all, done, err := wholeKeys() - return keysHolding{all, "b", &holdB, ""}, done, err - } - digests := 0 - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - mu.Lock() - defer mu.Unlock() - sends = append(sends, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - builds := map[string]string{} - for _, m := range []string{"a", "b"} { - builds[m] = current[m].Commit - } - for _, node := range names { - digests++ - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), fmt.Sprintf("d-%s-%d", node, digests), builds); err != nil { - return nil, err - } - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - - register := func(m, commit string, asked time.Time) { - t.Helper() - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit}, - inventory.Source{Repository: "novox/" + m, Seat: "git", Path: "modules/" + m, BuiltFrom: commit, Head: commit, - Asked: asked}); err != nil { - t.Fatal(err) - } - } - record := func(id, m, commit string, at time.Time) { - t.Helper() - b := inventory.Build{ID: id, Module: m, Commit: commit, Repository: "novox/" + m, Path: "modules/" + m, Asked: at, At: at} - b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit}) - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - old := time.Now().Add(-2 * time.Hour) - for _, m := range []string{"a", "b"} { - record("build-"+m+"-1", m, "c1", old) - register(m, "c1", old) - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, m); err != nil { - t.Fatal(err) - } - } - } - for _, n := range []string{"anchor", "laptop"} { - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"a": "c1", "b": "c1"}); err != nil { - t.Fatal(err) - } - } - record("build-a-2", "a", "c2", time.Now().Add(-time.Minute)) - register("a", "c2", time.Now().Add(-time.Minute)) - - // A machine's report on what it was sent, heard as the controller hears it. - listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}} - seq := int64(0) - reports := func(node string) { - t.Helper() - last, found, err := inv.SentTo(ctx, node) - if err != nil || !found { - t.Fatalf("%s was sent nothing: %v", node, err) - } - seq++ - body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest, - "report_sequence": seq}) - var report link.Report - if err := json.Unmarshal(body, &report); err != nil { - t.Fatal(err) - } - if _, err := listener.Heard(ctx, report); err != nil { - t.Fatal(err) - } - } - plan := func() inventory.Plan { - t.Helper() - p, err := inv.PlanByID(ctx, "plan-1160") - if err != nil { - t.Fatal(err) - } - return p - } - - built := time.Now().UTC() - p := inventory.Plan{ID: "plan-1160", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: time.Now().UTC(), - State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, - Delivery: &inventory.PlanDelivery{Order: []inventory.Edge{{From: "b", To: "a", Kind: edgeChangeOrder}}}, - Modules: map[string]*inventory.PlanModule{ - "a": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-a-2"}, - "b": {}, - }} - first, stopFirst := context.WithCancel(ctx) - defer stopFirst() - waitFirst := startRolloutEvents(first, open, js) - // Ends with the test before the cleanups restore what the look reads, which run after it (issue 1338). - t.Cleanup(func() { stopFirst(); waitFirst() }) - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - eventually(t, "tier 0 sent to anchor alone", func() bool { - mu.Lock() - defer mu.Unlock() - return len(sends) == 1 && strings.Join(sends[0], ",") == "anchor" - }) - reports("anchor") - // The note of the move to tier 1 is read in the step that sees it: later steps of the tier say their own. - var moved string - eventually(t, "tier 1 reached on anchor's report alone", func() bool { - if q := plan(); q.Tier == 1 { - moved = q.Note - return true - } - return false - }) - if !strings.Contains(moved, "laptop was away: sent when it is heard again") || strings.Contains(moved, "anchor") { - t.Errorf("1161/AC2: the plan says %q; want laptop named away and not anchor", moved) - } - // Tier 1's module is taken from its key once that is built, which the walk's look does by itself. - holdB.Store(false) - askToLook() - eventually(t, "tier 1 sent to anchor alone", func() bool { - mu.Lock() - defer mu.Unlock() - return len(sends) == 2 && strings.Join(sends[1], ",") == "anchor" - }) - // 1161/AC3: laptop was sent nothing of the plan's builds, so it still wants them for the send it gets when heard again. - was, known, err := inv.SentBuilds(ctx, "laptop") - if err != nil || !known { - t.Fatalf("1161/AC3: laptop's last send: %v, known %v", err, known) - } - current, err := inv.CurrentBuilds(ctx) - if err != nil { - t.Fatal(err) - } - if was["a"] == current["a"].Commit || was["a"] != "c1" { - t.Errorf("1161/AC3: laptop was last sent a at %q and a is now %q; want it still at c1, owed c2", was["a"], current["a"].Commit) - } - if got := plan().Away; strings.Join(got, ",") != "laptop" { - t.Errorf("1161/AC2: the plan keeps %v away; want laptop", got) - } -} - -// keysHolding is the stand-in builder's keys but for one module, whose key is not built while held; only the key of -// the fingerprint fp when that is given. -type keysHolding struct { - planKeyReads - module string - held *atomic.Bool - fp string -} - -func (k keysHolding) Build(ctx context.Context, module, fp string) (builderkeys.Key, error) { - if module == k.module && k.held.Load() && (k.fp == "" || fp == k.fp) { - return builderkeys.Key{Module: module, Fingerprint: fp}, nil - } - return k.planKeyReads.Build(ctx, module, fp) -} - -// holdKeys has the stand-in builder's keys of these modules (at any commit) unbuilt until the returned flag is cleared, -// so a walk that took them stays on its tier as it did while its build ran. It restores what it replaced at the end of -// the test, and the flag is set until the test lowers it. -func holdKeys(t *testing.T, modules ...string) *atomic.Bool { - t.Helper() - held := &atomic.Bool{} - held.Store(true) - was := openPlanKeys - openPlanKeys = func() (planKeyReads, func(), error) { - all, done, err := was() - var keys planKeyReads = all - for _, m := range modules { - keys = keysHolding{keys, m, held, ""} - } - return keys, done, err - } - t.Cleanup(func() { openPlanKeys = was }) - return held -} diff --git a/cmd/mesh-controller/backups_before_send_integration_test.go b/cmd/mesh-controller/backups_before_send_integration_test.go deleted file mode 100644 index 33e30492..00000000 --- a/cmd/mesh-controller/backups_before_send_integration_test.go +++ /dev/null @@ -1,312 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "os" - "strings" - "sync" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -// aBackupSeatOnTheBus is one machine's node-backup holder as a test makes it: it counts the `now` asks it hears and -// answers `backed-up` with the last night it is told (novox/hq ADR 0404). -type aBackupSeatOnTheBus struct { - mu sync.Mutex - nows []string - night map[string]any // module → lastNight -} - -func (f *aBackupSeatOnTheBus) serve(t *testing.T, conn *nats.Conn, node string) { - t.Helper() - reply := func(m *nats.Msg, result any) { - body, _ := json.Marshal(map[string]any{"result": result, "error": "", "node": node}) - _ = m.Respond(body) - } - now, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "now", node), func(m *nats.Msg) { - var a struct { - Module string `json:"module"` - } - _ = json.Unmarshal(m.Data, &a) - f.mu.Lock() - f.nows = append(f.nows, a.Module) - f.mu.Unlock() - reply(m, map[string]any{"started": a.Module}) - }) - if err != nil { - t.Fatal(err) - } - read, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "backed-up", node), func(m *nats.Msg) { - f.mu.Lock() - defer f.mu.Unlock() - var out []map[string]any - for module, n := range f.night { - out = append(out, map[string]any{"module": module, "lastNight": n}) - } - reply(m, out) - }) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = now.Unsubscribe(); _ = read.Unsubscribe() }) - if err := conn.Flush(); err != nil { - t.Fatal(err) - } -} - -func (f *aBackupSeatOnTheBus) answers(module string, ok bool, words string) { - f.mu.Lock() - defer f.mu.Unlock() - f.night = map[string]any{module: map[string]any{"ok": ok, "at": time.Now().UTC().Add(time.Second), "error": words}} -} - -func (f *aBackupSeatOnTheBus) asks() int { - f.mu.Lock() - defer f.mu.Unlock() - return len(f.nows) -} - -// Acceptance test 3 of design 47, on the controller's side: a delivery plan that moves a module keeping data waits -// for its backups on both machines before it asks a build; a backup that fails holds it, with the machine, the -// module and the backup's words, and a condition says the same; the verb the delivery's release calls asks the -// backups again, and the plan goes on. One bus and one store for the scenarios. -func TestADeliveryThatMovesABackedUpModuleWaitsForItsBackupAndAFailedOneHoldsItAndIsSaid(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := asksRecorded(t) - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - before := handActConn - handActConn = js.Conn() - t.Cleanup(func() { handActConn = before }) - - if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "keeper", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}) - notes, err := catalogue.ParseManifest([]byte(`{"module":"notes","version":"1","data":{"own":[` + - `{"id":"files","path":"${dir:files}","class":"valuable","why":"people's notes"}]},` + - `"resources":[{"id":"files","type":"directory"}]}`)) - if err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, notes, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", - Path: "modules/notes", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - for _, node := range []string{"anchor", "laptop"} { - if _, err := assign(ctx, open, node, "keeper"); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, node, "notes"); err != nil { - t.Fatal(err) - } - } - seats := map[string]*aBackupSeatOnTheBus{"anchor": {}, "laptop": {}} - for node, s := range seats { - s.serve(t, js.Conn(), node) - } - // Short waits: the bound and the look are variables, so nothing here waits for the minutes of a real backup. - wasWithin, wasEvery := backupsWithin, backupsPollEvery - backupsWithin, backupsPollEvery = time.Minute, 10*time.Millisecond - t.Cleanup(func() { backupsWithin, backupsPollEvery = wasWithin, wasEvery }) - - newPlan := func(id string) { - t.Helper() - p := inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1", Created: time.Now().UTC(), - State: inventory.PlanBuilding, Tiers: [][]string{{"notes"}}, Modules: map[string]*inventory.PlanModule{"notes": {}}} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - } - read := func(id string) inventory.Plan { - t.Helper() - p, err := inv.PlanByID(ctx, id) - if err != nil { - t.Fatal(err) - } - return p - } - - // 1. Both machines are asked; while one has not answered, no build is asked. - newPlan("plan-one") - advancePlans(ctx, open) - if seats["anchor"].asks() != 1 || seats["laptop"].asks() != 1 || len(*asked) != 0 { - t.Fatalf("asks anchor=%d laptop=%d builds=%v", seats["anchor"].asks(), seats["laptop"].asks(), *asked) - } - seats["anchor"].answers("notes", true, "") - advancePlans(ctx, open) - if len(*asked) != 0 || read("plan-one").Backups.State != inventory.BackupPending { - t.Fatalf("a tier was asked while a backup was pending: %v %+v", *asked, read("plan-one").Backups) - } - seats["laptop"].answers("notes", true, "") - advancePlans(ctx, open) - if len(*asked) != 1 || read("plan-one").Backups.State != inventory.BackupDone { - t.Fatalf("the tier was not asked once every backup was done: %v %+v", *asked, read("plan-one").Backups) - } - finish := read("plan-one") - finish.State = inventory.PlanDone - if err := inv.SavePlan(ctx, &finish); err != nil { - t.Fatal(err) - } - - // 2. A backup that fails holds the plan: not failed, nothing asked, the words kept, a condition raised. - seats["anchor"].night, seats["laptop"].night = nil, nil - newPlan("plan-two") - advancePlans(ctx, open) - seats["anchor"].answers("notes", true, "") - seats["laptop"].answers("notes", false, "no space left on device") - advancePlans(ctx, open) - held := read("plan-two") - if held.State != inventory.PlanBuilding || held.Backups.State != inventory.BackupFailed || len(*asked) != 1 { - t.Fatalf("the plan was not held: %s %+v builds=%v", held.State, held.Backups, *asked) - } - var failed *inventory.PlanBackup - for i := range held.Backups.Asked { - if held.Backups.Asked[i].State == inventory.BackupFailed { - failed = &held.Backups.Asked[i] - } - } - if failed == nil || failed.Machine != "laptop" || failed.Module != "notes" || failed.Words != "no space left on device" || - strings.Contains(held.Note, "tried again") { - t.Fatalf("the failed backup: %+v note=%q", failed, held.Note) - } - advancePlans(ctx, open) // not retried on the next look - if again := read("plan-two"); again.Backups.State != inventory.BackupFailed || len(*asked) != 1 || - seats["laptop"].asks() != 2 { - t.Fatalf("a held plan was asked again by itself: %+v builds=%v asks=%d", again.Backups, *asked, seats["laptop"].asks()) - } - said, err := probeBackupHeld(ctx, &doctor{open: open}) - if err != nil || len(said) != 1 || said[0].Kind != kindBackupHeld || !strings.Contains(said[0].Said, "no space left on device") { - t.Fatalf("the condition: %+v %v", said, err) - } - - // 3. The verb the release calls: the failed backup is asked again, and only it, and the plan goes on. - seats["laptop"].answers("notes", false, "no space left on device") - if err := deliveryCommand(ctx, []string{"backups-again", "plan-two", "--why", "a person released it", "--by", - "mesh-delivery for jochen"}); err != nil { - t.Fatal(err) - } - if err := deliveryCommand(ctx, []string{"backups-again", "plan-two", "--why", "again"}); err == nil { - t.Fatal("a plan with no failed backup was asked again") - } - seats["laptop"].night = nil - advancePlans(ctx, open) - if seats["laptop"].asks() != 3 || seats["anchor"].asks() != 2 { - t.Fatalf("asked again: laptop=%d anchor=%d", seats["laptop"].asks(), seats["anchor"].asks()) - } - seats["laptop"].answers("notes", true, "") - advancePlans(ctx, open) - // One key taken, not two: plan-one and plan-two carry the same module at the same commit, so they share one key - // build id (key_takein.go keyBuildID), and a walk finds a key already taken (keyTakes.taken, plan_keys.go) and takes - // it in no more. So that the smaller count hides no plan that stopped, plan-two's module must be built from that key. - if got := read("plan-two"); got.Backups.State != inventory.BackupDone || len(*asked) != 1 || - got.Modules["notes"].Build == "" || got.Modules["notes"].State == "failed" || got.Modules["notes"].Key && got.Modules["notes"].State == "asked" { - t.Fatalf("the plan did not go on: %+v module=%+v builds=%v", got.Backups, got.Modules["notes"], *asked) - } - if said, _ := probeBackupHeld(ctx, &doctor{open: open}); len(said) != 0 { - t.Fatalf("the condition stays: %+v", said) - } -} - -// Issue 1086: a plan cut from a merge asked its first tier at the cut, so a module keeping data was sent without the -// backup the plan promised. The cut now takes the backups first (askFirstTierAtCut, batches.go), and the walk's own -// step asks the tier once they are done. -func TestAPlanCutFromAMergeOfABackedUpModuleAsksItsBackupAndNotItsTierUntilItIsDone(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - settings := t.TempDir() + "/merge-window.json" - if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil { - t.Fatal(err) - } - t.Setenv(mergeWindowFileVar, settings) - everyHeadWaits(t) - asked := asksRecorded(t) - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - before := handActConn - handActConn = js.Conn() - t.Cleanup(func() { handActConn = before }) - - if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "keeper", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, - inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - notes, err := catalogue.ParseManifest([]byte(`{"module":"notes","version":"1","data":{"own":[` + - `{"id":"files","path":"${dir:files}","class":"valuable","why":"people's notes"}]},` + - `"resources":[{"id":"files","type":"directory"}]}`)) - if err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, notes, inventory.Source{Repository: "novox/notes", Seat: "git", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - for _, m := range []string{"keeper", "notes"} { - if _, err := assign(ctx, open, "anchor", m); err != nil { - t.Fatal(err) - } - } - seat := &aBackupSeatOnTheBus{} - seat.serve(t, js.Conn(), "anchor") - wasWithin, wasEvery := backupsWithin, backupsPollEvery - backupsWithin, backupsPollEvery = time.Minute, 10*time.Millisecond - t.Cleanup(func() { backupsWithin, backupsPollEvery = wasWithin, wasEvery }) - - // 1. A merge of notes is heard and its window closes: the batch is cut through cutBatch. - hear(t, open, repoMerge("notes", "c-notes", t0), t0.Add(time.Second)) - cutAt(t, open, t0.Add(time.Second+90*time.Second)) - ws, _ := walks(t, open) - if len(ws) != 1 { - t.Fatalf("%d walk(s)", len(ws)) - } - // 2. Right after the cut no tier is asked, the record has anchor/notes pending, the seat heard one `now`. - cut, err := inv.PlanByID(ctx, ws[0].ID) - if err != nil { - t.Fatal(err) - } - if len(*asked) != 0 || askedInTier(cut) { - t.Fatalf("the first tier was asked at the cut: %v %+v", *asked, cut.Modules) - } - if cut.Backups == nil || cut.Backups.State != inventory.BackupPending || len(cut.Backups.Asked) != 1 || - cut.Backups.Asked[0].Machine != "anchor" || cut.Backups.Asked[0].Module != "notes" { - t.Fatalf("the backups of the cut: %+v", cut.Backups) - } - if seat.asks() != 1 { - t.Fatalf("the seat heard %d `now`", seat.asks()) - } - // 3. The backup answers done: the walk's own step asks the first tier, once, and asks no second backup. - seat.answers("notes", true, "") - advancePlans(ctx, open) - advancePlans(ctx, open) - after, err := inv.PlanByID(ctx, ws[0].ID) - if err != nil { - t.Fatal(err) - } - if len(*asked) != 1 || after.Backups.State != inventory.BackupDone || seat.asks() != 1 { - t.Fatalf("builds=%v backups=%+v asks=%d", *asked, after.Backups, seat.asks()) - } -} diff --git a/cmd/mesh-controller/batches.go b/cmd/mesh-controller/batches.go deleted file mode 100644 index 44f6426a..00000000 --- a/cmd/mesh-controller/batches.go +++ /dev/null @@ -1,1797 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "slices" - "sort" - "strconv" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// **Merges are assembled in a rolling window, and each batch is delivered by one walk** (novox/hq ADR 0276, -// issue 362). -// -// Every merge opened a walk of its own, and the next merge of the branch superseded it. On 2026-10-10 two -// catalogue merges eighteen seconds apart left one walk that no delivery held, and the operator started it by -// hand 58 minutes later. A merge now joins the open **batch**. Each merge restarts the **merge window**; it -// closes when none came for `merge-window` (90 seconds) or at the latest `merge-window-at-most` (10 minutes) -// after the batch's first merge. A closed batch becomes one walk when no walk is open: one plan for every -// commit in it, with at most one commit per repository, the latest of its branch, which contains the earlier -// ones. The walk names every merge it answers. -// -// - One walk at a time beside a related one (ADR 0407 decision 4, issue 973). Merges that come while a walk -// runs join the next batch; a closed batch that shares nothing with an open walk is cut beside it. A walk -// that started is never superseded; one still waiting for its delivery's word is folded into the walk of a -// batch related to it, which answers every merge it answered — the only supersession left. -// - A merge heard after a later merge of its branch was put in a walk is answered by that walk: a batch -// still assembling, an open walk that builds everything it moves, or a walk done that did, which then -// names it at once. Otherwise it joins the next batch. -// - A failed walk marks nothing delivered. Each earlier merge it carried in a later commit is walked on its -// own commit, the newest first, one walk each, until one is delivered; the older ones are then answered -// by that one. -// -// **A merge waits only for a batch it shares something with** (novox/hq ADR 0304 decision 1, ADR 0306 decisions -// 2 and 3, issue 525): it joins a batch it is related to, and one that is related to none and whose delivery -// group has no member still to merge is a batch whose window is closed at once (batchToJoin). Such a batch is cut -// at once, beside any open walk it shares nothing with, and queued behind the first one it is related to -// (cutUnrelated). -// -// A merge on the controller's own path (a module whose walk waits for nobody's word) never shares a batch with -// one that waits for mesh-delivery's: each kind has a batch of its own, an own-path batch is cut first and folds -// no waiting catalogue walk, and a catalogue walk let go while another walk runs starts once it ended — so no -// catalogue batch's walk starts without the word (decided during the build, 2026-10-10). -// -// The batch, its merges and their times are in the store (batched_merge, and the batch's own plan record in -// the state `assembling` or `queued`), so a restarted controller resumes the window where it stood. - -// The merge window's defaults. The controller's settings `merge-window` and `merge-window-at-most` reach it in -// the file MESH_MERGE_WINDOW_FILE names (module.json), read at every look, so a change takes effect without a -// restart. -var ( - mergeWindowDefault = 90 * time.Second - mergeWindowAtMostDefault = 10 * time.Minute -) - -// mergeWindowFileVar names the file the controller's window settings are written to. -const mergeWindowFileVar = "MESH_MERGE_WINDOW_FILE" - -// batchLateAfter is how long past its maximum a batch may still be assembling, with no walk open, before -// the controller is said to have failed to cut it (ADR 0276 decision 9). -const batchLateAfter = time.Minute - -// mergeGrace is how long before a batch's first merge a merge may have been made and still be told apart from one -// made before the controller began keeping merges (owedLate). -const mergeGrace = 10 * time.Minute - -// edgeChangeOrder is the order the rules of a plan draw among its modules (novox/hq ADR 0419 clauses 3 and 4, delivery_edges.go) -// made an edge of its tiers: the module after depends on the module before, and is gated on it. -const edgeChangeOrder = "change-order" - -// mergeWindowOf is the window and its maximum, from the controller's settings; a value unreadable is its -// default, and said. -func mergeWindowOf() (window, atMost time.Duration) { - window, atMost = mergeWindowDefault, mergeWindowAtMostDefault - path := os.Getenv(mergeWindowFileVar) - if path == "" { - return window, atMost - } - raw, err := os.ReadFile(path) - if err != nil { - fmt.Fprintf(os.Stderr, "the merge window's settings cannot be read (%v); %s and %s are used\n", err, window, atMost) - return window, atMost - } - var values map[string]any - if err := json.Unmarshal(raw, &values); err != nil { - fmt.Fprintf(os.Stderr, "the merge window's settings are not JSON (%v); %s and %s are used\n", err, window, atMost) - return window, atMost - } - for key, to := range map[string]*time.Duration{"merge-window": &window, "merge-window-at-most": &atMost} { - v, has := values[key] - if !has || v == "" { - continue - } - if d, ok := durationSetting(v); ok { - *to = d - } else { - fmt.Fprintf(os.Stderr, "the setting %s=%v is no duration; %s is used\n", key, v, *to) - } - } - return window, atMost -} - -// durationSetting reads a duration as a setting says it: "90s", "10m", or a number of seconds. -func durationSetting(v any) (time.Duration, bool) { - switch t := v.(type) { - case string: - if d, err := time.ParseDuration(strings.TrimSpace(t)); err == nil && d >= 0 { - return d, true - } - if n, err := strconv.ParseFloat(strings.TrimSpace(t), 64); err == nil && n >= 0 { - return time.Duration(n * float64(time.Second)), true - } - case float64: - if t >= 0 { - return time.Duration(t * float64(time.Second)), true - } - } - return 0, false -} - -// mergeFacts is what hearMerge judges a merge by, kept apart so the catch-up (ADR 0457) judges a merge the way hearing it will. -type mergeFacts struct { - merged time.Time - touches []inventory.Entry - later []inventory.BatchedMerge - owed bool - from, packaging []inventory.Entry - already int - addsModuleToBuilds bool // a first merge into a repository the mesh builds nothing from: firstBuilds acts on it -} - -// factsOfMerge reads what hearing a merge turns on: the modules it touches whatever the history, the later merges of its -// branch, whether it is owed late, and the modules it moves. -func factsOfMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, now time.Time, - entries []inventory.Entry, read map[string][]inventory.ReadRepository) (mergeFacts, error) { - repository := m.Owner + "/" + m.Repo - f := mergeFacts{} - var err error - f.merged, err = time.Parse(time.RFC3339Nano, m.MergedAt) - if err != nil { - f.merged = now - } - // **What it touches, history or not** (review of this change): a merge heard after a later merge of its - // branch was cut reads as history for every module that cut marked seen, and was dropped from the record - // here, never named by the walk that carried it. - raw := m - raw.MergedAt = "" - f.touches = wouldMove(raw, entries, read) - if len(f.touches) > 0 { - if f.later, err = inv.LaterMergesOf(ctx, repository, m.Base, f.merged.UTC()); err != nil { - return f, err - } - } - if f.owed, err = owedLate(ctx, inv, m, f.merged, f.touches); err != nil { - return f, err - } - f.from, f.packaging, f.already = mergeCandidates(m, entries, read) - f.addsModuleToBuilds = f.already == 0 && len(addedDirs(m, entries, read)) > 0 - return f, nil -} - -// plans says hearing the merge puts it in a batch or a walk. -func (f mergeFacts) plans() bool { - return len(f.from) > 0 || len(f.packaging) > 0 || len(f.later) > 0 || f.owed -} - -// heldMerge marks a failure to act on a merge, whatever it was, as "not now": the announcement is held on the bus and -// handed over again, so the merge joins the next batch, never taken (issue 1583, after 921). The bus acknowledges it -// only when hearMerge answers without ErrTryAgain, and a merge heard once is not announced again: the bus answering -// "jetstream not enabled", a bucket missing for a moment or a store read cut short each lost the merge when only an -// outage of a known kind (notNow) was held. The delay between hands-over is bounded (link.MergeRedeliverAfter). -func heldMerge(err error) error { - if err == nil { - return nil - } - if errors.Is(err, link.ErrTryAgain) { - return err - } - return fmt.Errorf("%w: the merge is not kept yet: %w", link.ErrTryAgain, err) -} - -// hearMerge puts a merge the forge announced into the batch that answers it, at the moment it was heard, -// and cuts what is due. -func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.Time) error { - // One merge acted on at a time, whoever hands it over (novox/hq issue 266). - actingOnMerges.Lock() - defer actingOnMerges.Unlock() - - inv := f.open.inventory - repository := m.Owner + "/" + m.Repo - if kept, known, err := inv.MergeOf(ctx, repository, m.Commit); err != nil { - return heldMerge(err) - } else if known { - fmt.Printf("%s merged into %s (%.8s) was heard before; %s answers it\n", repository, m.Base, m.Commit, - orWords(kept.Plan, "a walk of its own commit, once the walks before it end")) - return nil - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return heldMerge(err) - } - read, err := readForPlanning(ctx, inv) - if err != nil { - return heldMerge(err) - } - facts, err := factsOfMerge(ctx, inv, m, now, entries, read) - if err != nil { - return heldMerge(err) - } - merged, touches := facts.merged, facts.touches - if !facts.plans() { - already := facts.already - // "Already built from it" and "nothing reads it" are different facts, and reading the first - // as the second sends somebody looking for a broken trigger when the mesh is up to date. - if already > 0 { - fmt.Printf("%s merged into %s (%.8s); %d module(s) the mesh holds are already built from it\n", - repository, m.Base, m.Commit, already) - return nil - } - if already == 0 { - if done, err := f.firstBuilds(ctx, m, entries, read, now); err != nil || done { - return heldMerge(err) - } - } - fmt.Print(nothingReadsIt(repository, m)) - return nil - } - // A module the merge adds is registered from the builder's first done key of it, which the builder's write wakes - // (novox/hq ADR 0501 §6): nothing is asked here. - // **Its delivery group's word, before the plans are held** (ADR 0306 decision 2): a slow answer holds up no walk. - own := ownPath(touches) - waits, holding, err := groupWaitHolding(ctx, inv, m, deliverySeatHeld(entries)) - if err != nil { - return heldMerge(err) - } - release, err := inv.HoldPlans(ctx, true) - if err != nil { - return heldMerge(err) - } - defer release() - // Again under the hold: a controller handing over may have kept it since. - if _, known, err := inv.MergeOf(ctx, repository, m.Commit); err != nil || known { - return heldMerge(err) - } - var moves []string - for _, e := range touches { - moves = append(moves, e.Manifest.Module) - } - sort.Strings(moves) - event, err := json.Marshal(keptMerge{SourceMoved: m, Moves: moves}) - if err != nil { - return err - } - heard := inventory.BatchedMerge{Repository: repository, Branch: m.Base, Commit: m.Commit, Merged: merged.UTC(), - Heard: now, Event: event} - - // **A merge heard after a later merge of its branch** is answered by the walk holding that one. - // **A merge on the controller's own path never shares a batch with one that waits for the delivery's - // word** (decided during the build, 2026-10-10): batched together, the catalogue's deliveries would start - // with the controller's and skip their turn. Each kind has a batch of its own. - if p, ok, err := answeredByALaterMerge(ctx, inv, heard, touches, own, deliverySeatHeld(entries)); err != nil { - return heldMerge(err) - } else if ok { - heard.Plan = p.ID - if _, err := inv.AddMerge(ctx, heard); err != nil { - return heldMerge(err) - } - if p.Batch() { - if err := keepBatch(ctx, inv, &p, now, ""); err != nil { - return unsaved(err) - } - } else if err := nameMerges(ctx, inv, &p); err != nil { - return unsaved(err) - } - fmt.Printf("%s merged into %s (%.8s) after a later merge of its branch was put in %s (%s): that %s "+ - "answers it\n", repository, m.Base, m.Commit, p.ID, p.State, walkOrBatch(p)) - return cutBatchesHeld(ctx, f.open, now) - } - - batch, why, err := batchToJoin(ctx, inv, heard, entries, read, now, own, waits) - if err != nil { - return heldMerge(err) - } - heard.Plan = batch.ID - if added, err := inv.AddMerge(ctx, heard); err != nil || !added { - return heldMerge(err) - } - holdFor(&batch, m.Head, waits, holding, own, now) - if err := keepBatch(ctx, inv, &batch, now, ""); err != nil { - return unsaved(err) - } - fmt.Printf("%s merged into %s (%.8s); %s, it joins %s — %s\n", repository, m.Base, m.Commit, why, batch.ID, - batchWords(batch, now)) - return cutBatchesHeld(ctx, f.open, now) -} - -// holdFor keeps in a batch that a merge of a group joined whether that group still has a member to merge (ADR 0418 -// clause 3): held while it has, the window running again from the moment the last hold is cleared. A group with a -// merge of the controller's own path makes its batch an own-path one: such a plan waits for nobody's word, and is -// started by the controller when the group's batch closes (clause 6). -func holdFor(batch *inventory.Plan, head, why string, holding, own bool, now time.Time) { - if batch.Delivery == nil { - batch.Delivery = &inventory.PlanDelivery{} - } - if batch.Delivery.Batch == nil { - batch.Delivery.Batch = &inventory.PlanBatch{Own: own} - } - w := batch.Delivery.Batch - w.Own = w.Own || own - if head == "" { - return - } - if holding { - if w.Holds == nil { - w.Holds = map[string]string{} - } - w.Holds[head] = why - return - } - if _, held := w.Holds[head]; held { - holds := map[string]string{} - for h, y := range w.Holds { - if h != head { - holds[h] = y - } - } - w.Holds = holds - if len(holds) == 0 { - w.Holds, w.HeldAt = nil, now - } - } -} - -// owedLate says a merge that reads as history is still the record's to answer (review of this change): a cut -// marks every module it moved seen at that moment, so a merge of that branch -// handed over late — made before the cut, heard after it — reads as history and was dropped. One that touches -// a module the mesh holds, of a branch whose merges the controller keeps, made since it began keeping them, -// is kept; one made before it began is old news, as before. -func owedLate(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, merged time.Time, - touches []inventory.Entry) (bool, error) { - if len(touches) == 0 { - return false, nil - } - kept, first, err := inv.KeptOnBranch(ctx, m.Owner+"/"+m.Repo, m.Base) - if err != nil || !kept { - return false, err - } - return !merged.Before(first.Add(-mergeGrace)), nil -} - -// answeredByALaterMerge is the plan that answers a merge heard after a later merge of its branch was put in -// one (ADR 0276 decision 4): a batch not yet cut, whose walk plans every file of the repository's merges; an -// open walk, or a walk done, that builds every module this merge would move — built from the branch after the -// later merge, which contains it. A failed or stopped walk answers nothing more, and a walk folded into -// another is followed to that one. False when none does: the merge joins the next batch. -func answeredByALaterMerge(ctx context.Context, inv *inventory.Inventory, heard inventory.BatchedMerge, - moves []inventory.Entry, own, held bool) (inventory.Plan, bool, error) { - later, err := inv.LaterMergesOf(ctx, heard.Repository, heard.Branch, heard.Merged) - if err != nil || len(later) == 0 { - return inventory.Plan{}, false, err - } - for _, l := range later { - p, err := followTakeOver(ctx, inv, l.Plan) - if err != nil { - return inventory.Plan{}, false, err - } - switch { - case p.Batch(): - // A batch of the other kind is not this merge's: it joins the batch of its own kind. - if p.OwnPath() == own { - return p, true, nil - } - case p.Open() || p.State == inventory.PlanDone: - // A walk that waited for nobody's word is not a catalogue merge's to be answered by while the delivery - // seat has a holder: its delivery would skip its turn (decided during the build, 2026-10-10). With no - // holder on record nothing waits, and any walk that built it answers. - waited := p.Delivery != nil && p.Delivery.Awaits != "" - if buildsEvery(p, moves) && (own || waited || !held) { - return p, true, nil - } - } - } - return inventory.Plan{}, false, nil -} - -// followTakeOver is a plan, or the walk that took it over when it was folded before it started. -func followTakeOver(ctx context.Context, inv *inventory.Inventory, id string) (inventory.Plan, error) { - p, err := inv.PlanByID(ctx, id) - for hops := 0; err == nil && hops < 16 && p.State == inventory.PlanSuperseded && p.Delivery != nil && - p.Delivery.TakenOverBy != ""; hops++ { - p, err = inv.PlanByID(ctx, p.Delivery.TakenOverBy) - } - return p, err -} - -// buildsEvery says a walk builds every one of the modules, and that there is one: a merge that moves nothing -// is no walk's to name. -func buildsEvery(p inventory.Plan, modules []inventory.Entry) bool { - if len(modules) == 0 { - return false - } - for _, e := range modules { - if _, in := p.Modules[e.Manifest.Module]; !in { - return false - } - } - return true -} - -// ownPath says a merge touches a module on the controller's own path, whose walk waits for nobody's word. -func ownPath(touches []inventory.Entry) bool { - for _, e := range touches { - if onTheOwnPath(e.Manifest.Module, e.Manifest) { - return true - } - } - return false -} - -// **A merge waits only for a batch it shares something with** (novox/hq ADR 0304 decision 1, as ADR 0306 -// decisions 2 and 3 read it). batchToJoin is the batch a merge heard now joins, and why in words: -// -// - an assembling or queued batch of its kind it is related to (related.go): it joins it and waits for the -// window, which its coming restarts. Related to two or more, the newer are folded into the oldest, which -// answers all their merges: a merge never waits in two windows, and what it shares with each walks in one. -// - none, and its delivery group has a member still to merge, or mesh-delivery could not say (groupWait, waits -// names why): a new batch, which waits for the window, beside the ones it shares nothing with; its words say -// why. -// - none, and no member of its group is still to merge: a new batch whose window is closed at once. Cut at -// once means only that: with a walk open that it is related to the batch is queued behind it as any closed -// batch is (ADR 0276 decision 4, ADR 0407 decision 4); beside one it shares nothing with it is cut. -func batchToJoin(ctx context.Context, inv *inventory.Inventory, heard inventory.BatchedMerge, entries []inventory.Entry, - read map[string][]inventory.ReadRepository, now time.Time, own bool, waits string) (inventory.Plan, string, error) { - edges, err := inv.Dependencies(ctx) - if err != nil { - return inventory.Plan{}, "", err - } - // **A group is one batch, whatever kinds its members are** (novox/hq ADR 0418 clause 6): a merge joins the batch - // of the other kind that holds a merge from its head. - if group, ok, err := batchOfGroup(ctx, inv, announcedOf(heard).Head, heard.Repository); err != nil { - return inventory.Plan{}, "", err - } else if ok && group.OwnPath() != own && group.Delivery != nil && group.Delivery.Batch != nil { - group.Delivery.Batch.CutAtOnce = false - return group, "its group " + announcedOf(heard).Head + " has a merge in " + group.ID + ", a batch of the other kind: " + - "a group is one batch", nil - } - joins, whys, _, err := relatedBatches(ctx, inv, []inventory.BatchedMerge{heard}, own, "", entries, read, edges) - if err != nil { - return inventory.Plan{}, "", err - } - if len(joins) > 0 { - into := joins[0] - why := "related to " + into.ID + " (" + whys[0] + ")" - if len(joins) > 1 { - plans, moves, err := foldInto(ctx, inv, &into, joins[1:]) - if err != nil { - return inventory.Plan{}, "", err - } - if err := inv.SavePlansMoving(ctx, plans, moves); err != nil { - return inventory.Plan{}, "", err - } - var folded []string - for i, b := range joins[1:] { - folded = append(folded, b.ID+" ("+whys[i+1]+")") - } - why += ", and to " + strings.Join(folded, ", ") + ", folded into it" - } - into.Delivery.Batch.CutAtOnce = false - return into, why, nil - } - if waits != "" { - // Its own window, beside the batches it shares nothing with: waiting is for its group's members, which - // are related to it by their head and join it (ADR 0306 decision 2). - b, err := newBatch(ctx, inv, heard, now, own, false, waits) - return b, waits + ", it waits for the window in a batch of its own", err - } - b, err := newBatch(ctx, inv, heard, now, own, true, "") - return b, "it shares nothing with a batch assembling and no member of a group is still to merge: it is cut at once", err -} - -// relatedBatches are the batches not yet cut of a kind, oldest first, parted into those related to merges (with -// the first reason of each) and the others. Each batch is judged by what its cut would move (ADR 0306 decision -// 3): its merges' moves as they were kept and as they read now — the catalogue may have moved since a merge was -// heard — widened to their dependents, and, for a catalogue batch, every walk waiting for its word, which the -// cut folds in (cutBatch's carry). A merge kept with no moves leaves its batch unknown, related to everything. -// except is a walk not to count as folded: the one being stopped. -func relatedBatches(ctx context.Context, inv *inventory.Inventory, merges []inventory.BatchedMerge, own bool, - except string, entries []inventory.Entry, read map[string][]inventory.ReadRepository, - edges []inventory.Edge) (joins []inventory.Plan, whys []string, others []inventory.Plan, err error) { - batches, err := inv.Batches(ctx) - if err != nil { - return nil, nil, nil, err - } - var kind []inventory.Plan - for _, b := range batches { - if b.OwnPath() == own { - kind = append(kind, b) - } - } - if len(kind) == 0 { - return nil, nil, nil, nil - } - carry := newFootprint() - if !own { - // An own-path batch folds no waiting walk (cutBatchesHeld); a catalogue batch folds those it is related to - // (cutUnrelated). Every one is counted here, which only makes a merge join a batch more readily: it waits - // for the window, never walks beside what it shares something with (issue 973). - plans, err := inv.OpenPlans(ctx) - if err != nil { - return nil, nil, nil, err - } - for _, p := range plans { - if p.Release != nil || !p.Waiting() || p.ID == except { - continue - } - kept, err := inv.MergesOf(ctx, p.ID) - if err != nil { - return nil, nil, nil, err - } - carry = unionFootprint(carry, footprintOfWalk(p, kept)) - } - } - mine := footprintAsCut(merges, entries, read, edges) - for _, b := range kind { - held, err := inv.MergesOf(ctx, b.ID) - if err != nil { - return nil, nil, nil, err - } - if b.Delivery == nil { - b.Delivery = &inventory.PlanDelivery{} - } - if b.Delivery.Batch == nil { - b.Delivery.Batch = &inventory.PlanBatch{Own: own} - } - if ok, why := related(mine, unionFootprint(footprintAsCut(held, entries, read, edges), carry), edges); ok { - joins, whys = append(joins, b), append(whys, why) - } else { - others = append(others, b) - } - } - return joins, whys, others, nil -} - -// footprintAsCut is a batch's footprint as a cut now would read it: footprintOfBatch, widened by the modules each -// merge moves as the catalogue reads now (planBatch judges each merge again at the cut, so what it moves may -// have changed since it was heard). Both are kept: when in doubt, related. -func footprintAsCut(merges []inventory.BatchedMerge, entries []inventory.Entry, - read map[string][]inventory.ReadRepository, edges []inventory.Edge) footprint { - f := footprintOfBatch(merges, edges) - var moved []string - for _, m := range merges { - s := announcedOf(m).SourceMoved - if s.Owner == "" { - continue - } - s.MergedAt = "" // news, whatever a cut marked seen since (as planBatch reads it) - for _, e := range wouldMove(s, entries, read) { - moved = append(moved, e.Manifest.Module) - } - } - for _, name := range reachableFrom(moved, edges) { - f.modules[name] = true - } - return f -} - -// unionFootprint is what two changes reach together. -func unionFootprint(a, b footprint) footprint { - f := newFootprint() - for _, from := range []footprint{a, b} { - for k := range from.repos { - f.repos[k] = true - } - for k := range from.modules { - f.modules[k] = true - } - for k := range from.heads { - f.heads[k] = true - } - } - f.unknown = a.unknown - if f.unknown == "" { - f.unknown = b.unknown - } - return f -} - -// foldInto folds batches into the oldest one related to a merge: the plans to write (into first, so no merge -// names a plan the store does not hold) and the merges to move, for one SavePlansMoving. The folded batches name -// the batch that took them over; a stop's closed window and its withheld modules (ADR 0299) go with their merges. -func foldInto(ctx context.Context, inv *inventory.Inventory, into *inventory.Plan, from []inventory.Plan) ([]*inventory.Plan, - []inventory.MergeMove, error) { - if into.Delivery == nil { - into.Delivery = &inventory.PlanDelivery{} - } - if into.Delivery.Batch == nil { - into.Delivery.Batch = &inventory.PlanBatch{Own: into.OwnPath()} - } - plans := []*inventory.Plan{into} - var moves []inventory.MergeMove - for i := range from { - f := from[i] - merges, err := inv.MergesOf(ctx, f.ID) - if err != nil { - return nil, nil, err - } - for _, m := range merges { - moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: into.ID, Alone: m.Alone}) - } - if f.Delivery == nil { - f.Delivery = &inventory.PlanDelivery{} - } - if w := f.Delivery.Batch; w != nil { - into.Delivery.Batch.Closed = into.Delivery.Batch.Closed || w.Closed - into.Delivery.Batch.Withheld = union(into.Delivery.Batch.Withheld, w.Withheld) - sort.Strings(into.Delivery.Batch.Withheld) - } - f.State = inventory.PlanSuperseded - f.Delivery.Batch = nil - f.Delivery.TakenOverBy = into.ID - f.Note = fmt.Sprintf("folded into %s before it was cut: a merge related to both joined it, and %s answers "+ - "its merges", into.ID, into.ID) - plans = append(plans, &f) - } - return plans, moves, nil -} - -// newBatch is a new batch of a kind, kept before its first merge names it: a merge never names a plan the store -// does not hold. cutAtOnce closes its window at once; waits says why it waits for its group, when it does. -func newBatch(ctx context.Context, inv *inventory.Inventory, first inventory.BatchedMerge, now time.Time, own, - cutAtOnce bool, waits string) (inventory.Plan, error) { - b := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch, - Commit: first.Commit, Merged: first.Merged, Created: now, State: inventory.PlanAssembling, - Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}, - Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own, CutAtOnce: cutAtOnce, Waits: waits}}} - if err := inv.SavePlan(ctx, &b); err != nil { - return inventory.Plan{}, err - } - return b, nil -} - -// keepBatch writes a batch as its merges and the clock say it now: the commits it holds, the merges they -// answer, its window, and assembling or queued behind a walk. Saved only when that changed. -func keepBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, now time.Time, behind string) error { - return keepBatchBecause(ctx, inv, b, now, behind, "") -} - -// keepBatchBecause is keepBatch for a batch queued behind a walk it is related to, which its words say (issue 973). -func keepBatchBecause(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, now time.Time, behind, why string) error { - merges, err := inv.MergesOf(ctx, b.ID) - if err != nil { - return err - } - window, atMost := mergeWindowOf() - // The note's seconds left are no change of the batch's: it is saved when what it holds or its state moved. - unNoted := func(p inventory.Plan) string { - p.Note = "" - return planSnapshot(p) - } - before := unNoted(*b) - commits, named := carriedOf(merges) - b.Commits = commits - if len(commits) > 0 { - newest := newestCommit(commits) - b.Repository, b.Branch, b.Commit, b.Merged = newest.Repository, newest.Branch, newest.Commit, newest.Merged - } - if b.Delivery == nil { - b.Delivery = &inventory.PlanDelivery{} - } - anchor := b.Created - if was := b.Delivery.Batch; was != nil && was.HeldAt.After(anchor) { - anchor = was.HeldAt // the maximum counts from the last hold's end (ADR 0418 clause 3) - } - closes, latest := windowOf(merges, anchor, window, atMost) - b.Delivery.Merges = named - var closed, cutAtOnce bool - var withheld []string - var waits string - var holds map[string]string - var heldAt time.Time - if was := b.Delivery.Batch; was != nil { - holds, heldAt = was.Holds, was.HeldAt - closed, withheld = was.Closed, was.Withheld // a stop's, which no look takes back (novox/hq ADR 0299) - cutAtOnce = was.CutAtOnce // the merge's, until another joins it (ADR 0304 decision 1) - waits = was.Waits // its group's, as said when it was made (ADR 0306 decision 2) - } - b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, BehindWhy: why, Own: b.OwnPath(), - Closed: closed, Withheld: withheld, CutAtOnce: cutAtOnce, Waits: waits, Holds: holds, HeldAt: heldAt} - b.State = inventory.PlanAssembling - if behind != "" && windowClosed(*b, now) { - b.State = inventory.PlanQueued - } else { - b.Delivery.Batch.Behind, b.Delivery.Batch.BehindWhy = "", "" - } - b.Note = batchWords(*b, now) - if unNoted(*b) == before { - return nil - } - if err := inv.SavePlan(ctx, b); err != nil { - return err - } - // The window's clock follows the record, only when the record moved: a merge made the quiet key again, a - // batch that opened made its most (ADR 0407 decision 1). - armWindow(ctx, *b, now) - return nil -} - -// windowOf is when a batch's window closes, unless another merge comes, and when at the latest: the last -// merge heard plus the window, and the batch's first merge plus the maximum. -func windowOf(merges []inventory.BatchedMerge, created time.Time, window, atMost time.Duration) (closes, latest time.Time) { - var last time.Time - for _, m := range merges { - if m.Heard.After(last) { - last = m.Heard - } - } - if last.IsZero() { - last = created - } - return last.Add(window), created.Add(atMost) -} - -// windowClosed says a batch's window is closed: closed by a stop, closed at once for a merge that shares -// nothing (ADR 0304 decision 1), no merge for the window's length, or its maximum reached. **A batch holding a group -// with a member still to merge is closed by none of the last three** (novox/hq ADR 0418 clause 3). -func windowClosed(b inventory.Plan, now time.Time) bool { - if b.Delivery == nil || b.Delivery.Batch == nil { - return true - } - w := b.Delivery.Batch - if w.Closed { - return true - } - if len(w.Holds) > 0 { - return false // a group's last member is still to merge: neither the window nor its maximum closes the batch - } - return w.CutAtOnce || !now.Before(w.ClosesAt) || !now.Before(w.AtMost) -} - -// carriedOf is a set of merges as a walk carries them: the latest merge of each repository's branch, and -// every merge named with the commit that carries it. -func carriedOf(merges []inventory.BatchedMerge) ([]inventory.PlanCommit, []inventory.PlanMerge) { - latest := map[string]inventory.BatchedMerge{} - spelled := map[string]string{} - for _, m := range merges { - k := strings.ToLower(m.Repository) + "\x00" + m.Branch - if l, ok := latest[k]; !ok || laterMerge(m, l) { - latest[k] = m - } - var e link.SourceMoved - if json.Unmarshal(m.Event, &e) == nil && e.Owner != "" { - spelled[k] = e.Owner + "/" + e.Repo - } - } - var commits []inventory.PlanCommit - for k, l := range latest { - repo := spelled[k] - if repo == "" { - repo = l.Repository - } - commits = append(commits, inventory.PlanCommit{Repository: repo, Branch: l.Branch, Commit: l.Commit, Merged: l.Merged}) - } - sort.Slice(commits, func(i, j int) bool { - if !strings.EqualFold(commits[i].Repository, commits[j].Repository) { - return strings.ToLower(commits[i].Repository) < strings.ToLower(commits[j].Repository) - } - return commits[i].Branch < commits[j].Branch - }) - var named []inventory.PlanMerge - for _, m := range merges { - k := strings.ToLower(m.Repository) + "\x00" + m.Branch - repo := spelled[k] - if repo == "" { - repo = m.Repository - } - n := namedOf(m, repo) - if l := latest[k]; l.Commit != m.Commit { - n.Carried = l.Commit - } - named = append(named, n) - } - sort.SliceStable(named, func(i, j int) bool { - return strings.ToLower(named[i].Repository) < strings.ToLower(named[j].Repository) - }) - return commits, named -} - -// keptMerge is a merge as the record keeps it: the forge's announcement, and the modules it moved when it was -// heard. The announcement reads back as a SourceMoved alone, which ignores the rest. -type keptMerge struct { - link.SourceMoved - Moves []string `json:"moves,omitempty"` - // FirstBuild says the merge is a first merge into a repository the mesh builds nothing from, kept to ask the first - // build of its modules once mesh-delivery's word on it is kept (novox/hq ADR 0432, issue 1146). - FirstBuild bool `json:"first_build,omitempty"` - // Held says which first builds of the merge were refused for a word the running build-agent does not read: the merge - // waits for a build-agent that reads them, and is never failed (novox/hq ADR 0482 §5, issue 1173). - Held []heldFirstBuild `json:"held,omitempty"` -} - -// announcedOf is a kept merge as the forge announced it, and what it moved; empty where the record says none. -func announcedOf(m inventory.BatchedMerge) keptMerge { - var k keptMerge - _ = json.Unmarshal(m.Event, &k) - return k -} - -// spelledOf is a kept merge's repository as the forge spelled it; empty when its announcement does not say. -func spelledOf(m inventory.BatchedMerge) string { - if e := announcedOf(m); e.Owner != "" { - return e.Owner + "/" + e.Repo - } - return "" -} - -// namedOf is one merge as a walk or batch names it: its commit, and its pull request and moves as announced. -func namedOf(m inventory.BatchedMerge, repository string) inventory.PlanMerge { - k := announcedOf(m) - return inventory.PlanMerge{Repository: repository, Commit: m.Commit, Number: k.Number, Title: k.Title, Moves: k.Moves, - Head: k.HeadSHA, Merged: m.Merged, Heard: m.Heard} -} - -// laterMerge says a was merged after b: by the forge's merge time, then by when each was heard. -func laterMerge(a, b inventory.BatchedMerge) bool { - if !a.Merged.Equal(b.Merged) { - return a.Merged.After(b.Merged) - } - return a.Heard.After(b.Heard) -} - -// newestCommit is the commit merged last among those a walk carries. -func newestCommit(commits []inventory.PlanCommit) inventory.PlanCommit { - newest := commits[0] - for _, c := range commits[1:] { - if c.Merged.After(newest.Merged) { - newest = c - } - } - return newest -} - -// nameMerges writes the merges a walk answers onto its record, after a merge heard late was given to it. -func nameMerges(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) error { - merges, err := inv.MergesOf(ctx, p.ID) - if err != nil { - return err - } - // The walk's own commits stand: a merge heard late is carried by the one of its repository's branch. - var named []inventory.PlanMerge - for _, m := range merges { - repo := m.Repository - if e := spelledOf(m); e != "" { - repo = e - } - n := namedOf(m, repo) - if c := p.CommitOn(m.Repository, m.Branch); c != "" && c != m.Commit { - n.Carried = c - } - named = append(named, n) - } - sort.SliceStable(named, func(i, j int) bool { - return strings.ToLower(named[i].Repository) < strings.ToLower(named[j].Repository) - }) - if p.Delivery == nil { - p.Delivery = &inventory.PlanDelivery{} - } - p.Delivery.Merges = named - return inv.SavePlan(ctx, p) -} - -// cutBatchesHeld takes the step that is due, for a caller holding the plans: after a failed walk, the search -// for the merge that brought the failure; with no walk open, the next merge of that search walked alone; else -// each closed batch cut into a walk with the waiting walks it is related to folded in, unless it is related to a -// walk open, behind which it is said queued (cutUnrelated). While the search runs everything waits. -func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error { - inv := open.inventory - // A merge found in two records is the half-done move of a crash: put right before anything is decided from it. - if repaired, err := inv.RepairMerges(ctx); err != nil { - return err - } else if repaired > 0 { - fmt.Printf("%d merge(s) found in two plans, the half of a move a restart interrupted, are kept by the plan they were moved to\n", repaired) - } - if err := searchAfterFailures(ctx, inv); err != nil { - return err - } - alone, err := inv.AloneMerges(ctx) - if err != nil { - return err - } - if alone, err = coverAlone(ctx, inv, alone); err != nil { - return err - } - plans, err := inv.OpenPlans(ctx) - if err != nil { - return err - } - var started, waiting []inventory.Plan - for i := range plans { - p := plans[i] - if p.Release != nil { - continue // the backlog's walk through the gates: no merge's - } - if p.Waiting() { - waiting = append(waiting, p) - } else { - started = append(started, p) - } - } - batches, err := inv.Batches(ctx) - if err != nil { - return err - } - // **A hold is looked at again at every cut** (ADR 0418 clause 3): a group whose last member merged or closed - // since is released, and its window runs again from now. - if batches, err = releaseHolds(ctx, inv, batches, now); err != nil { - return err - } - keepAll := func(behind string) error { - for i := range batches { - if err := keepBatch(ctx, inv, &batches[i], now, behind); err != nil { - return err - } - } - return nil - } - if len(alone) > 0 { - // A failed walk's search walks its merges alone, one at a time: everything waits for it (kept as it was, - // issue 973). - if len(started) > 0 { - return keepAll(started[0].ID) - } - if len(waiting) > 0 { - // The waiting walk is the one open; the search walks after it, and the batches wait behind both. - return keepAll(waiting[0].ID) - } - return walkAlone(ctx, open, alone[0], now) - } - return cutUnrelated(ctx, open, batches, started, waiting, now) -} - -// releaseHolds asks mesh-delivery again about each group a batch holds, and clears the holds of those with no member -// left to merge. A group mesh-delivery cannot be asked about stays held: the safe error is to wait. It answers the -// batches as kept. -func releaseHolds(ctx context.Context, inv *inventory.Inventory, batches []inventory.Plan, now time.Time) ([]inventory.Plan, error) { - var entries []inventory.Entry - for i := range batches { - b := &batches[i] - if b.Delivery == nil || b.Delivery.Batch == nil || len(b.Delivery.Batch.Holds) == 0 { - continue - } - if entries == nil { - var err error - if entries, err = inv.Catalogued(ctx); err != nil { - return nil, err - } - } - changed := false - for head := range b.Delivery.Batch.Holds { - open, isGroup, _, err := openMembersOf(ctx, inv, deliverySeatHeld(entries), head, "") - if err != nil || (isGroup && len(open) > 0) { - continue - } - holdFor(b, head, "", false, false, now) - changed = true - fmt.Printf("%s: group %s has no member left to merge; its window runs again from now\n", b.ID, head) - } - if changed { - if err := keepBatch(ctx, inv, b, now, ""); err != nil { - return nil, err - } - } - } - return batches, nil -} - -// cutUnrelated cuts the closed batches that share nothing with an open walk, each into a walk beside the open -// ones, and says the others queued behind the first walk they are related to (novox/hq ADR 0407 decision 4, -// issue 973). A batch cut in this look counts as open for the batches after it. -// -// - An own-path batch goes first and folds no waiting walk; a catalogue batch folds the waiting walks it is -// related to, and those it shares nothing with keep waiting. -// - **The controller's own path is related to everything** (decided for issue 973): a walk or batch that -// replaces the controller, the node-engine, the tool runner, the bus or mesh-delivery never runs beside -// another, and nothing runs beside it. -// - A batch whose window is still open is assembling; one related to an open walk is queued behind it, and -// ADR 0276 decision 4 stands for it: nothing supersedes a walk that started. -func cutUnrelated(ctx context.Context, open *stores, batches, started, waiting []inventory.Plan, now time.Time) error { - inv := open.inventory - sort.SliceStable(batches, func(i, j int) bool { return batches[i].OwnPath() && !batches[j].OwnPath() }) - var entries []inventory.Entry - var read map[string][]inventory.ReadRepository - var edges []inventory.Edge - judged := false - onPath, err := ownPathNow(ctx, inv) - if err != nil { - return err - } - openWalks, err := openWalksOf(ctx, inv, started, onPath) - if err != nil { - return err - } - for i := range batches { - b := &batches[i] - if !windowClosed(*b, now) { - if err := keepBatch(ctx, inv, b, now, ""); err != nil { - return err - } - continue - } - if !judged { - var err error - if entries, err = inv.Catalogued(ctx); err != nil { - return err - } - if read, err = readForPlanning(ctx, inv); err != nil { - return err - } - if edges, err = inv.Dependencies(ctx); err != nil { - return err - } - judged = true - } - held, err := inv.MergesOf(ctx, b.ID) - if err != nil { - return err - } - fp := footprintAsCut(held, entries, read, edges) - var fold, keep []inventory.Plan - if !b.OwnPath() { - for _, w := range waiting { - ww, err := openWalkOf(ctx, inv, w, onPath) - if err != nil { - return err - } - if ok, _ := related(fp, ww.fp, edges); ok { - fold = append(fold, w) - fp = unionFootprint(fp, ww.fp) - } else { - keep = append(keep, w) - } - } - } - if behind, why := relatedOpen(fp, b.OwnPath(), openWalks, edges); behind != "" { - if err := keepBatchBecause(ctx, inv, b, now, behind, why); err != nil { - return err - } - continue - } - if err := keepBatch(ctx, inv, b, now, ""); err != nil { - return err - } - if err := cutBatch(ctx, open, b, fold, now); err != nil { - return err - } - waiting = keep - if c := windowClockOf(); c != nil { - c.Forget(ctx, b.ID) // a batch cut has no window left to close - } - cut, err := inv.PlanByID(ctx, b.ID) - if err != nil { - return err - } - if !cut.Open() { - continue - } - w, err := openWalkOf(ctx, inv, cut, onPath) - if err != nil { - return err - } - if cut.Waiting() { - waiting = append(waiting, cut) - } else { - openWalks = append(openWalks, w) - } - } - return nil -} - -// cutBatch makes a closed batch one walk, folding in the walks that wait for their word: it keeps its id, and -// the folded walks name it as the walk that took them over. -func cutBatch(ctx context.Context, open *stores, batch *inventory.Plan, waiting []inventory.Plan, now time.Time) error { - inv := open.inventory - var carry []string - for _, w := range waiting { - merges, err := inv.MergesOf(ctx, w.ID) - if err != nil { - return err - } - for _, m := range merges { - if err := inv.AnswerMerge(ctx, m.Repository, m.Commit, batch.ID, m.Alone); err != nil { - return err - } - } - // What the folded walk was to build is built by the batch's: its merges were acted on when it was cut, - // so they read as history now. - for name := range w.Modules { - carry = append(carry, name) - } - } - if err := planBatch(ctx, open, batch, carry, now); err != nil { - return err - } - // Closed after the batch's walk is kept, never before: a controller replaced between the two leaves both - // open, which the next cut settles, rather than neither. - for i := range waiting { - w := waiting[i] - w.State = inventory.PlanSuperseded - if w.Delivery == nil { - w.Delivery = &inventory.PlanDelivery{} - } - w.Delivery.TakenOverBy = batch.ID - w.Note = fmt.Sprintf("superseded at tier %d by %s (%s) before it started: that walk answers its merges", - w.Tier, batch.ID, batch.Named()) - if err := inv.SavePlan(ctx, &w); err != nil { - return err - } - fmt.Printf(" %s is %s\n", w.ID, w.Note) - } - return nil -} - -// walkAlone walks one merge on its own commit (ADR 0276 decision 3): a plan of its own, cut at once. -func walkAlone(ctx context.Context, open *stores, m inventory.BatchedMerge, now time.Time) error { - inv := open.inventory - p := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: m.Repository, Branch: m.Branch, - Commit: m.Commit, Merged: m.Merged, Created: now, State: inventory.PlanQueued, Tiers: [][]string{}, - Modules: map[string]*inventory.PlanModule{}, Note: "walked alone after a failed walk carried it", - Delivery: &inventory.PlanDelivery{Alone: true}} - if err := inv.SavePlan(ctx, &p); err != nil { - return err - } - if err := inv.AnswerMerge(ctx, m.Repository, m.Commit, p.ID, true); err != nil { - return err - } - fmt.Printf("%s %.8s, carried by a walk that failed, is walked on its own commit as %s\n", m.Repository, m.Commit, p.ID) - return planBatch(ctx, open, &p, nil, now) -} - -// planBatch plans a batch's merges as one walk, under the batch's id: per repository the latest merge of its -// branch with every file the batch's merges of it changed, the moved modules and their dependents in tiers -// along the graph and the order of any delivery group among them. -func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry []string, now time.Time) error { - inv := open.inventory - merges, err := inv.MergesOf(ctx, batch.ID) - if err != nil { - return err - } - if len(merges) == 0 { - batch.State = inventory.PlanSuperseded - batch.Note = "a batch that holds no merge: nothing to walk" - batch.Delivery.Batch = nil - return inv.SavePlan(ctx, batch) - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return err - } - read, err := readForPlanning(ctx, inv) - if err != nil { - return err - } - edges, err := inv.Dependencies(ctx) - if err != nil { - return err - } - commits, named := carriedOf(merges) - var moved []string - var members []batchMember - unchanged := map[string]string{} - for _, m := range combinedMerges(merges) { - // **News, whatever was seen since** (review of this change): each merge was judged when it was heard, and - // a cut, a fold or a failed walk since marks its modules seen — so a merge walked alone after a failed - // walk, or folded, would read as history and move nothing. When it was made is not asked again here. - m.MergedAt = "" - // **Unchanged, said** (novox/hq issue 611): read before the merge is acted on, from the same records. - for n, why := range unchangedOfMerge(m, entries, read) { - unchanged[n] = why - } - names, err := movesOfMerge(ctx, inv, m, entries, read) - if err != nil { - return err - } - members = append(members, batchMember{merge: m, moved: names}) - for _, n := range names { - if !slices.Contains(moved, n) { - moved = append(moved, n) - } - } - } - // **Whether it waits for its delivery's word** is read from what its merges moved (novox/hq ADR 0239), never - // from what a folded walk carried along, which holds dependents too. - awaits := awaitsFor(entries, moved) - held := map[string]bool{} - for _, e := range entries { - held[e.Manifest.Module] = true - } - var also []string - for _, n := range carry { - // One the catalogue no longer holds would fail the walk's ask; it is nobody's to build. - if held[n] && !slices.Contains(moved, n) { - moved = append(moved, n) - also = append(also, n) - } - } - sort.Strings(also) - ordered, err := changeOrderEdges(ctx, merges, members, entries, moved, edges) - if err != nil { - return err - } - var withheld []string - plan := planOfMoves(moved, append(append([]inventory.Edge{}, edges...), ordered...)) - placedBy(&plan, ordered) - // **Withheld, a stopped merge's modules** (novox/hq ADR 0299): a module is built at its branch, which holds the - // merge a stop left out of the walk these merges were walked again from — built here only as a dependent, it - // would deliver that merge. One a merge of this batch moves is built: that merge carries the stopped change, - // and the stop named it. - if batch.Delivery != nil && batch.Delivery.Batch != nil { - if out := withhold(&plan, batch.Delivery.Batch.Withheld, moved); len(out) > 0 { - withheld = out - fmt.Printf(" withheld, a stopped merge's modules built only as dependents: %s\n", strings.Join(out, ", ")) - } - } - plan.ID, plan.Revision, plan.Created, plan.Epoch = batch.ID, batch.Revision, now, batch.Epoch - plan.Commits = commits - newest := newestCommit(commits) - plan.Repository, plan.Branch, plan.Commit, plan.Merged = newest.Repository, newest.Branch, newest.Commit, newest.Merged - plan.Delivery = awaits - if plan.Delivery == nil { - plan.Delivery = &inventory.PlanDelivery{} - } - plan.Delivery.Merges = named - plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone - plan.Delivery.Withheld = withheld - plan.Delivery.Order = ordered - // **Left where it is, a module whose program did not change** (novox/hq issue 721): its check found it the same as - // the build the mesh holds. - for n, why := range leaveSameAsRunning(ctx, inv, &plan, moved, append(append([]inventory.Edge{}, edges...), ordered...)) { - unchanged[n] = why - } - plan.Delivery.Unchanged = unchangedLeft(unchanged, plan.Modules) - // **Its moments and its class** (novox/hq ADR 0282 decision 6): measured, never acted on. - plan.Times = walkTimesAtCut(*batch, plan, entries, now) - if len(moved) == 0 || len(plan.Modules) == 0 { - plan.State = inventory.PlanDone - plan.Tiers = [][]string{} - plan.Note = "moved nothing the mesh holds: nothing to walk" - if len(plan.Delivery.Unchanged) > 0 { - plan.Note = "moved nothing the mesh holds: nothing to walk; " + unchangedSaid(plan.Delivery.Unchanged) - } - if err := inv.SavePlan(ctx, &plan); err != nil { - return err - } - *batch = plan - fmt.Printf("%s (%s): its merges move nothing the mesh holds\n", plan.ID, plan.Named()) - return nil - } - if hasCycle(plan.Tiers, edges) { - fmt.Printf(" the last tier depends on itself: %s — built together, in no order\n", - strings.Join(plan.Tiers[len(plan.Tiers)-1], ", ")) - } - if len(also) > 0 { - fmt.Printf(" %s, left unbuilt by a walk folded into it, are planned here again\n", strings.Join(also, ", ")) - } - var tiers []string - for i, t := range plan.Tiers { - var names []string - for _, n := range t { - if m := plan.Modules[n]; m != nil && m.Placed != "" { - n += " " + m.Placed - } - names = append(names, n) - } - tiers = append(tiers, fmt.Sprintf("%d: %s", i, strings.Join(names, ", "))) - } - var answers []string - for _, n := range named { - answers = append(answers, n.Repository+"@"+short(n.Commit)) - } - fmt.Printf("%s cut: %s, answering %s; %d module(s) in %d tier(s)\n %s\n", plan.ID, plan.Named(), - strings.Join(answers, ", "), len(plan.Modules), len(plan.Tiers), strings.Join(tiers, "\n ")) - if plan.Waiting() { - plan.Note = waitingNote(plan) - if err := inv.SavePlan(ctx, &plan); err != nil { - return err - } - *batch = plan - fmt.Printf(" %s waits for %s's word before its first tier is asked\n", plan.ID, plan.Delivery.Awaits) - return nil - } - // Written and its first tier asked as one act on the plans (novox/hq issue 213), under the caller's hold. - if err := askFirstTierAtCut(&plan, - func(p *inventory.Plan) (bool, error) { return backupsFirst(ctx, inv, p) }, - func() error { return askTier(ctx, inv, &plan, plan.Tiers[plan.Tier]) }); err != nil { - return err - } - if err := inv.SavePlan(ctx, &plan); err != nil { - return err - } - *batch = plan - return nil -} - -// askFirstTierAtCut asks a freshly cut plan's first tier, after the backups of the data it moves (novox/hq ADR 0404, -// issue 1086): the cut asks the tier itself, so advanceOnce, which asks the backups before a first tier, finds it -// asked and never takes them. A plan whose backups wait, failed or could not be worked out asks nothing; it is saved -// unasked and the walk's own step goes on from its record. -func askFirstTierAtCut(p *inventory.Plan, backups func(*inventory.Plan) (bool, error), ask func() error) error { - proceed, err := backups(p) - if err != nil { - p.Note = backupsNote + "not asked, the backups of the plan could not be worked out: " + err.Error() + - "; nothing is asked or sent, and the plan tries again" - return nil - } - if !proceed { - return nil - } - return ask() -} - -// placedBy writes on each module of a plan the edges of its order that put it after another module of the plan, with the -// rule that drew each (novox/hq ADR 0419 clause 5). -func placedBy(plan *inventory.Plan, ordered []inventory.Edge) { - for _, e := range ordered { - m, in := plan.Modules[e.From] - if _, before := plan.Modules[e.To]; !in || !before { - continue - } - said := "after " + e.To + " (" + e.Why + ")" - if m.Placed == "" { - m.Placed = said - } else if !strings.Contains(m.Placed, said) { - m.Placed += "; " + said - } - } -} - -// unchangedSaid names the modules a walk says matched what runs, for its note. -func unchangedSaid(unchanged map[string]string) string { - names := make([]string, 0, len(unchanged)) - for n := range unchanged { - names = append(names, n) - } - sort.Strings(names) - return "unchanged, the build the mesh holds is theirs: " + strings.Join(names, ", ") -} - -// unchangedLeft is what a walk says matched what runs: every module a merge left unchanged that the walk does not -// build anyway, as another merge's module or a dependent (novox/hq issue 611). Nil when none is. -func unchangedLeft(unchanged map[string]string, builds map[string]*inventory.PlanModule) map[string]string { - out := map[string]string{} - for n, why := range unchanged { - if _, built := builds[n]; !built { - out[n] = why - } - } - if len(out) == 0 { - return nil - } - return out -} - -// withhold takes from a plan every module named that its merges do not move, and the tiers left empty: the -// modules taken. -func withhold(plan *inventory.Plan, names, moved []string) []string { - var out []string - for _, n := range names { - if _, in := plan.Modules[n]; in && !slices.Contains(moved, n) { - delete(plan.Modules, n) - out = append(out, n) - } - } - if len(out) == 0 { - return nil - } - var tiers [][]string - for _, t := range plan.Tiers { - var kept []string - for _, n := range t { - if !slices.Contains(out, n) { - kept = append(kept, n) - } - } - if len(kept) > 0 { - tiers = append(tiers, kept) - } - } - if tiers == nil { - tiers = [][]string{} - } - plan.Tiers = tiers - sort.Strings(out) - return out -} - -// walkTimesAtCut is a walk's own moments as it is cut (novox/hq ADR 0282 decision 6): when its batch's window -// closed — no merge for the window's length, or its maximum, whichever came first — when it was cut, and its -// class. A merge walked alone had no window. -func walkTimesAtCut(batch, walk inventory.Plan, entries []inventory.Entry, now time.Time) *inventory.PlanTimes { - cut := now - t := &inventory.PlanTimes{Cut: &cut, Class: classOf(walk, entries)} - if batch.Delivery != nil && batch.Delivery.Batch != nil { - w := batch.Delivery.Batch - closed := w.ClosesAt - if !w.AtMost.IsZero() && w.AtMost.Before(closed) { - closed = w.AtMost - } - if w.CutAtOnce && !batch.Created.IsZero() { - closed = batch.Created // closed when it was made (ADR 0304 decision 1) - } - if !closed.IsZero() { - if closed.After(now) { - closed = now - } - t.WindowClosed = &closed - } - } - return t -} - -// resolverSeat is the seat of the mesh's resolver: a module claiming it is a core module (ADR 0282 decision 1). -const resolverSeat = "mesh-dns-resolver" - -// classOf is a walk's class (novox/hq ADR 0282 decision 1): core when it walks a module on the controller's own -// path or one holding the mesh's resolver, leaf otherwise. -func classOf(walk inventory.Plan, entries []inventory.Entry) string { - modules := make([]string, 0, len(walk.Modules)) - for m := range walk.Modules { - modules = append(modules, m) - } - return classOfModules(modules, entries) -} - -// classOfModules is the class of modules moved together: core when one is on the controller's own path or holds -// the mesh's resolver, leaf otherwise (ADR 0282 decision 1). -func classOfModules(modules []string, entries []inventory.Entry) string { - resolvers := map[string]bool{} - for _, e := range entries { - for _, c := range e.Manifest.Claims { - if c.Name == resolverSeat { - resolvers[e.Manifest.Module] = true - } - } - } - onPath := ownPathOf(entries) - for _, m := range modules { - if _, own := onPath[m]; own || resolvers[m] { - return inventory.ClassCore - } - } - return inventory.ClassLeaf -} - -// combinedMerges is a batch's merges as one merge per repository's branch: the latest, with every file the -// merges of it changed and said to be a module's — on a linear trunk the latest contains the others. A file is -// removed when the last merge of the batch that changed it removed it. merges are in the order they were made. -func combinedMerges(merges []inventory.BatchedMerge) []link.SourceMoved { - type combined struct { - latest inventory.BatchedMerge - m link.SourceMoved - said bool - removed map[string]bool - paths []string - dirs []string - cut bool - } - by := map[string]*combined{} - var keys []string - for _, b := range merges { - var e link.SourceMoved - if err := json.Unmarshal(b.Event, &e); err != nil { - continue - } - k := strings.ToLower(b.Repository) + "\x00" + b.Branch - c, ok := by[k] - if !ok { - c = &combined{latest: b, m: e, said: true, removed: map[string]bool{}} - by[k] = c - keys = append(keys, k) - } else if laterMerge(b, c.latest) { - c.latest, c.m = b, e - } - c.said = c.said && e.ModuleDirsSaid - c.cut = c.cut || e.PathsTruncated - c.paths, c.dirs = union(c.paths, e.Paths), union(c.dirs, e.ModuleDirs) - for _, path := range e.Paths { - c.removed[path] = slices.Contains(e.Removed, path) - } - for _, path := range e.Removed { - c.removed[path] = true - } - } - sort.Strings(keys) - var out []link.SourceMoved - for _, k := range keys { - c := by[k] - m := c.m - m.Paths, m.ModuleDirs, m.PathsTruncated, m.ModuleDirsSaid = c.paths, c.dirs, c.cut, c.said - m.Removed = nil - for _, path := range c.paths { - if c.removed[path] { - m.Removed = append(m.Removed, path) - } - } - out = append(out, m) - } - return out -} - -// union is a and b without repeats, in order. -func union(a, b []string) []string { - out := append([]string{}, a...) - for _, s := range b { - if !slices.Contains(out, s) { - out = append(out, s) - } - } - return out -} - -// batchMember is one repository's merge in a batch and what it moves. -type batchMember struct { - merge link.SourceMoved - moved []string -} - -// changeOrderEdges is the order of a batch's moved modules as edges of its plan's tiers (novox/hq ADR 0419): drawn -// per module from the module graph and from what the merges say of the manifests they touch (delivery_edges.go), -// whatever repository the merges are in and whether or not they form a delivery group. No `after:` line is read. -// A cycle is said, never lost: the plan puts the modules left in its last tier, and the composed check fails the -// group before it merges. -func changeOrderEdges(ctx context.Context, merges []inventory.BatchedMerge, members []batchMember, - entries []inventory.Entry, moved []string, edges []inventory.Edge) ([]inventory.Edge, error) { - // What each repository's branch held before the batch: the first parent of its earliest merge in it. - earliest := map[string]inventory.BatchedMerge{} - for _, k := range merges { - key := strings.ToLower(k.Repository) + "\x00" + k.Branch - if e, ok := earliest[key]; !ok || laterMerge(e, k) { - earliest[key] = k - } - } - var sources []changeSource - for _, m := range members { - s := changeSource{Owner: m.merge.Owner, Repo: m.merge.Repo, New: m.merge.Commit, - Paths: m.merge.Paths, Truncated: m.merge.PathsTruncated, Removed: m.merge.Removed} - if e, ok := earliest[strings.ToLower(m.merge.Owner+"/"+m.merge.Repo)+"\x00"+m.merge.Base]; ok { - s.Since = e.Commit - } - if s.Truncated { - s.Paths, s.Truncated = truncatedPaths(entries, s.Owner, s.Repo, moved), false - } - sources = append(sources, s) - } - changes, err := changesOf(ctx, sources) - if err != nil { - return nil, fmt.Errorf("what the merges say of their manifests cannot be read, so the order of their modules "+ - "cannot be drawn: %w", err) - } - order := orderEdges(moved, changes, edges) - for _, e := range order.Edges { - fmt.Printf(" order: %s\n", edgeSaid(e)) - } - if said := order.CycleSaid(); said != "" { - fmt.Printf(" %s\n", said) - } - return order.Edges, nil -} - -// searchAfterFailures starts the search a failed walk leaves (ADR 0276 decision 3): every merge it answered -// in a later commit of its repository waits to be walked on its own commit. A stopped walk starts none: a -// person ended it. -func searchAfterFailures(ctx context.Context, inv *inventory.Inventory) error { - plans, err := inv.RecentPlans(ctx, 20) - if err != nil { - return err - } - for _, p := range plans { - if p.State != inventory.PlanFailed || p.Delivery == nil || p.Delivery.Stopped != "" || - strings.HasPrefix(p.Note, "stopped by hand") || strings.HasPrefix(p.Note, "closed by hand") { - continue - } - merges, err := inv.MergesOf(ctx, p.ID) - if err != nil { - return err - } - for _, m := range merges { - if m.Alone || p.CommitOf(m.Repository) == m.Commit { - continue // its own commit was walked - } - if err := inv.AnswerMerge(ctx, m.Repository, m.Commit, "", true); err != nil { - return err - } - fmt.Printf("%s failed; %s %.8s, which it carried in %.8s, is walked on its own commit\n", p.ID, - m.Repository, m.Commit, p.CommitOf(m.Repository)) - } - } - return nil -} - -// coverAlone gives every merge waiting to be walked alone to a walk done that holds a later merge of its -// branch — one walked alone, or the failed walk retried by a person: the search stops at the first merge -// delivered, and that walk answers the older ones. What is left waits, the newest first. -func coverAlone(ctx context.Context, inv *inventory.Inventory, alone []inventory.BatchedMerge) ([]inventory.BatchedMerge, error) { - var left []inventory.BatchedMerge - for _, m := range alone { - later, err := inv.LaterMergesOf(ctx, m.Repository, m.Branch, m.Merged) - if err != nil { - return nil, err - } - covered := false - for _, l := range later { - p, err := inv.PlanByID(ctx, l.Plan) - if err != nil { - return nil, err - } - if p.State != inventory.PlanDone { - continue - } - if err := inv.AnswerMerge(ctx, m.Repository, m.Commit, p.ID, true); err != nil { - return nil, err - } - if err := nameMerges(ctx, inv, &p); err != nil { - return nil, err - } - fmt.Printf("%s %.8s is answered by %s, which delivered a later merge of its branch on its own commit\n", - m.Repository, m.Commit, p.ID) - covered = true - break - } - if !covered { - left = append(left, m) - } - } - return left, nil -} - -// batchWords is a batch in the operator's words (ADR 0276 decision 7): "assembling: 13 s left (at the latest -// 00:31:09); grouped: novox/mesh-catalog@48bda475 (answers 553b7191), …; plan not yet calculated". -func batchWords(b inventory.Plan, now time.Time) string { - grouped := groupedWords(b) - if b.Delivery == nil || b.Delivery.Batch == nil { - return "grouped: " + grouped + "; plan not yet calculated" - } - w := b.Delivery.Batch - if w.Own { - grouped += " (on the controller's own path)" - } - if b.State == inventory.PlanQueued { - because := "" - if w.BehindWhy != "" { - because = " (" + w.BehindWhy + ")" - } - return fmt.Sprintf("queued behind %s%s; grouped: %s; plan not yet calculated", w.Behind, because, grouped) - } - if w.CutAtOnce { - return fmt.Sprintf("cut at once, sharing nothing with a batch assembling; grouped: %s; plan not yet calculated", - grouped) - } - left := w.ClosesAt.Sub(now) - if w.AtMost.Before(w.ClosesAt) { - left = w.AtMost.Sub(now) - } - if left < 0 { - left = 0 - } - waits := "" - if w.Waits != "" { - waits = " — " + w.Waits - } - return fmt.Sprintf("assembling: %s left (at the latest %s)%s; grouped: %s; plan not yet calculated", - secondsWords(left), w.AtMost.Local().Format("15:04:05"), waits, grouped) -} - -// groupedWords is a batch's commits, each with the earlier merges it answers. -func groupedWords(b inventory.Plan) string { - var out []string - for _, c := range b.Carried() { - s := c.Repository + "@" + short(c.Commit) - var answers []string - if b.Delivery != nil { - for _, m := range b.Delivery.Merges { - if strings.EqualFold(m.Repository, c.Repository) && m.Carried == c.Commit { - answers = append(answers, short(m.Commit)) - } - } - } - if len(answers) > 0 { - s += " (answers " + strings.Join(answers, ", ") + ")" - } - out = append(out, s) - } - if len(out) == 0 { - return "nothing yet" - } - return strings.Join(out, ", ") -} - -// batchLines is a batch's grouped list as `plans` prints it under its line (asked by the operator, 2026-10-10): -// one line per repository, its pull request and title, the earlier merges it answers, and what its merges move. -func batchLines(b inventory.Plan) []string { - var out []string - for _, c := range b.Carried() { - var head *inventory.PlanMerge - var answers []string - moves := map[string]bool{} - if b.Delivery != nil { - for i := range b.Delivery.Merges { - m := &b.Delivery.Merges[i] - if !strings.EqualFold(m.Repository, c.Repository) { - continue - } - for _, n := range m.Moves { - moves[n] = true - } - switch { - case m.Commit == c.Commit: - head = m - case m.Carried == c.Commit: - answers = append(answers, pullWords(*m)) - } - } - } - line := repoName(c.Repository) + " " + short(c.Commit) - if head != nil && head.Number > 0 { - line = repoName(c.Repository) + " " + pullWords(*head) - } - if len(answers) > 0 { - line += " (answers " + strings.Join(answers, ", ") + ")" - } - if len(moves) > 0 { - line += " · " + strings.Join(sortedKeysOf(boolsToStrings(moves)), ", ") - } - out = append(out, line) - } - return out -} - -// pullWords is a merge as its pull request: "#175 a tap shows its outcome", the title cut at fifty runes; the -// commit where the announcement named no pull request. -func pullWords(m inventory.PlanMerge) string { - if m.Number == 0 { - return short(m.Commit) - } - s := fmt.Sprintf("#%d", m.Number) - if t := cutTitle(m.Title, 50); t != "" { - s += " " + t - } - return s -} - -// cutTitle is a title cut at n runes, with an ellipsis where it was cut. -func cutTitle(title string, n int) string { - r := []rune(strings.TrimSpace(title)) - if len(r) <= n { - return string(r) - } - return strings.TrimSpace(string(r[:n-1])) + "…" -} - -// boolsToStrings is a set's members, for sortedKeysOf. -func boolsToStrings(set map[string]bool) map[string]string { - out := make(map[string]string, len(set)) - for k := range set { - out[k] = "" - } - return out -} - -// secondsWords is a short wait as a person reads it. -func secondsWords(d time.Duration) string { - if d < 2*time.Minute { - return fmt.Sprintf("%d s", int(d.Round(time.Second)/time.Second)) - } - return d.Round(time.Second).String() -} - -// walkOrBatch is what a plan record is, in a word. -func walkOrBatch(p inventory.Plan) string { - if p.Batch() { - return "batch" - } - return "walk" -} - -// orWords is s, or the words for when it is empty. -func orWords(s, none string) string { - if s == "" { - return none - } - return s -} diff --git a/cmd/mesh-controller/batches_integration_test.go b/cmd/mesh-controller/batches_integration_test.go deleted file mode 100644 index 38c83a2b..00000000 --- a/cmd/mesh-controller/batches_integration_test.go +++ /dev/null @@ -1,727 +0,0 @@ -//go:build integration - -package main - -import ( - "bytes" - "io" - "os" - "slices" - "strings" - "sync/atomic" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -func windowed(t *testing.T) *stores { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - settings := t.TempDir() + "/merge-window.json" - if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil { - t.Fatal(err) - } - t.Setenv(mergeWindowFileVar, settings) - noGroups(t) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, - inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0", - Head: "c0"}); err != nil { - t.Fatal(err) - } - for _, m := range []struct{ module, repository, path string }{ - {"app", "novox/mesh-catalog", "modules/app"}, - {"notes", "novox/mesh-catalog", "modules/notes"}, - {"one", "novox/one", ""}, - {"two", "novox/two", ""}, - {"three", "novox/three", ""}, - } { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1"}, - inventory.Source{Repository: m.repository, Seat: "git", Path: m.path, Ref: "main", BuiltFrom: "c0", - Head: "c0"}); err != nil { - t.Fatal(err) - } - } - return open -} - -func hear(t *testing.T, open *stores, m link.SourceMoved, now time.Time) { - t.Helper() - if err := (following{open: open}).hearMerge(t.Context(), m, now); err != nil { - t.Fatal(err) - } -} - -func cutAt(t *testing.T, open *stores, now time.Time) { - t.Helper() - if err := cutBatchesHeld(t.Context(), open, now); err != nil { - t.Fatal(err) - } -} - -func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - claude := catalogueMerge("553b7191claude", "app", t0) - dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second)) - hear(t, open, claude, t0.Add(time.Second)) - hear(t, open, dunst, t0.Add(18*time.Second)) - if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 { - t.Fatalf("within the window: %d walk(s), %d batch(es)", len(ws), len(bs)) - } - cutAt(t, open, t0.Add(18*time.Second+89*time.Second)) - if ws, _ := walks(t, open); len(ws) != 0 { - t.Fatalf("cut before the window closed: %+v", ws) - } - cutAt(t, open, t0.Add(18*time.Second+90*time.Second)) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 0 { - t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs)) - } - w := ws[0] - if w.Commit != dunst.Commit || len(w.Commits) != 1 || w.Commits[0].Commit != dunst.Commit { - t.Fatalf("the walk is at %s %+v, not the later commit", w.Commit, w.Commits) - } - for _, m := range []string{"app", "notes"} { - if _, in := w.Modules[m]; !in { - t.Fatalf("the walk does not build %s, which one of its merges moved: %v", m, w.Modules) - } - } - want := []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: claude.Commit, Carried: dunst.Commit}, - {Repository: "novox/mesh-catalog", Commit: dunst.Commit}} - if w.Delivery == nil || !slices.EqualFunc(w.Delivery.Merges, want, sameMerge) { - t.Fatalf("the walk answers %+v, want %+v", w.Delivery, want) - } - // The cut takes its first tier from the builder's keys (a look takes them in later; none runs here), and says at - // which commit: the one the branch holds, which is the later merge the walk carries. - for _, m := range w.Tiers[w.Tier] { - if s := w.Modules[m]; s == nil || !s.Key || s.State != "asked" || s.KeyAt != dunst.Commit { - t.Fatalf("the walk did not take %s from its key at the commit it carries (%s): %+v", m, dunst.Commit, s) - } - } - // Heard again, as the bus may hand it over twice: never a second merge, nor a second walk. - hear(t, open, claude, t0.Add(5*time.Minute)) - if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 { - t.Fatalf("a merge heard twice made %d walk(s) and %d batch(es)", len(ws), len(bs)) - } -} - -func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "feat/three") - asksRecorded(t) - for i, r := range []string{"one", "two", "three"} { - hear(t, open, fromHead(repoMerge(r, "c-"+r, t0.Add(time.Duration(i)*20*time.Second)), "feat/three"), - t0.Add(time.Duration(i)*20*time.Second+time.Second)) - } - cutAt(t, open, t0.Add(41*time.Second+90*time.Second)) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 0 { - t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs)) - } - w := ws[0] - if len(w.Commits) != 3 || len(w.Delivery.Merges) != 3 { - t.Fatalf("the walk carries %+v and answers %+v", w.Commits, w.Delivery.Merges) - } - for _, r := range []string{"one", "two", "three"} { - if w.CommitOf("novox/"+r) != "c-"+r { - t.Fatalf("the walk carries %s at %q", r, w.CommitOf("novox/"+r)) - } - if _, in := w.Modules[r]; !in { - t.Fatalf("the walk does not build %s: %v", r, w.Modules) - } - } - // Each module of the first tier is taken from its key at its own repository's commit (the cut asks no build). - for _, r := range w.Tiers[w.Tier] { - if s := w.Modules[r]; s == nil || !s.Key || s.State != "asked" || s.KeyAt != "c-"+r { - t.Fatalf("%s was not taken from its key at its own commit: %+v", r, s) - } - } -} - -func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "feat/both", "feat/busy") - asksWithPaths(t) - hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0) - hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(80*time.Second)), "feat/both"), t0.Add(80*time.Second)) - cutAt(t, open, t0.Add(100*time.Second)) - if ws, _ := walks(t, open); len(ws) != 0 { - t.Fatal("the window closed 90 seconds after its first merge, not after its last") - } - cutAt(t, open, t0.Add(170*time.Second)) - if ws, _ := walks(t, open); len(ws) != 1 || len(ws[0].Commits) != 2 { - t.Fatalf("the window did not close 90 seconds after its last merge: %+v", ws) - } - - busy := windowed(t) - groupsWaiting(t, "feat/both", "feat/busy") - asksWithPaths(t) - var last time.Time - for i := 0; i < 12; i++ { - last = t0.Add(time.Duration(i) * time.Minute) - repo := []string{"one", "two", "three"}[i%3] - hear(t, busy, fromHead(repoMerge(repo, "c"+string(rune('a'+i)), last), "feat/busy"), last) - if ws, _ := walks(t, busy); len(ws) > 0 { - if i != 10 { - t.Fatalf("a busy window closed at merge %d (minute %d), not at its maximum", i, i) - } - break - } - } - ws, _ := walks(t, busy) - if len(ws) != 1 { - t.Fatalf("ten minutes of merges a minute apart were never cut: %d walk(s)", len(ws)) - } -} - -func TestTheAssemblingBatchIsShown(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/553b7191claude") - asksWithPaths(t) - var said []inventory.Plan - was := inventory.PlanSaved - inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { said = append(said, p) } - t.Cleanup(func() { inventory.PlanSaved = was }) - now := time.Now().UTC() - claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second)) - claude.Number, claude.Title = 174, "claude-code: an agent proposes a section" - dunst := catalogueMerge("48bda475dunst", "notes", now.Add(-2*time.Second)) - dunst.Number, dunst.Title = 175, "dunst: the font the operator chose" - hear(t, open, claude, now.Add(-19*time.Second)) - hear(t, open, dunst, now.Add(-time.Second)) - hear(t, open, fromHead(repoMerge("one", "a6bc0931one", now), dunst.Head), now) - out := captured(t, func() error { return plansCommand(t.Context(), nil) }) - lines := strings.Split(out, "\n") - first := lines[0] - for _, want := range []string{"assembling: ", " s left (at the latest ", "grouped: novox/mesh-catalog@48bda475 " + - "(answers 553b7191), novox/one@a6bc0931; plan not yet calculated"} { - if !strings.Contains(first, want) { - t.Fatalf("plans' first line %q does not say %q", first, want) - } - } - // Under it, one line per repository: the pull request, what it answers, what it moves. - if len(lines) < 3 || strings.TrimSpace(lines[1]) != "mesh-catalog #175 dunst: the font the operator chose (answers "+ - "#174 claude-code: an agent proposes a section) · app, notes" || - strings.TrimSpace(lines[2]) != "one a6bc0931 · one" { - t.Fatalf("the grouped list reads %q", lines[1:4]) - } - if len(said) == 0 || said[len(said)-1].State != inventory.PlanAssembling || said[len(said)-1].Delivery.Batch == nil || - len(said[len(said)-1].Delivery.Merges) != 3 { - t.Fatalf("the batch was not said as assembling with its merges: %+v", said) - } -} - -func captured(t *testing.T, run func() error) string { - t.Helper() - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - was := os.Stdout - os.Stdout = w - runErr := run() - os.Stdout = was - _ = w.Close() - var b bytes.Buffer - _, _ = io.Copy(&b, r) - if runErr != nil { - t.Fatal(runErr) - } - return b.String() -} - -func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(90*time.Second)) - ws, _ := walks(t, open) - if len(ws) != 1 || !ws[0].Open() { - t.Fatalf("the first batch was not cut: %+v", ws) - } - first := ws[0] - hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute)) - cutAt(t, open, t0.Add(5*time.Minute)) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID { - t.Fatalf("the merge during the walk: %d walk(s), batches %+v", len(ws), bs) - } - if got, _ := open.inventory.PlanByID(t.Context(), first.ID); !got.Open() { - t.Fatalf("the started walk was %s by the next merge", got.State) - } - if !strings.HasPrefix(batchWords(bs[0], t0.Add(5*time.Minute)), "queued behind "+first.ID) { - t.Fatalf("a queued batch reads %q", batchWords(bs[0], t0.Add(5*time.Minute))) - } - first.State = inventory.PlanDone - if err := open.inventory.SavePlan(t.Context(), &first); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(6*time.Minute)) - ws, bs = walks(t, open) - if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" { - t.Fatalf("the queued batch was not cut when the walk ended: %+v %+v", ws, bs) - } -} - -func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - ctx := t.Context() - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(90*time.Second)) - ws, _ := walks(t, open) - if len(ws) != 1 || !ws[0].Waiting() { - t.Fatalf("the walk does not wait for its word: %+v", ws) - } - waiting := ws[0] - hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute)) - cutAt(t, open, t0.Add(2*time.Minute+90*time.Second)) - ws, bs := walks(t, open) - if len(bs) != 0 || len(ws) != 2 { - t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs)) - } - folded, err := open.inventory.PlanByID(ctx, waiting.ID) - if err != nil { - t.Fatal(err) - } - newer := ws[0] - if folded.State != inventory.PlanSuperseded || folded.Delivery.TakenOverBy != newer.ID { - t.Fatalf("the waiting walk is %s, taken over by %q", folded.State, folded.Delivery.TakenOverBy) - } - if len(newer.Delivery.Merges) != 2 || newer.CommitOf("novox/one") != "c2" { - t.Fatalf("the newer walk answers %+v", newer.Delivery.Merges) - } - if _, in := newer.Modules["one"]; !in { - t.Fatalf("what the folded walk was to build is not built: %v", newer.Modules) - } -} - -func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - later := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second)) - hear(t, open, later, t0.Add(18*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - w := ws[0] - w.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &w); err != nil { - t.Fatal(err) - } - // Moves only notes, which the done walk built from the branch after it: answered there, at once. - hear(t, open, catalogueMerge("553b7191early", "notes", t0), t0.Add(15*time.Minute)) - got, _ := open.inventory.PlanByID(ctx, w.ID) - if got.State != inventory.PlanDone || len(got.Delivery.Merges) != 2 || - !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: "553b7191early", - Carried: later.Commit}) { - t.Fatalf("the late merge is not named by the walk that carried it: %+v", got.Delivery.Merges) - } - // Moves app, which that walk never built: the next batch walks it. - hear(t, open, catalogueMerge("1111aaaaapp", "app", t0.Add(time.Second)), t0.Add(16*time.Minute)) - _, bs := walks(t, open) - if len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "1111aaaaapp" { - t.Fatalf("a late merge moving what the walk never built did not join the next batch: %+v", bs) - } -} - -func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - // Made before the cut marks their modules seen, as merges are: walked again, they are news all the same. - oldest := catalogueMerge("aaaa0001", "app", t0.Add(-3*time.Minute)) - middle := catalogueMerge("bbbb0002", "app", t0.Add(-3*time.Minute+10*time.Second)) - newest := catalogueMerge("cccc0003", "notes", t0.Add(-3*time.Minute+20*time.Second)) - for i, m := range []link.SourceMoved{oldest, middle, newest} { - hear(t, open, m, t0.Add(time.Duration(i)*10*time.Second+time.Second)) - } - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - failed := ws[0] - failed.State, failed.Note = inventory.PlanFailed, "notes failed to build in tier 0" - if err := open.inventory.SavePlan(ctx, &failed); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(3*time.Minute)) - got, _ := open.inventory.PlanByID(ctx, failed.ID) - if got.State != inventory.PlanFailed || len(got.Delivery.Merges) != 3 { - t.Fatalf("the failed walk's record changed: %s %+v", got.State, got.Delivery.Merges) - } - ws, _ = walks(t, open) - alone := ws[0] - if alone.ID == failed.ID || alone.Commit != middle.Commit || len(alone.Delivery.Merges) != 1 || - !sameMerge(alone.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) { - t.Fatalf("the newest earlier merge was not walked alone on its own commit: %+v", alone) - } - // The cut takes the first tier from the builder's keys and asks no build, so nothing is recorded as asked: the - // module is taken from its key at the commit the merge was walked on. - if app := alone.Modules["app"]; app == nil || !app.Key || app.State != "asked" || app.KeyAt != middle.Commit { - t.Fatalf("the merge walked alone does not build app at its own commit: %v", alone.Modules) - } - // Retried, the failed walk would name merges the search answers now. - if _, err := retryPlan(ctx, open, failed.ID); err == nil || !strings.Contains(err.Error(), "walked alone") { - t.Fatalf("a failed walk whose merges are searched was retried: %v", err) - } - alone.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &alone); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(4*time.Minute)) - ws, bs := walks(t, open) - if ws[0].ID != alone.ID || len(bs) != 0 { - t.Fatalf("the search went on after a merge was delivered: %+v", ws[0]) - } - done, _ := open.inventory.PlanByID(ctx, alone.ID) - if len(done.Delivery.Merges) != 2 || !sameMerge(done.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", - Commit: oldest.Commit, Carried: middle.Commit}) { - t.Fatalf("the oldest merge is not answered by the walk that delivered the one after it: %+v", done.Delivery.Merges) - } - // A stopped walk starts no search: a person ended it. - hear(t, open, catalogueMerge("dddd0004", "app", t0.Add(5*time.Minute)), t0.Add(5*time.Minute)) - hear(t, open, catalogueMerge("eeee0005", "notes", t0.Add(5*time.Minute+time.Second)), t0.Add(5*time.Minute+time.Second)) - cutAt(t, open, t0.Add(8*time.Minute)) - ws, _ = walks(t, open) - if _, err := stopWalk(ctx, open.inventory, ws[0].ID, "mesh-delivery for jochen", "not now"); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(9*time.Minute)) - if again, _ := walks(t, open); again[0].ID != ws[0].ID { - t.Fatalf("a stopped walk's earlier merge was walked alone: %+v", again[0]) - } -} - -func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) { - for _, grouped := range []bool{true, false} { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - for _, m := range []string{"mesh-host"} { - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, - inventory.Source{Repository: "novox/" + m, Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - } - host := repoMerge("mesh-host", "h1", t0) - ctl := repoMerge("mesh-controller", "k1", t0.Add(time.Second)) - host.Head, ctl.Head = "feat/together", "feat/together" - if !grouped { - ctl.Head = "feat/alone" - } - hear(t, open, ctl, t0.Add(2*time.Second)) - hear(t, open, host, t0.Add(3*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - if len(ws) != 1 { - t.Fatalf("%d walks", len(ws)) - } - tiers := ws[0].Tiers - if grouped { - if len(tiers) != 2 || !slices.Equal(tiers[0], []string{"mesh-host"}) || - !slices.Equal(tiers[1], []string{"mesh-controller"}) { - t.Fatalf("the group's order is not the walk's tiers: %v", tiers) - } - } else if len(tiers) != 1 { - t.Fatalf("two merges of no group were ordered: %v", tiers) - } - } -} - -func TestABatchSurvivesARestart(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "feat/both") - asksWithPaths(t) - hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0) - hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(30*time.Second)), "feat/both"), t0.Add(30*time.Second)) - - // A new controller: nothing of the first one's but the store. - again, err := openStores(t.Context()) - if err != nil { - t.Fatal(err) - } - t.Cleanup(again.Close) - hear(t, again, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0.Add(60*time.Second)) // the bus hands it over again - cutAt(t, again, t0.Add(119*time.Second)) - if ws, bs := walks(t, again); len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 { - t.Fatalf("the restarted controller's batch: %d walk(s), %+v", len(ws), bs) - } - cutAt(t, again, t0.Add(120*time.Second)) - ws, bs := walks(t, again) - if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 { - t.Fatalf("the restarted controller did not cut the batch at its window: %+v %+v", ws, bs) - } -} - -func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - c2 := repoMerge("one", "c2", t0.Add(-50*time.Second)) - hear(t, open, c2, t0) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - w := ws[0] - w.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &w); err != nil { - t.Fatal(err) - } - hear(t, open, repoMerge("one", "c1", t0.Add(-60*time.Second)), t0.Add(15*time.Minute)) - got, _ := open.inventory.PlanByID(ctx, w.ID) - if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one", - Commit: "c1", Carried: "c2"}) { - t.Fatalf("the late merge was not named by the walk that carried it: %+v", got.Delivery.Merges) - } - - // A walk of two that built only two: a late merge of one is not its to name. - hear(t, open, repoMerge("two", "d2", t0.Add(16*time.Minute)), t0.Add(16*time.Minute)) - cutAt(t, open, t0.Add(18*time.Minute)) - ws, _ = walks(t, open) - w = ws[0] - delete(w.Modules, "two") - w.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &w); err != nil { - t.Fatal(err) - } - hear(t, open, repoMerge("two", "d1", t0.Add(15*time.Minute)), t0.Add(20*time.Minute)) - if got, _ := open.inventory.PlanByID(ctx, w.ID); len(got.Delivery.Merges) != 1 { - t.Fatalf("a walk that never built what the late merge moves named it: %+v", got.Delivery.Merges) - } - if _, bs := walks(t, open); len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "d1" { - t.Fatalf("the late merge did not join the next batch: %+v", bs) - } -} - -func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) { - open := windowed(t) - asked := asksWithPaths(t) - // The started walk's key (one at c1) is not built, so that walk stays started as it did while its build ran: with - // the stand-in builder's key done at once, one step would carry it to done and nothing would be beside the other. - wholeKeys, held := openPlanKeys, &atomic.Bool{} - held.Store(true) - openPlanKeys = func() (planKeyReads, func(), error) { - all, done, err := wholeKeys() - return keysHolding{all, "one", held, fpOfModuleAt("one", "c1")}, done, err - } - ctx := t.Context() - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(2*time.Minute)) - started, _ := walks(t, open) - waiting := inventory.Plan{ID: "plan-waiting", Repository: "novox/one", Branch: "main", Commit: "d1", Created: t0, - State: inventory.PlanBuilding, Tiers: [][]string{{"one"}}, Modules: map[string]*inventory.PlanModule{"one": {}}, - Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}} - if err := open.inventory.SavePlan(ctx, &waiting); err != nil { - t.Fatal(err) - } - if _, err := letGo(ctx, open.inventory, waiting.ID, catalogue.DeliverySeat, "its turn"); err != nil { - t.Fatalf("the word was refused beside %s: %v", started[0].ID, err) - } - before := len(*asked) - advanceHeld(ctx, open) - got, _ := open.inventory.PlanByID(ctx, waiting.ID) - if len(*asked) != before || !strings.Contains(got.Note, "starts once "+started[0].ID) { - t.Fatalf("a walk let go beside a started one asked (%d → %d) or does not say it waits: %q", before, len(*asked), got.Note) - } - // Read as a wait, an hour on: its note on the line, never LATE, and no tier of it late for S3. - later := time.Now().Add(time.Hour) - if line := planLine(got, later); !strings.Contains(line, "starts once "+started[0].ID) || strings.Contains(line, "LATE") { - t.Fatalf("a deferred walk reads %q", line) - } - facts, _, err := gatherPlans(ctx, open.inventory, later, nil) - if err != nil { - t.Fatal(err) - } - for _, f := range facts { - if f.id == got.ID { - t.Fatalf("a deferred walk is watched as a tier running late: %+v", f) - } - } - // Read again: the steps since wrote the started walk (what its key waits for), so the copy read at its cut is stale. - done, err := open.inventory.PlanByID(ctx, started[0].ID) - if err != nil { - t.Fatal(err) - } - done.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &done); err != nil { - t.Fatal(err) - } - advanceHeld(ctx, open) - if len(*asked) != before+1 { - t.Fatalf("the walk did not start once the started one ended: asked %v", *asked) - } -} - -func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asked := asksWithPaths(t) - ctx := t.Context() - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - hear(t, open, catalogueMerge("aaaa0001", "app", t0), t0) - hear(t, open, repoMerge("mesh-controller", "k1", t0.Add(10*time.Second)), t0.Add(10*time.Second)) - hear(t, open, catalogueMerge("bbbb0002", "notes", t0.Add(20*time.Second)), t0.Add(20*time.Second)) - _, bs := walks(t, open) - var ownBatch, catalogueBatch inventory.Plan - for _, b := range bs { - if b.OwnPath() { - ownBatch = b - } else { - catalogueBatch = b - } - } - if len(bs) != 2 || ownBatch.ID == "" || catalogueBatch.ID == "" || len(catalogueBatch.Delivery.Merges) != 2 || - len(ownBatch.Delivery.Merges) != 1 { - t.Fatalf("a controller merge and two catalogue merges in one window: %+v", bs) - } - if !strings.Contains(batchWords(ownBatch, t0.Add(30*time.Second)), "own path") { - t.Fatalf("the own-path batch does not say so: %q", batchWords(ownBatch, t0.Add(30*time.Second))) - } - // Both windows closed, the controller's batch is cut first and starts; the catalogue batch queues behind it, - // is cut when the controller's walk ended, and waits for its word with both merges. - cutAt(t, open, t0.Add(2*time.Minute)) - ws, bs := walks(t, open) - if len(ws) != 1 || ws[0].Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k1" { - t.Fatalf("the controller's batch was not cut first into a started walk: %+v", ws) - } - ownWalk := ws[0] - for _, m := range []string{"app", "notes"} { - if _, in := ownWalk.Modules[m]; in { - t.Fatalf("the controller's walk builds %s, a catalogue module: it skipped its turn", m) - } - } - if len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != ownWalk.ID || bs[0].OwnPath() { - t.Fatalf("the catalogue batch does not queue behind the controller's walk: %+v", bs) - } - ownWalk.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &ownWalk); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(3*time.Minute)) - ws, bs = walks(t, open) - if len(bs) != 0 || ws[0].ID != catalogueBatch.ID || !ws[0].Waiting() || len(ws[0].Delivery.Merges) != 2 { - t.Fatalf("the catalogue batch was not cut into a waiting walk once the controller's ended: %+v %+v", ws, bs) - } - for _, m := range []string{"app", "notes"} { - if _, in := ws[0].Modules[m]; !in { - t.Fatalf("the catalogue walk does not build %s: %v", m, ws[0].Modules) - } - } - for _, a := range *asked { - if a[1] == "modules/app" || a[1] == "modules/notes" { - t.Fatalf("a catalogue module was asked without the word: %v", *asked) - } - } - - // A catalogue walk waiting for its word when an own-path batch is cut keeps waiting beside the own-path - // walk, is not folded into it, and its word is taken meanwhile: it starts once the own-path walk ended. - catalogueWalk := ws[0] - // The controller's second walk keeps its key unbuilt, so it stays started as it did while its build ran; with the - // stand-in builder's key done at once one step would carry it to done and nothing would be beside the catalogue walk. - wholeKeys, held := openPlanKeys, &atomic.Bool{} - held.Store(true) - openPlanKeys = func() (planKeyReads, func(), error) { - all, done, err := wholeKeys() - return keysHolding{all, "mesh-controller", held, fpOfModuleAt("mesh-controller", "k2")}, done, err - } - hear(t, open, repoMerge("mesh-controller", "k2", t0.Add(4*time.Minute)), t0.Add(4*time.Minute)) - cutAt(t, open, t0.Add(6*time.Minute)) - ws, _ = walks(t, open) - kept, _ := open.inventory.PlanByID(ctx, catalogueWalk.ID) - if !kept.Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k2" || ws[0].Waiting() { - t.Fatalf("the waiting catalogue walk was not kept waiting beside the controller's walk: %s %+v", kept.State, ws) - } - if _, in := ws[0].Modules["app"]; in { - t.Fatalf("the controller's walk folded the waiting catalogue walk in: %v", ws[0].Modules) - } - if _, err := letGo(ctx, open.inventory, catalogueWalk.ID, catalogue.DeliverySeat, "its turn"); err != nil { - t.Fatal(err) - } - before := len(*asked) - advanceHeld(ctx, open) - if len(*asked) != before { - t.Fatalf("the catalogue walk started beside the controller's: asked %v", (*asked)[before:]) - } - // Read again: the step above wrote the walk (what its key waits for), so the copy read at its cut is stale. - own2, err := open.inventory.PlanByID(ctx, ws[0].ID) - if err != nil { - t.Fatal(err) - } - own2.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &own2); err != nil { - t.Fatal(err) - } - advanceHeld(ctx, open) - if len(*asked) < before+1 || (*asked)[before][1] != "modules/app" { - t.Fatalf("the catalogue walk did not start once the controller's ended: %v", (*asked)[before:]) - } -} - -func TestALateCatalogueMergeIsNotAnsweredByAnOwnPathWalk(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - for _, m := range []struct { - module string - claims []catalogue.Claim - }{ - {"nats", nil}, - {"mesh-delivery", []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, - } { - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1", Claims: m.claims}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m.module, Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - } - if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - mixed := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "m1xed", MergedAt: t0.Format(time.RFC3339Nano), - Paths: []string{"modules/nats/module.json", "modules/app/module.json"}, ModuleDirs: []string{"modules/nats", "modules/app"}, - ModuleDirsSaid: true} - hear(t, open, mixed, t0.Add(time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - if len(ws) != 1 || ws[0].Waiting() { - t.Fatalf("the mixed merge's walk waited, or was not cut: %+v", ws) - } - done := ws[0] - done.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &done); err != nil { - t.Fatal(err) - } - hear(t, open, catalogueMerge("ear1ier", "app", t0.Add(-30*time.Second)), t0.Add(3*time.Minute)) - got, _ := open.inventory.PlanByID(ctx, done.ID) - _, bs := walks(t, open) - if len(got.Delivery.Merges) != 1 || len(bs) != 1 || bs[0].OwnPath() || bs[0].Delivery.Merges[0].Commit != "ear1ier" { - t.Fatalf("the late catalogue merge was answered by the own-path walk (%+v) rather than the next catalogue batch (%+v)", - got.Delivery.Merges, bs) - } -} diff --git a/cmd/mesh-controller/batches_test.go b/cmd/mesh-controller/batches_test.go deleted file mode 100644 index cbbdc11b..00000000 --- a/cmd/mesh-controller/batches_test.go +++ /dev/null @@ -1,293 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "os" - "slices" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// The suite's tests of a merge, but for the merge window's own (this file's), plan the merge as it is heard: a -// window of nothing closes at once, so a merge with no walk open is cut into its walk at once, as every merge -// was planned before novox/hq ADR 0276. The window's tests set it through the controller's settings. -func init() { - mergeWindowDefault = 0 - // No forge in a test: no manifest is found at any commit, so a merge draws no edge of the manifests' change - // (delivery_edges.go). A test that wants some replaces these (forgeHolding). - manifestAt = func(context.Context, string, string, string, string) (catalogue.Manifest, bool, error) { - return catalogue.Manifest{}, false, nil - } - parentOf = func(context.Context, string, string, string) (string, error) { return "", nil } -} - -// windowed is a mesh whose controller's settings give a merge window of 90 seconds and at most 10 minutes, as its -// settings file says them, and -// modules built from the catalogue (app, notes) and from three repositories of their own. - -// t0 is the moment a test's clock starts: merges are dated by it, and heard after it. -var t0 = time.Now().UTC().Truncate(time.Second) - -// catalogueMerge is a merge of the catalogue changing one module's directory, made at a moment, from a pull -// request's head branch of its own, as the forge announces every merge. Whether its head names a delivery group -// with a member still to merge is mesh-delivery's word, which windowed stubs as no group at all (noGroups); a test -// whose merges wait for the window as a group's says so (groupsWaiting). -func catalogueMerge(commit, module string, made time.Time) link.SourceMoved { - return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Head: "pr/" + commit, Commit: commit, - MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"modules/" + module + "/module.json"}, - ModuleDirs: []string{"modules/" + module}, ModuleDirsSaid: true} -} - -// repoMerge is a merge of a repository of one module's own, from a head branch of its own (catalogueMerge). -func repoMerge(repo, commit string, made time.Time) link.SourceMoved { - return link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Head: "pr/" + commit, Commit: commit, - MergedAt: made.Format(time.RFC3339Nano), Paths: []string{"main.go"}} -} - -// noGroups stubs mesh-delivery's word on groups for a test: no head names a group. -func noGroups(t *testing.T) { - t.Helper() - was := deliveryGroupOf - deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { return nil, false, nil } - t.Cleanup(func() { deliveryGroupOf = was }) -} - -// groupsWaiting stubs mesh-delivery's word on groups for a test of the merge window's own timing: each head named -// is one whose group mesh-delivery cannot say, so a merge from it waits for the window (ADR 0306 decision 2) and -// nothing holds the batch open; every other head names no group. A group with a member still to merge, which holds -// its batch open however long it takes (ADR 0418 clause 3), is groupStates'. -func groupsWaiting(t *testing.T, heads ...string) { - t.Helper() - was := deliveryGroupOf - deliveryGroupOf = func(_ context.Context, _ bool, head string) ([]groupMember, bool, error) { - if slices.Contains(heads, head) { - return nil, false, errors.New("the test's mesh-delivery does not say") - } - return nil, false, nil - } - t.Cleanup(func() { deliveryGroupOf = was }) -} - -// everyHeadWaits stubs mesh-delivery's word on groups for a test whose merges each wait for the window as they did -// before novox/hq ADR 0304: mesh-delivery cannot say any head's group. -func everyHeadWaits(t *testing.T) { - t.Helper() - was := deliveryGroupOf - deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { - return nil, false, errors.New("the test's mesh-delivery does not say") - } - t.Cleanup(func() { deliveryGroupOf = was }) -} - -// groupStates stubs mesh-delivery's word on one group: its members and their states, which a test changes as the -// members merge. -func groupStates(t *testing.T, head string, states map[string]string) { - t.Helper() - groupStatesFrom(t, head, states, true) -} - -// groupStatesFrom is groupStates for a mesh-delivery that does (composed) or does not yet say the group's gate is the -// composed check's: only the first holds the group's batch open for its last member. -func groupStatesFrom(t *testing.T, head string, states map[string]string, composed bool) { - t.Helper() - was := deliveryGroupOf - deliveryGroupOf = func(_ context.Context, _ bool, asked string) ([]groupMember, bool, error) { - if asked != head { - return nil, false, nil - } - var members []groupMember - for repository, state := range states { - members = append(members, groupMember{Repository: repository, State: state, Composed: composed}) - } - return members, true, nil - } - t.Cleanup(func() { deliveryGroupOf = was }) -} - -// fromHead is a merge from a head branch a test names: merges from one head are members of one delivery group, -// related by it (novox/hq ADR 0304 decision 1), so they share a batch whatever else they share. -func fromHead(m link.SourceMoved, head string) link.SourceMoved { - m.Head = head - return m -} - -// lone is a merge that names no head: known to be no delivery group's, it waits for no window unless it is -// related to a batch assembling (ADR 0304 decision 1). -func lone(m link.SourceMoved) link.SourceMoved { - m.Head = "" - return m -} - -// hear is a merge heard at a moment. - -// cutAt is the cutter looking at a moment. - -// walks is every plan record that is a walk, newest first, and the batches not yet cut. -func walks(t *testing.T, open *stores) (walks, batches []inventory.Plan) { - t.Helper() - recent, err := open.inventory.RecentPlans(t.Context(), 50) - if err != nil { - t.Fatal(err) - } - for _, p := range recent { - if p.Batch() { - batches = append(batches, p) - } else { - walks = append(walks, p) - } - } - return walks, batches -} - -// The replay of issue 362: two catalogue merges 18 seconds apart are one batch, cut once into one walk at the -// later commit, which names both merges — the earlier carried by the later. - -// Merges of three repositories in one window are one walk, one commit for each. - -// Each merge restarts the window: a merge at second 80 keeps the batch open until second 170; the window -// closes at most ten minutes after its first merge however busy. - -// `plans` lists the batch being assembled first, in the operator's words: how long is left, when at the latest, -// what is grouped, and that the plan is not yet calculated. It says so on the bus as plan-moved, too. - -// captured is what a command printed. - -// A merge heard during a walk it is related to (the same repository, issue 973) joins the next batch, which is -// cut when the walk ends; the walk that started is never superseded (ADR 0276 decision 4, ADR 0407 decision 4). - -// A walk waiting for its delivery's word is folded into the next batch's walk, which answers its merges; it -// names the walk that took it over. - -// A merge heard after a later merge of its branch was walked is answered by that walk when it builds all it -// moves — named at once on a walk done — and joins the next batch when it does not. - -// A failed walk marks nothing delivered: its earlier merges are walked alone, on their own commit, the newest -// first; the first delivered answers the older ones, and the search stops. - -// A delivery group's order holds inside the walk (ADR 0249 unchanged): merges of two repositories from one head -// branch name, the node-engine's before the controller's, are tiered so; without the group, one tier. - -// A restarted controller resumes the window where it stood: the batch, its merges and their times are in the -// store, a merge handed over again is not doubled, and the batch is cut when its window closes. - -// The window's settings reach the controller in its settings file, read at every look; one that says no -// duration, or no file, is the default. -func TestTheWindowIsTheControllersSetting(t *testing.T) { - file := t.TempDir() + "/merge-window.json" - t.Setenv(mergeWindowFileVar, file) - for _, c := range []struct { - content string - window, atMost time.Duration - }{ - {`{"merge-window": "60s", "merge-window-at-most": "5m"}`, time.Minute, 5 * time.Minute}, - {`{"merge-window": 30, "merge-window-at-most": ""}`, 30 * time.Second, mergeWindowAtMostDefault}, - {`{"merge-window": "soon"}`, mergeWindowDefault, mergeWindowAtMostDefault}, - } { - if err := os.WriteFile(file, []byte(c.content), 0o600); err != nil { - t.Fatal(err) - } - if w, m := mergeWindowOf(); w != c.window || m != c.atMost { - t.Errorf("%s reads as %s and %s, want %s and %s", c.content, w, m, c.window, c.atMost) - } - } - t.Setenv(mergeWindowFileVar, file+".gone") - if w, m := mergeWindowOf(); w != mergeWindowDefault || m != mergeWindowAtMostDefault { - t.Errorf("no file reads as %s and %s", w, m) - } -} - -// S16 names every merge a waiting walk answers, never one commit a supersession may have ended (issue 362). -func TestAWaitingWalkNamesTheMergesItAnswers(t *testing.T) { - f := calm(t0) - f.waits = []waitFacts{{id: "plan-2", repository: "novox/mesh-catalog", commit: "48bda475dunst", awaits: "mesh-delivery", - since: t0.Add(-31 * time.Minute), merges: []inventory.PlanMerge{ - {Repository: "novox/mesh-catalog", Commit: "553b7191claude", Carried: "48bda475dunst"}, - {Repository: "novox/mesh-catalog", Commit: "48bda475dunst"}}}} - got := watchWaits(f) - if len(got) != 1 || !strings.Contains(got[0].Summary, "novox/mesh-catalog@553b7191") || - !strings.Contains(got[0].Summary, "novox/mesh-catalog@48bda475") { - t.Fatalf("the waiting walk does not name both merges: %+v", got) - } -} - -// A merge heard after a later merge of its branch was cut is named by that walk however its modules read since -// the cut, in a repository of one module (review of this change); a walk that never built what it moves names -// nothing, and the merge joins the next batch. - -// A related walk waits for the delivery's word too: a waiting walk let go beside a started one it is related to -// (the same repository, issue 973) takes the word and starts once that one ended, asking nothing before. - -// A file a merge of the batch removed and a later one brought back is not removed; one removed last is. -func TestARemovedFileIsTheLastMergesWord(t *testing.T) { - ev := func(commit string, paths, removed []string) inventory.BatchedMerge { - m := link.SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: commit, Paths: paths, Removed: removed} - b, _ := json.Marshal(m) - return inventory.BatchedMerge{Repository: "novox/one", Branch: "main", Commit: commit, Event: b, - Merged: t0.Add(time.Duration(len(commit)) * time.Second)} - } - got := combinedMerges([]inventory.BatchedMerge{ - ev("a", []string{"x/module.json", "y/module.json"}, []string{"x/module.json", "y/module.json"}), - ev("bb", []string{"x/module.json"}, nil), - ev("ccc", []string{"z.go"}, nil)}) - if len(got) != 1 || got[0].Commit != "ccc" || !slices.Equal(got[0].Removed, []string{"y/module.json"}) || - len(got[0].Paths) != 3 { - t.Fatalf("combined as %+v", got) - } -} - -// A merge on the controller's own path never shares a batch with one that waits for mesh-delivery's word -// (decided during the build, 2026-10-10): the two are batches of their own, cut one after the other; a walk on -// the own path folds no waiting catalogue walk but batches its merges again behind it; the catalogue's walk -// still waits for the word, so no catalogue delivery skips its turn. - -// `plans` names a walk by what it moves (asked by the operator, 2026-10-10): the repository's pull request and -// title, the modules it moves and the machines running them, then the state words; a record naming no pull -// request is named by its commit, and a title is cut at fifty runes. -func TestAPlanLineNamesWhatItMoves(t *testing.T) { - now := time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC) - p := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1", - State: inventory.PlanBuilding, Tiers: [][]string{{"messenger", "telegram"}}, TierEntered: now.Add(-2 * time.Minute), - Modules: map[string]*inventory.PlanModule{"messenger": {State: "asked"}, "telegram": {State: "asked"}}, - Commits: []inventory.PlanCommit{{Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1c1c1c1c1"}}, - Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-catalog", - Commit: "c1c1c1c1c1", Number: 175, Title: "a tap shows its outcome", Moves: []string{"telegram", "messenger"}}}}} - running := func(module string) []string { - if module == "messenger" || module == "telegram" { - return []string{"novox"} - } - return nil - } - if got, want := planLineOn(p, now, tierAtLeast, running), - "mesh-catalog #175 a tap shows its outcome · messenger, telegram → novox · tier 1 of 1, building for 2m0s"; got != want { - t.Fatalf("the line reads %q, want %q", got, want) - } - if got := planLine(p, now); !strings.HasPrefix(got, "mesh-catalog #175 a tap shows its outcome · messenger, telegram · tier") { - t.Fatalf("without the machines the line reads %q", got) - } - old := p - old.Commits, old.Delivery = nil, nil - if got := planLine(old, now); !strings.HasPrefix(got, "mesh-catalog c1c1c1c1 · tier 1 of 1") { - t.Fatalf("a record naming no pull request reads %q", got) - } - long := p - long.Delivery.Merges[0].Title = strings.Repeat("abcdefghij", 6) - if got := planHeadline(long, nil); !strings.Contains(got, "#175 "+strings.Repeat("abcdefghij", 4)+"abcdefghi…") { - t.Fatalf("a long title is not cut at fifty runes: %q", got) - } -} - -// sameMerge compares what a walk names of a merge: its repository, commit and the commit carrying it. -func sameMerge(a, b inventory.PlanMerge) bool { - return a.Repository == b.Repository && a.Commit == b.Commit && a.Carried == b.Carried -} - -// A catalogue merge heard after a later merge of its branch was walked without the word (a merge that touched -// the bus too, on the controller's own path) is not answered by that walk while the delivery seat has a holder: -// its delivery would skip its turn. It joins the next catalogue batch. diff --git a/cmd/mesh-controller/batches_watch.go b/cmd/mesh-controller/batches_watch.go deleted file mode 100644 index ae933763..00000000 --- a/cmd/mesh-controller/batches_watch.go +++ /dev/null @@ -1,153 +0,0 @@ -package main - -import ( - "context" - "fmt" - "sort" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// **Every wait of a batch has a bound and a condition** (novox/hq ADR 0276 decision 9): a batch still -// assembling a minute past its maximum while no walk is open is a cut the controller failed to make (S18), -// and a batch waiting behind an open walk longer than that walk's bound waits on a walk gone wrong (S19). - -// The kinds S18 and S19 raise. -const ( - kindBatchNotCut = "batch-not-cut" - kindBatchBehindWalk = "batch-behind-walk" - // kindGroupNotMerged is raised for a group whose last member has not merged by the merge window's maximum after - // its first (novox/hq ADR 0418 clause 3). - kindGroupNotMerged = "group-not-merged" -) - -// batchFacts is one batch not yet cut, as the watchdogs read it. -type batchFacts struct { - id, state, grouped string - // atMost is when its window closes at the latest; closed when it closed. - atMost, closed time.Time - // behind is the open walk it waits behind, and walkBound that walk's bound: a tier's bound for each of its - // tiers. - behind string - walkBound time.Duration - // holds are the groups the batch is held for, by head, with why; created is when the batch was made, the first - // merge of the group heard; atMostWindow is the maximum a window may take (the controller's setting). - holds map[string]string - created time.Time - atMostWindow time.Duration -} - -// gatherBatches is every batch not yet cut, with the bound of the walk it waits behind. -func gatherBatches(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]batchFacts, error) { - batches, err := inv.Batches(ctx) - if err != nil || len(batches) == 0 { - return nil, err - } - bounds, err := measuredTierBounds(ctx, inv, now) - if err != nil { - return nil, err - } - var out []batchFacts - for _, b := range batches { - _, atMostWindow := mergeWindowOf() - f := batchFacts{id: b.ID, state: b.State, grouped: groupedWords(b), created: b.Created, atMostWindow: atMostWindow} - if w := b.Delivery; w != nil && w.Batch != nil { - f.holds = w.Batch.Holds - f.atMost, f.closed, f.behind = w.Batch.AtMost, w.Batch.ClosesAt, w.Batch.Behind - if f.atMost.Before(f.closed) { - f.closed = f.atMost - } - if w.Batch.CutAtOnce && !b.Created.IsZero() { - // Its window closed when it was made, which is its latest too (ADR 0304 decision 1). - f.atMost, f.closed = b.Created, b.Created - } - } - if f.behind != "" { - if walk, err := inv.PlanByID(ctx, f.behind); err == nil { - f.walkBound = bounds.of(walk.Repository) * time.Duration(max(1, len(walk.Tiers))) - } - } - out = append(out, f) - } - return out, nil -} - -// watchBatchesNotCut is S18: a batch still assembling a minute past its maximum, while no walk is open. -func watchBatchesNotCut(f *signalFacts) []conditions.Observation { - var out []conditions.Observation - for _, b := range f.batches { - if b.state != inventory.PlanAssembling || b.atMost.IsZero() || f.now.Sub(b.atMost) <= batchLateAfter || - len(b.holds) > 0 { // a batch held for a group waits on a member, which has its own condition - continue - } - late := f.now.Sub(b.atMost) - out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchNotCut, - Severity: conditions.Warning, - Summary: fmt.Sprintf("the batch %s is still assembling %s past its latest close (%s), with no walk open: "+ - "the controller did not cut it; grouped: %s", b.id, ago(late), b.atMost.UTC().Format(time.RFC3339), b.grouped), - Said: fmt.Sprintf("assembling since %s past its maximum", ago(late)), - Headline: "Merged changes are not being delivered", - Explanation: fmt.Sprintf("Merges collected for one delivery should have been planned %s ago and were not. "+ - "Nothing is lost; the mesh keeps them until it plans them.", humanDuration(late)), - Resolved: "Merged changes are being delivered again"}) - } - return out -} - -// watchBatchesBehind is S19: a batch waiting behind an open walk longer than that walk's bound, naming it. -func watchBatchesBehind(f *signalFacts) []conditions.Observation { - var out []conditions.Observation - for _, b := range f.batches { - if b.state != inventory.PlanQueued || b.behind == "" || b.walkBound <= 0 || f.now.Sub(b.closed) <= b.walkBound { - continue - } - in := f.now.Sub(b.closed) - out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, Kind: kindBatchBehindWalk, - Severity: conditions.Warning, - Summary: fmt.Sprintf("the batch %s has waited %s behind the walk %s, longer than that walk's bound (%s); "+ - "grouped: %s; `plans %s` says where that walk stands", b.id, ago(in), b.behind, ago(b.walkBound), - b.grouped, b.behind), - Said: fmt.Sprintf("queued behind %s for %s", b.behind, ago(in)), - Headline: "Merged changes wait behind a slow delivery", - Explanation: fmt.Sprintf("Merges collected for the next delivery have waited %s for the delivery before "+ - "them, which is taking longer than it should. Nothing is lost.", humanDuration(in)), - Resolved: "Merged changes no longer wait behind a slow delivery"}) - } - return out -} - -// watchGroupsNotMerged is the condition of a batch held for a group whose last member has not merged by the merge -// window's maximum after the first member merged, naming the member and why (novox/hq ADR 0418 clause 3): nothing of -// the group is delivered while it waits. -func watchGroupsNotMerged(f *signalFacts) []conditions.Observation { - var out []conditions.Observation - for _, b := range f.batches { - if len(b.holds) == 0 || b.created.IsZero() || f.now.Sub(b.created) <= b.atMostWindow { - continue - } - heads := make([]string, 0, len(b.holds)) - for h := range b.holds { - heads = append(heads, h) - } - sort.Strings(heads) - var why []string - for _, h := range heads { - why = append(why, b.holds[h]) - } - waited := f.now.Sub(b.created) - out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, - Kind: kindGroupNotMerged, Severity: conditions.Warning, - Summary: fmt.Sprintf("%s; its first member merged %s ago, past the merge window's maximum (%s), and nothing "+ - "of the group is delivered while it waits (batch %s)", strings.Join(why, "; "), ago(waited), ago(b.atMostWindow), b.id), - Said: "a group waits for a member that has not merged", - Headline: "A change waits for the rest of its group to merge", - Explanation: fmt.Sprintf("The changes of a group are delivered together, once the last one is merged. "+ - "One has not merged in %s. Nothing is lost; the merged ones are kept and delivered with it.", - humanDuration(waited)), - Resolved: "The group's last change merged"}) - } - return out -} diff --git a/cmd/mesh-controller/beside_test.go b/cmd/mesh-controller/beside_test.go deleted file mode 100644 index d3626c82..00000000 --- a/cmd/mesh-controller/beside_test.go +++ /dev/null @@ -1,117 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// An unrelated closed batch is cut into a walk beside an open one, and that walk asks its first tier at once -// (issue 973, novox/hq ADR 0407 decision 4): a hotfix is never held by an unrelated delivery. -func TestAnUnrelatedBatchWalksBesideAnOpenWalk(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - // Both modules' keys are not built, so each walk stays open as it did while its build ran: with the stand-in - // builder's key done at once, one step would carry a one-module walk to done. - wholeKeys, held := openPlanKeys, &atomic.Bool{} - held.Store(true) - openPlanKeys = func() (planKeyReads, func(), error) { - all, done, err := wholeKeys() - return keysHolding{keysHolding{all, "one", held, ""}, "two", held, ""}, done, err - } - ctx := t.Context() - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(90*time.Second)) - if ws, _ := walks(t, open); len(ws) != 1 || !ws[0].Open() { - t.Fatalf("the first walk is not open: %+v", ws) - } - heard := t0.Add(100 * time.Second) - hear(t, open, lone(repoMerge("two", "c2", heard)), heard) - advanceHeld(ctx, open) - ws, bs := walks(t, open) - if len(ws) != 2 || len(bs) != 0 { - t.Fatalf("%d walk(s) and %d batch(es); want the unrelated batch cut beside the open walk", len(ws), len(bs)) - } - for _, w := range ws { - if !w.Open() || w.Waiting() || deferred(w) || !askedAny(w) { - t.Errorf("%s (%s) is not walking: open %v, deferred %v, asked %v, note %q", w.ID, w.Named(), w.Open(), - deferred(w), askedAny(w), w.Note) - } - } -} - -// A closed batch related to an open walk is queued behind it with the reason, while one that shares nothing is -// cut beside it; the walk that started is never superseded (ADR 0276 decision 4, issue 973). -func TestARelatedBatchWaitsBehindTheOpenWalk(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - ctx := t.Context() - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(90*time.Second)) - first := func() inventory.Plan { ws, _ := walks(t, open); return ws[len(ws)-1] }() - heard := t0.Add(100 * time.Second) - hear(t, open, lone(repoMerge("one", "c2", heard)), heard) - hear(t, open, lone(repoMerge("three", "c3", heard.Add(time.Second))), heard.Add(time.Second)) - ws, bs := walks(t, open) - if len(ws) != 2 || len(bs) != 1 { - t.Fatalf("%d walk(s) and %d batch(es); want the unrelated merge walking and the related one queued", len(ws), len(bs)) - } - b := bs[0] - words := batchWords(b, heard) - if b.State != inventory.PlanQueued || b.Delivery.Batch.Behind != first.ID || b.CommitOf("novox/one") != "c2" || - !strings.Contains(words, "queued behind "+first.ID) || !strings.Contains(words, "both change novox/one") { - t.Fatalf("the related batch is %s, behind %q, reading %q", b.State, b.Delivery.Batch.Behind, words) - } - if got, _ := open.inventory.PlanByID(ctx, first.ID); !got.Open() { - t.Fatalf("the started walk was %s by the related merge", got.State) - } - first.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &first); err != nil { - t.Fatal(err) - } - cutAt(t, open, heard.Add(2*time.Second)) - if _, bs := walks(t, open); len(bs) != 0 { - t.Fatalf("the queued batch was not cut once the walk it waited for ended: %+v", bs) - } -} - -// A batch on the controller's own path never walks beside another walk, and nothing walks beside it, whatever -// the two share (issue 973): it replaces the controller, so it goes alone. -func TestAnOwnPathBatchNeverWalksBeside(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - ctx := t.Context() - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(90*time.Second)) - ws, _ := walks(t, open) - first := ws[0] - heard := t0.Add(100 * time.Second) - hear(t, open, lone(repoMerge("mesh-controller", "k1", heard)), heard) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 1 || !bs[0].OwnPath() || bs[0].State != inventory.PlanQueued || - bs[0].Delivery.Batch.Behind != first.ID || !strings.Contains(batchWords(bs[0], heard), "own path") { - t.Fatalf("an own-path batch beside an open walk: %d walk(s), batches %+v", len(ws), bs) - } - first.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &first); err != nil { - t.Fatal(err) - } - cutAt(t, open, heard.Add(time.Second)) - ws, _ = walks(t, open) - if len(ws) != 2 || !ws[0].Open() || ws[0].CommitOf("novox/mesh-controller") != "k1" { - t.Fatalf("the own-path batch was not cut once the walk ended: %+v", ws) - } - own := ws[0] - // Unrelated to the own-path walk by every footprint, a merge still waits for it. - later := heard.Add(time.Minute) - hear(t, open, lone(repoMerge("two", "c2", later)), later) - ws, bs = walks(t, open) - if len(ws) != 2 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != own.ID { - t.Fatalf("a merge beside the own-path walk: %d walk(s), batches %+v", len(ws), bs) - } -} diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index c6318a61..77dc7156 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -229,10 +229,6 @@ type answers struct { // pair that answers "has it caught up", which waiting alone cannot (the sent digest is // recorded at send, not at apply). reported []inventory.Reported - // plans is what the last merges produced and where each stands (novox/hq ADR 0162). - plans []inventory.Plan - // tierBounds is how long each repository's plan tier may take before it is late (novox/hq issue 296). - tierBounds tierBounds // refused is why a machine cannot be worked out at all, by name. A different thing from every // other answer here: those are about a machine that was told something, and this is about one // that cannot be told anything — it never reaches waiting, because nothing was computed for it diff --git a/cmd/mesh-controller/bus_step.go b/cmd/mesh-controller/bus_step.go index e52f97ce..f7e6e6f5 100644 --- a/cmd/mesh-controller/bus_step.go +++ b/cmd/mesh-controller/bus_step.go @@ -229,7 +229,7 @@ func busCommand(ctx context.Context, args []string) error { fmt.Printf("bus upgrade %d: %s %s → %s on %s; streams snapshotted at %s; %s\n", step.ID, b.module, step.From, short(b.to), strings.Join(moving, ", "), where, map[bool]string{true: "reversible: putting the old build back undoes it", false: "NOT reversible: the snapshot is the only way back"}[*reversible]) - sent, err := sendRollout(withBusStep(withScope(ctx, sendScope{person: true})), open, moving) + sent, err := sendToEach(withBusStep(ctx), open, moving) if err != nil { _ = inv.EndBusStep(ctx, step.ID, "failed", "the send was refused: "+err.Error()) return fmt.Errorf("the bus's machine could not be sent its new build: %w — nothing was replaced", err) diff --git a/cmd/mesh-controller/bus_waits.go b/cmd/mesh-controller/bus_waits.go deleted file mode 100644 index 0c02e870..00000000 --- a/cmd/mesh-controller/bus_waits.go +++ /dev/null @@ -1,195 +0,0 @@ -package main - -import ( - "fmt" - "sort" - "strings" - "sync" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A send held for the bus's planned step (novox/hq issue 336). -// -// **A wait only a person can end is said to that person at once.** No send replaces the bus but its planned -// step (sendToEach, ADR 0236), and the step is a person's `bus upgrade`. So while a new bus build waits, every -// walk whose tier sends to the bus's machine is refused, and tries again each tick. The refusal was kept only -// as the walk's note; nothing was raised, and the operator found the hold by asking why a walk did not move. -// Row S17 raises it on the first tick after the first refusal, in the bus's scope, for the operator: what -// waits, behind which bus build, since when, and the verb. A walk held only by it is not late, so S3 leaves it -// out, as it leaves out a walk under a paused build seat. It clears once the bus's machine has been sent the -// build the mesh holds — the step was taken, and no send is refused for the bus any more — whatever the walks' -// notes still say. Whether the new bus came up healthy is the step's own condition (probe DB). - -// kindBusStepWaiting is S17's kind: bus..step-waiting. -const kindBusStepWaiting = "bus-step-waiting" - -// busFacts is a new bus build waiting for its planned step, and the sends held for it. -type busFacts struct { - module, to string - // from is the build each machine of the bus runs; machines those whose bus the step would replace. - from map[string]string - machines []string - waits []busWaitFacts -} - -// busWaitFacts is one walk whose send was refused because it would replace the bus. -type busWaitFacts struct { - plan, repository, commit string - // modules are what its tier sends: what waits. - modules []string - // since is the first refusal: as this controller saw it, or, read back, the save that kept the refusal. - since time.Time -} - -// busRefusedFirst is when this controller first saw each walk refused for the bus's step. The walk's note -// keeps the refusal across a restart; this keeps its moment more exactly than the walk's last save. -var busRefusedFirst = &firstSeen{at: map[string]time.Time{}} - -type firstSeen struct { - mu sync.Mutex - at map[string]time.Time -} - -// mark keeps the first moment an id was seen. -func (s *firstSeen) mark(id string, at time.Time) { - s.mu.Lock() - defer s.mu.Unlock() - if _, seen := s.at[id]; !seen { - s.at[id] = at - } -} - -// of is when an id was first seen; zero when it was not. -func (s *firstSeen) of(id string) time.Time { - s.mu.Lock() - defer s.mu.Unlock() - return s.at[id] -} - -// keepOnly forgets every id not given: a walk no longer held is not held since then. -func (s *firstSeen) keepOnly(ids map[string]bool) { - s.mu.Lock() - defer s.mu.Unlock() - for id := range s.at { - if !ids[id] { - delete(s.at, id) - } - } -} - -// refusedForTheBus is whether an error is the refusal of a send for the bus's planned step. -func refusedForTheBus(err error) bool { - return err != nil && strings.Contains(err.Error(), errBusWaits.Error()) -} - -// busWaitsOf is the sends held for the bus's step, from the open walks: each walk not waiting for its -// delivery's word whose note keeps a refusal for this very bus build, while the build would still replace -// the bus on a machine. first is when this controller first saw a walk refused, zero when it did not. -func busWaitsOf(plans []inventory.Plan, b busPending, first func(string) time.Time) busFacts { - f := busFacts{module: b.module, to: b.to, from: b.from} - for _, n := range b.machines { - if b.moves(n) { - f.machines = append(f.machines, n) - } - } - if len(f.machines) == 0 { - return f - } - for _, p := range plans { - if !p.Open() || p.Waiting() || !strings.Contains(p.Note, errBusWaits.Error()) || !strings.Contains(p.Note, short(b.to)) { - continue - } - since := p.Updated - if seen := first(p.ID); !seen.IsZero() && (since.IsZero() || seen.Before(since)) { - since = seen - } - var modules []string - if p.Tier >= 0 && p.Tier < len(p.Tiers) { - modules = append(modules, p.Tiers[p.Tier]...) - } else { - modules = planModules(p) - } - sort.Strings(modules) - f.waits = append(f.waits, busWaitFacts{plan: p.ID, repository: p.Repository, commit: p.Commit, modules: modules, - since: since}) - } - sort.Slice(f.waits, func(i, j int) bool { return f.waits[i].since.Before(f.waits[j].since) }) - return f -} - -// heldByTheBus is the walks of a bus's facts, by id: what S3 leaves out. -func (b busFacts) heldByTheBus() map[string]bool { - out := map[string]bool{} - for _, w := range b.waits { - out[w.plan] = true - } - return out -} - -// busUpgradeVerb is the call that ends the wait, as the summary names it: through the mesh MCP server, with -// why, saying whether the new version can be undone ("reversible": "true") or not ("irreversible": "true"). -const busUpgradeVerb = "`mesh_call mesh-controller.bus {\"upgrade\": \"true\", \"why\": \"…\", \"reversible\" or \"irreversible\": \"true\"}`" - -// watchBusWaits is S17: a send held for the bus's planned step, said at once to the operator. -func watchBusWaits(f *signalFacts) []conditions.Observation { - b := f.bus - if len(b.waits) == 0 || len(b.machines) == 0 { - return nil - } - since := b.waits[0].since - var walks, what []string - for _, w := range b.waits { - walks = append(walks, fmt.Sprintf("%s (%s %s, tier: %s)", w.plan, w.repository, short(w.commit), - strings.Join(w.modules, ", "))) - what = append(what, deliveryWhat(w.modules, w.repository)) - } - var from []string - for _, n := range b.machines { - from = append(from, short(orNotKnown(b.from[n]))) - } - machines := namesWords(b.machines, 3) - in := f.now.Sub(since) - return []conditions.Observation{{Scope: conditions.ScopeBus, ID: b.module, Token: "step-waiting", - Kind: kindBusStepWaiting, Severity: conditions.Warning, Resolver: conditions.ResolverOperator, - Machine: b.machines[0], Also: b.machines[1:], - Summary: fmt.Sprintf("sends to %s wait for the bus's planned step: a new bus build (%s %s → %s) would replace "+ - "the bus there, which only a person's %s does; waiting since %s: %s", strings.Join(b.machines, ", "), - b.module, strings.Join(sortedUnique(from), ", "), short(b.to), busUpgradeVerb, - since.UTC().Format(time.RFC3339), strings.Join(walks, "; ")), - Said: fmt.Sprintf("%d walk(s) refused since %s", len(b.waits), since.UTC().Format(time.RFC3339)), - Headline: clipWords("Sends to "+machines+" wait for a bus upgrade", conditions.HeadlineMax), - Explanation: clipWords(fmt.Sprintf("A new version of the mesh's message system is built, and only a person "+ - "installs it. Until then nothing else is sent to %s: %s waits, for %s so far.", machines, - namesWords(sortedUnique(what), 3), humanDuration(in)), conditions.ExplanationMax), - Needs: "start the bus upgrade " + FromMeshMCPServer, - Resolved: "Resolved: the bus upgrade started, and sends go on"}} -} - -// sortedUnique is a list sorted, each once. -func sortedUnique(xs []string) []string { - seen := map[string]bool{} - var out []string - for _, x := range xs { - if !seen[x] { - seen[x] = true - out = append(out, x) - } - } - sort.Strings(out) - return out -} - -// clipWords keeps a plain sentence within a bound, at a word. -func clipWords(s string, n int) string { - if len(s) <= n { - return s - } - cut := strings.LastIndex(s[:n-1], " ") - if cut <= 0 { - cut = n - 1 - } - return s[:cut] + "…" -} diff --git a/cmd/mesh-controller/bus_waits_integration_test.go b/cmd/mesh-controller/bus_waits_integration_test.go deleted file mode 100644 index 6fec986c..00000000 --- a/cmd/mesh-controller/bus_waits_integration_test.go +++ /dev/null @@ -1,70 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestASendHeldForTheBusStepIsFoundFromTheWalksItHolds(t *testing.T) { - now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC) - b := aBusOnAnchor() - walk := func(id, repo, note string, updated time.Time) inventory.Plan { - return inventory.Plan{ID: id, Repository: repo, Commit: "c0ffee001122", State: inventory.PlanRolling, Tier: 0, - Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{"mesh-host": {}}, Note: note, - Updated: updated} - } - for _, c := range []struct { - name string - bus busPending - plans []inventory.Plan - first map[string]time.Time - want []string - since time.Time - }{ - {"a walk refused for the bus", b, - []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-27*time.Minute))}, nil, - []string{"plan-1"}, now.Add(-27 * time.Minute)}, - {"refused earlier than its last save, as this controller saw it", b, - []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-5*time.Minute))}, - map[string]time.Time{"plan-1": now.Add(-28 * time.Minute)}, []string{"plan-1"}, now.Add(-28 * time.Minute)}, - {"a walk refused for another reason", b, - []inventory.Plan{walk("plan-1", "novox/mesh-host", "tier 0: the build seat is paused — tried again", now)}, nil, - nil, time.Time{}}, - {"refused for an older bus build than the one held now", func() busPending { o := b; o.to = "99999999cccc"; return o }(), - []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}}, - {"the bus already runs the build held: its step started", func() busPending { - o := aBusOnAnchor() - o.from = map[string]string{"anchor": o.to} - return o - }(), []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}}, - {"a walk waiting for its delivery's word asks no send", b, func() []inventory.Plan { - p := walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now) - p.Delivery = &inventory.PlanDelivery{Awaits: "mesh-delivery"} - return []inventory.Plan{p} - }(), nil, nil, time.Time{}}, - } { - t.Run(c.name, func(t *testing.T) { - got := busWaitsOf(c.plans, c.bus, func(id string) time.Time { return c.first[id] }) - var ids []string - for _, w := range got.waits { - ids = append(ids, w.plan) - } - if strings.Join(ids, ",") != strings.Join(c.want, ",") { - t.Fatalf("held for the bus: %v, want %v", ids, c.want) - } - if len(c.want) > 0 { - if !got.waits[0].since.Equal(c.since) { - t.Errorf("waiting since %s, want %s", got.waits[0].since, c.since) - } - if strings.Join(got.machines, ",") != "anchor" || got.to != b.to || got.module != "nats" { - t.Errorf("behind %+v", got) - } - } - }) - } -} diff --git a/cmd/mesh-controller/bus_waits_test.go b/cmd/mesh-controller/bus_waits_test.go deleted file mode 100644 index 668d7cff..00000000 --- a/cmd/mesh-controller/bus_waits_test.go +++ /dev/null @@ -1,99 +0,0 @@ -package main - -import ( - "context" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" -) - -// novox/hq issue 336: a send held because it would replace the bus outside its planned step waits for a -// person, so a person is told at once — what waits, behind which bus build, since when, and the verb that -// ends it — and the wait is not read as a walk running late. - -// aBusOnAnchor is a new bus build waiting for its step on anchor: nats 88135ad0 there, 32307bd1 held. -func aBusOnAnchor() busPending { - return busPending{module: "nats", machines: []string{"anchor"}, from: map[string]string{"anchor": "88135ad0aaaa"}, - to: "32307bd1bbbb", same: map[string]bool{}} -} - -// refusedNote is the note a walk keeps when its send was refused for the bus, as advanceHeld writes it. -func refusedNote(b busPending, tier int) string { - held := "sending anchor would replace the bus (nats " + short(b.from["anchor"]) + " → " + short(b.to) + - "), which is a planned step: `bus upgrade --why …` snapshots its streams first and checks them after (novox/hq ADR 0236)" - return "tier " + string(rune('0'+tier)) + ": " + errBusWaits.Error() + ": " + held + " — tried again" -} - -// **Said at once, not as lateness, and cleared when the step starts**: the walk held only for the bus raises -// the bus's condition on the first tick, before any tier bound, naming what waits, the bus build from and to, -// since when and `bus upgrade`, for the operator; S3 says nothing of it even past its bound; and the -// condition clears once the bus's machine runs the new build. -func TestASendHeldForTheBusStepIsSaidAtOnceAndNotAsLateness(t *testing.T) { - now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC) - b := aBusOnAnchor() - held := func(entered time.Duration) *signalFacts { - f := calm(now) - f.plans = []planFacts{{id: "plan-1", repository: "novox/mesh-host", commit: "c0ffee00", tier: 0, tiers: 1, - entered: now.Add(-entered), bound: 30 * time.Minute, waiting: refusedNote(b, 0), bus: true}} - f.bus = busFacts{module: "nats", to: b.to, from: b.from, machines: []string{"anchor"}, - waits: []busWaitFacts{{plan: "plan-1", repository: "novox/mesh-host", commit: "c0ffee001122", - modules: []string{"mesh-host"}, since: now.Add(-time.Minute)}}} - return f - } - - f := held(time.Minute) - got := watchBusWaits(f) - if len(got) != 1 { - t.Fatalf("a send held for the bus a minute ago raised %+v", got) - } - o := got[0] - if o.Key() != "bus.nats.step-waiting" || o.Kind != kindBusStepWaiting || o.Resolver != conditions.ResolverOperator { - t.Fatalf("raised %s (%s), resolver %q", o.Key(), o.Kind, o.Resolver) - } - for _, want := range []string{"anchor", "mesh-host", "88135ad0", "32307bd1", "mesh_call mesh-controller.bus", - `"upgrade": "true"`, `"reversible"`, `"irreversible"`, "2026-10-08T18:29:00Z", "plan-1"} { - if !strings.Contains(o.Summary, want) { - t.Errorf("its summary does not say %q: %s", want, o.Summary) - } - } - if !strings.Contains(o.Explanation, "mesh-host") || !strings.Contains(o.Headline, "bus upgrade") || o.Needs == "" { - t.Errorf("its words do not say what waits and what the operator does: %+v", o) - } - if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation, - Resolved: o.Resolved, Needs: o.Needs}, "anchor"); !ok { - t.Errorf("its words are not plain: %s", why) - } - - // Past S3's bound: still the bus's wait, never a stalled walk. - late := held(45 * time.Minute) - if s3 := watchPlans(late); len(s3) != 0 { - t.Fatalf("a walk held only by the bus step was said stalled: %+v", s3) - } - // A walk held for something else past its bound is still stalled. - other := held(45 * time.Minute) - other.plans[0].bus = false - if s3 := watchPlans(other); len(s3) != 1 { - t.Fatalf("a walk held for something else past its bound raised %+v", s3) - } - - // Through the keeper: open at the first tick, cleared when the step started. - store := conditions.NewInMemory() - k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, Teller: &conditions.Told{}, - Now: func() time.Time { return now }}) - defer k.Close(context.Background()) - w := &watchdogs{keeper: k, started: now.Add(-time.Hour)} - w.see(t.Context(), held(time.Minute)) - open, err := k.Open(t.Context()) - if err != nil || len(open) != 1 || open[0].Key != "bus.nats.step-waiting" { - t.Fatalf("after the first tick, open: %+v (%v)", open, err) - } - started := held(time.Minute) - started.bus = busFacts{module: "nats", to: b.to} - started.plans[0].bus = false - w.see(t.Context(), started) - if open, _ := k.Open(t.Context()); len(open) != 0 { - t.Fatalf("the step started, and open: %+v", open) - } -} diff --git a/cmd/mesh-controller/busy.go b/cmd/mesh-controller/busy.go deleted file mode 100644 index 960a1bab..00000000 --- a/cmd/mesh-controller/busy.go +++ /dev/null @@ -1,103 +0,0 @@ -package main - -import ( - "fmt" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// A first-node gate waits for a busy container (novox/hq ADR 0335 §4). -// -// When a machine reports, on what the gate sent it, that a container of a judged module was left running -// because it said busy, the judging has not started: the build is not yet what runs there. The walk says it -// waits for a busy container on that machine, counts no pass and no bound, and starts the judging — its -// Since — at the report that no longer lists the container waiting. The wait is the node-engine's, bounded -// at 15 minutes; the controller's own bound (busyGateBound past the engine's) is for an engine that stopped -// reporting while it waited, so no walk waits for ever. An engine older than the field never lists one, and -// its gate starts as before. - -// busyGateSlack is how long past the engine's bound the controller still waits for the report that says the -// container replaced: a pass of the engine (its asks every 20 seconds) and the report's way here. -var busyGateSlack = 5 * time.Minute - -// busyEngineBound is the node-engine's bound on a wait (novox/hq ADR 0335 §3), counted again here on this -// controller's clock from when it first read the wait. -const busyEngineBound = 15 * time.Minute - -// busyGateEnds is when the gate stops waiting: the engine's own word on its bound, or this controller's count of -// it from when it first read the wait, whichever is sooner, and the slack — so neither a clock ahead on the -// machine nor a wrong word from it holds a walk longer. -func busyGateEnds(b *inventory.GateBusy) time.Time { - until := b.Seen.Add(busyEngineBound) - if !b.Until.IsZero() && b.Until.Before(until) { - until = b.Until - } - return until.Add(busyGateSlack) -} - -// busyWait decides, before a judging, whether the gate waits for a busy container on a machine it judges: -// true while it waits, the gate saying so. False when nothing waits, or no longer: a wait that ended moves the -// gate's Since to its end. Pure but for the gate it is handed. -func busyWait(g *inventory.PlanGate, pairs []judged, f gateFacts, now time.Time) bool { - if g.Busy != nil && g.Busy.Ended != nil { - return false - } - for _, j := range pairs { - b, waits := busyOn(j, f) - if !waits { - continue - } - if g.Busy == nil { - g.Busy = &inventory.GateBusy{Machine: j.node, Module: j.module, Container: b.Container, Since: b.Since, - Seen: now} - } - if g.Busy.Seen.IsZero() { - g.Busy.Seen = now - } - g.Busy.Until, g.Busy.Said = b.Until, b.Said - if now.After(busyGateEnds(g.Busy)) { - // The engine said it would have replaced it by now: the judging starts, and finds the module not - // yet healthy while the machine still lists it waiting (busyStill), until the gate's own bound. - end := now - g.Busy.Ended, g.Busy.PastBound = &end, true - g.Since, g.Passes, g.LastPass, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, nil, nil, nil - return false - } - g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, g.Busy.Words(), nil - g.Read(now, false, g.Busy.Words()) - return true - } - if g.Busy == nil { - return false - } - // Replaced: the judging starts at the report that said so. - end := now - if r, said := f.reports[g.Busy.Machine]; said && r.At != nil && r.At.After(g.Busy.Since) && r.At.Before(now) { - end = r.At.UTC() - } - g.Busy.Ended = &end - g.Since, g.Passes, g.LastPass, g.Last, g.Healthy, g.HealthyAt, g.Applied = &end, 0, nil, "", nil, nil, nil - return false -} - -// busyOn is the container of a judged module its machine last reported, on what the gate sent it, as left -// running because it said busy. -func busyOn(j judged, f gateFacts) (inventory.BusyWait, bool) { - r, said := f.reports[j.node] - if !said || r.At == nil || !f.reportedOn(j.node, r) { - return inventory.BusyWait{}, false - } - return inventory.WaitingFor(r.Busy, j.module) -} - -// busyStill is a judged module whose container its machine still lists waiting once the gate stopped waiting -// for it: not yet what was sent, so not healthy, whatever its health says of the container still running. -func busyStill(j judged, f gateFacts) (string, bool) { - b, waits := busyOn(j, f) - if !waits { - return "", false - } - return fmt.Sprintf("%s still runs its busy container %s, past its node-engine's bound of %s", - j.node, b.Container, b.Until.UTC().Format(time.RFC3339)), true -} diff --git a/cmd/mesh-controller/busy_test.go b/cmd/mesh-controller/busy_test.go deleted file mode 100644 index 4b8433e5..00000000 --- a/cmd/mesh-controller/busy_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A walk whose module's container waits as busy on its first machine says so, and starts that machine's -// first-machine gate only once the machine reports the container replaced (novox/hq ADR 0335 §4). -func TestAGateWaitsForABusyContainerAndStartsWhenItIsReplaced(t *testing.T) { - sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC) - g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent, - Sent: map[string]inventory.SentDeclaration{"novox": {Digest: "d1", Sequence: 7}}} - pairs := []judged{{module: "build-agent", node: "novox"}, {module: "mesh-controller", node: "novox"}} - reported := sent.Add(20 * time.Second) - waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", - State: inventory.BusyWaiting, Since: reported, Until: reported.Add(15 * time.Minute), - Said: "busy: novox/mesh-controller (check of #280), in hand since 00:58:00Z"}} - facts := func(at time.Time, busy []inventory.BusyWait) gateFacts { - // As judgeMoves hands it: the report held against the gate's own send. - return gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"novox": {Node: "novox", - Outcome: inventory.OutcomeApplied, At: &at, Declared: "d1", ReportedSequence: 7, Busy: busy}}} - } - - now := reported.Add(time.Minute) - if !busyWait(g, pairs, facts(reported, waiting), now) { - t.Fatal("a gate whose module's container waits as busy judged it") - } - if !g.Since.Equal(sent) || g.Busy == nil || g.Busy.Machine != "novox" || g.Busy.Ended != nil { - t.Fatalf("the gate does not keep the wait: %+v %+v", g.Since, g.Busy) - } - for _, said := range []string{gateLine(g), g.Last} { - if !strings.Contains(said, "waits for a busy container on novox") { - t.Errorf("the walk does not say it waits for a busy container there: %s", said) - } - } - if !strings.Contains(g.Last, "check of #280") { - t.Errorf("the wait does not say what the container holds: %s", g.Last) - } - - // Ten minutes on, still busy, within the engine's bound: still waiting, nothing judged, no bound run. - now = reported.Add(10 * time.Minute) - if !busyWait(g, pairs, facts(reported.Add(9*time.Minute), waiting), now) || g.Passes != 0 { - t.Fatalf("the gate stopped waiting within the bound: %+v", g) - } - - // The machine reports it replaced: the judging starts at that report. - replacedAt := reported.Add(12 * time.Minute) - now = replacedAt.Add(5 * time.Second) - replaced := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", - State: "replaced-when-idle", Since: reported, Until: reported.Add(15 * time.Minute)}} - if busyWait(g, pairs, facts(replacedAt, replaced), now) { - t.Fatal("the gate still waits after the container was replaced") - } - if g.Since == nil || !g.Since.Equal(replacedAt) || g.Busy.Ended == nil || !g.Busy.Ended.Equal(replacedAt) { - t.Fatalf("the gate does not start at the report that said it replaced: since %v, %+v", g.Since, g.Busy) - } - if strings.Contains(gateLine(g), "waits for a busy container") { - t.Errorf("a gate judging still says it waits: %s", gateLine(g)) - } - // And a later report listing it waiting again does not hold a gate that started. - if busyWait(g, pairs, facts(now, waiting), now.Add(time.Minute)) { - t.Error("a gate that started waits again") - } -} - -// An engine that stops reporting while it waits holds no walk for ever: past the engine's bound and the slack, -// the judging starts, and finds the module not yet healthy while its container is still listed waiting. -func TestAGateWaitsNoLongerThanTheEnginesBound(t *testing.T) { - sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC) - g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent} - pairs := []judged{{module: "build-agent", node: "novox"}} - waiting := []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", - State: inventory.BusyWaiting, Since: sent, Until: sent.Add(15 * time.Minute)}} - f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied, - At: &sent, Current: true, Busy: waiting}}} - now := sent.Add(15*time.Minute + busyGateSlack + time.Second) - if busyWait(g, pairs, f, now) { - t.Fatal("a gate waited past the engine's bound and the slack") - } - if !g.Busy.PastBound || !g.Since.Equal(now) { - t.Fatalf("the gate does not start, said past the bound: %+v %v", g.Busy, g.Since) - } - if why, still := busyStill(pairs[0], f); !still || !strings.Contains(why, "past its node-engine's bound") { - t.Fatalf("a module still running its busy container reads as what was sent: %q", why) - } -} - -// A report from an engine older than the field lists nothing: the gate starts at the send, as before; and a -// report not about the gate's send says nothing of it. -func TestAGateOfAnEngineThatNeverWaitsStartsAsBefore(t *testing.T) { - sent := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC) - g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent, - Sent: map[string]inventory.SentDeclaration{"ace": {Digest: "d2", Sequence: 9}}} - pairs := []judged{{module: "build-agent", node: "ace"}} - at := sent.Add(time.Minute) - older := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied, - At: &at, Declared: "d2", ReportedSequence: 9}}} - if busyWait(g, pairs, older, at) || g.Busy != nil || !g.Since.Equal(sent) { - t.Fatalf("a gate on an engine that never waits did not start at the send: %+v %v", g.Busy, g.Since) - } - elsewhere := gateFacts{sent: g.Sent, reports: map[string]inventory.Reported{"ace": {Node: "ace", Outcome: inventory.OutcomeApplied, - At: &at, Declared: "d1", ReportedSequence: 8, Busy: []inventory.BusyWait{{ID: "build-agent.server", - State: inventory.BusyWaiting, Until: at.Add(time.Hour)}}}}} - if busyWait(g, pairs, elsewhere, at) { - t.Fatal("a wait reported about an earlier declaration held the gate") - } -} - -// The wait is bounded on this controller's clock: an engine whose clock runs an hour ahead, saying it waits until -// then, holds the gate no longer than 15 minutes and the slack from when the controller first read the wait. -func TestAGateWaitIsBoundedOnTheControllersClock(t *testing.T) { - seen := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC) - g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &seen} - pairs := []judged{{module: "build-agent", node: "novox"}} - ahead := seen.Add(time.Hour) - f := gateFacts{reports: map[string]inventory.Reported{"novox": {Node: "novox", Outcome: inventory.OutcomeApplied, - At: &seen, Current: true, Busy: []inventory.BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", - State: inventory.BusyWaiting, Since: ahead, Until: ahead.Add(15 * time.Minute)}}}}} - if !busyWait(g, pairs, f, seen) || !g.Busy.Seen.Equal(seen) { - t.Fatalf("the gate did not start waiting, seen now: %+v", g.Busy) - } - if !busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack-time.Second)) { - t.Fatal("the gate stopped waiting within the bound on its own clock") - } - if busyWait(g, pairs, f, seen.Add(15*time.Minute+busyGateSlack+time.Second)) || !g.Busy.PastBound { - t.Fatalf("an engine's later word held the gate past the controller's bound: %+v", g.Busy) - } -} - -// `plan` says, for a machine whose node-engine is older than the field, each container whose busy is not sent. -func TestAPlanSaysABusyAnOlderEngineIsNotSent(t *testing.T) { - modules := []catalogue.Manifest{{Module: "build-agent", Resources: []map[string]any{ - {"id": "server", "type": "container", "busy": map[string]any{"command": "mesh-builder busy"}}, - {"id": "agent-env", "type": "file"}}}} - lines := busyNotSent(modules, false) - if len(lines) != 1 || !strings.Contains(lines[0], "build-agent.server declares busy") || - !strings.Contains(lines[0], "replaced at once") { - t.Fatalf("the plan does not say the busy not sent: %v", lines) - } - if lines := busyNotSent(modules, true); len(lines) != 0 { - t.Fatalf("a busy sent is said as not sent: %v", lines) - } -} diff --git a/cmd/mesh-controller/calls_verb_test.go b/cmd/mesh-controller/calls_verb_test.go index 4ff06dcb..d0958017 100644 --- a/cmd/mesh-controller/calls_verb_test.go +++ b/cmd/mesh-controller/calls_verb_test.go @@ -24,28 +24,4 @@ func TestAVerbAnswersInsideEveryWaitOnIt(t *testing.T) { } } -// A push — named or through command — answers before it runs: it sends the machine holding the bus -// first, and the broker reloading its user list forgets the answer it was about to permit. -func TestAPushAnswersBeforeItSends(t *testing.T) { - for _, c := range []struct { - verb string - args map[string]any - want bool - }{ - {"push", map[string]any{"node": "anchor", "why": "w"}, true}, - {"push", map[string]any{"why": "w"}, true}, - {"command", map[string]any{"command": "builds"}, false}, - {"status", map[string]any{}, false}, - {"assign", map[string]any{"node": "anchor", "module": "m"}, false}, - } { - argv, err := argvFor(c.verb, c.args) - if err != nil { - t.Fatalf("%s %v: %v", c.verb, c.args, err) - } - if got := answersFirst(argv); got != c.want { - t.Errorf("%s %v answers first: %v, want %v", c.verb, c.args, got, c.want) - } - } -} - // `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call. diff --git a/cmd/mesh-controller/catch_up.go b/cmd/mesh-controller/catch_up.go deleted file mode 100644 index f69e4afc..00000000 --- a/cmd/mesh-controller/catch_up.go +++ /dev/null @@ -1,643 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "net/url" - "path" - "regexp" - "slices" - "sort" - "strconv" - "strings" - "sync" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-sdk/go/asks" - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// **The controller catches up on merges it did not hear** (novox/hq ADR 0457, issue 1336). A merge the forge announced while the -// controller was being replaced was never heard, and nothing asked for it again: the trunk held a merge no walk answered. At its -// start, and each time it hears a merge, the controller reads the last catchUpWindow merge commits of a trunk from the forge, and -// hears each one it holds no record of as if the forge had announced it then, on the warrant the router holds for it. -// It lasts until release 6 of design 47 deletes delivery plans and batches (ADR 0429 §7, §8). - -// catchUpWindow is how many of a trunk's newest merge commits are read; an older merge is a person's to decide. -const catchUpWindow = 50 - -// kindMergeNotHeard is the condition of a merge the controller did not hear and holds no warrant to plan. -const kindMergeNotHeard = "merge-not-heard" - -// trunkMerge is a merge commit of a trunk as the forge lists it. -type trunkMerge struct { - Commit string - // Head is the merge commit's second parent: the pull request's head that was merged. - Head string - Subject string - At time.Time -} - -// trunkCommit is a commit of a trunk as the forge lists it, newest first: a merge commit has a second parent. -type trunkCommit struct { - Commit, SecondParent, Subject string - At time.Time -} - -// trunkCommitsPage is one page (fifty) of a trunk's commits, newest first; empty past the last. A variable so a test needs no -// forge. -var trunkCommitsPage = func(ctx context.Context, owner, repo, branch string, page int) ([]trunkCommit, error) { - var found struct { - Result []struct { - SHA string `json:"sha"` - Parents []struct { - SHA string `json:"sha"` - } `json:"parents"` - Commit struct { - Message string `json:"message"` - Committer struct { - Date string `json:"date"` - } `json:"committer"` - } `json:"commit"` - } `json:"result"` - } - apiPath := fmt.Sprintf("/repos/%s/%s/commits?sha=%s&limit=50&page=%d&stat=false&verification=false&files=false", - owner, repo, url.QueryEscape(branch), page) - missing, err := askForgeApi(ctx, apiPath, &found) - if err != nil { - return nil, err - } - if missing { - return nil, fmt.Errorf("the forge holds no branch %s of %s/%s", branch, owner, repo) - } - var out []trunkCommit - for _, c := range found.Result { - at, _ := time.Parse(time.RFC3339, c.Commit.Committer.Date) - tc := trunkCommit{Commit: c.SHA, Subject: firstLine(c.Commit.Message), At: at.UTC()} - if len(c.Parents) >= 2 { - tc.SecondParent = c.Parents[1].SHA - } - out = append(out, tc) - } - return out, nil -} - -// readTrunkMerges is the last catchUpWindow merge commits of a repository's trunk, oldest first: pages are read until that many -// merges are held or the trunk ends. -func readTrunkMerges(ctx context.Context, owner, repo, branch string) ([]trunkMerge, error) { - var merges []trunkMerge - for page := 1; page <= 8 && len(merges) < catchUpWindow; page++ { - commits, err := trunkCommitsPage(ctx, owner, repo, branch, page) - if err != nil { - return nil, err - } - if len(commits) == 0 { - break - } - for _, c := range commits { - // A merge of the trunk is a pull request's merge into it. A commit with a second parent that is none is a merge made - // inside some branch ("Merge remote-tracking branch 'origin/main' into fix/..."), which the trunk's history lists too and - // which no Approve ever covered: read as a gap it would raise a condition for nothing a person decides. - if c.SecondParent != "" && mergesIntoTrunk(c.Subject, branch) { - merges = append(merges, trunkMerge{Commit: c.Commit, Head: c.SecondParent, Subject: c.Subject, At: c.At}) - } - } - } - return lastMerges(merges), nil -} - -// sinceTheRecord are the merges that can be gaps (novox/hq ADR 0463 §2): those made since the controller's record of merges begins on -// the mesh (first, the earliest merge it holds of any repository and branch, less mergeGrace as owedLate judges). A merge before it is -// history on every trunk, and a trunk with no merge kept of its own still has the merges made since as gaps. With no merge kept -// anywhere there is no record to have a gap in. -func sinceTheRecord(merges []trunkMerge, kept bool, first time.Time) []trunkMerge { - if !kept { - return nil - } - var out []trunkMerge - for _, m := range merges { - if !m.At.Before(first.Add(-mergeGrace)) { - out = append(out, m) - } - } - return out -} - -// lastMerges is the newest catchUpWindow of merges, given newest first, oldest first. -func lastMerges(newestFirst []trunkMerge) []trunkMerge { - if len(newestFirst) > catchUpWindow { - newestFirst = newestFirst[:catchUpWindow] - } - out := slices.Clone(newestFirst) - slices.Reverse(out) - return out -} - -var mergeSubject = regexp.MustCompile(`^Merge pull request '(.*)' \(#(\d+)\) from (\S+) into (\S+)$`) - -// pullOfSubject is the pull request a merge commit's subject names: its title and number. -func pullOfSubject(subject string) (title string, number int, ok bool) { - m := mergeSubject.FindStringSubmatch(strings.TrimSpace(subject)) - if m == nil { - return "", 0, false - } - n, err := strconv.Atoi(m[2]) - if err != nil || n <= 0 { - return "", 0, false - } - return m[1], n, true -} - -// mergesIntoTrunk says a commit's subject is a pull request's merge into this trunk. -func mergesIntoTrunk(subject, branch string) bool { - m := mergeSubject.FindStringSubmatch(strings.TrimSpace(subject)) - return m != nil && m[4] == branch -} - -// headBranchOfSubject is the branch a merge commit's subject says was merged. -func headBranchOfSubject(subject string) string { - if m := mergeSubject.FindStringSubmatch(strings.TrimSpace(subject)); m != nil { - return m[3] - } - return "" -} - -// readMergeEvent is the merge as the forge would have announced it: the commit, the pull request, the files it changed and the -// directories among them that hold a module.json. A variable so a test needs no forge. -var readMergeEvent = func(ctx context.Context, owner, repo, branch string, tm trunkMerge) (link.SourceMoved, error) { - m := link.SourceMoved{Owner: owner, Repo: repo, Base: branch, Head: headBranchOfSubject(tm.Subject), HeadSHA: tm.Head, - Commit: tm.Commit, MergedAt: tm.At.UTC().Format(time.RFC3339)} - if title, number, ok := pullOfSubject(tm.Subject); ok { - m.Number, m.Title = number, title - } - var found struct { - Result struct { - Files []struct { - Filename string `json:"filename"` - Status string `json:"status"` - } `json:"files"` - } `json:"result"` - } - missing, err := askForgeApi(ctx, "/repos/"+owner+"/"+repo+"/git/commits/"+url.PathEscape(tm.Commit), &found) - if err != nil { - return m, err - } - if missing || len(found.Result.Files) == 0 { - return m, fmt.Errorf("the forge named no files for the merge %.8s of %s/%s", tm.Commit, owner, repo) - } - for _, f := range found.Result.Files { - m.Paths = append(m.Paths, f.Filename) - if f.Status == "removed" || f.Status == "deleted" { - m.Removed = append(m.Removed, f.Filename) - } - } - // The forge lists a merge's files by the page; a list as long as a page may be a beginning. - m.PathsTruncated = len(m.Paths) >= 300 - dirs := map[string]bool{} - for _, p := range m.Paths { - for d := path.Dir(p); ; d = path.Dir(d) { - dirs[d] = true - if d == "." || d == "/" { - break - } - } - } - if len(dirs) > 64 { - return m, nil // too many to look at: the old rule stands, a directory is a module when its manifest changed - } - var holding []string - for d := range dirs { - file := "module.json" - if d != "." { - file = d + "/module.json" - } - _, exists, err := manifestCached(ctx, owner, repo, file, tm.Commit) - if err != nil { - return m, err - } - if exists { - holding = append(holding, d) - } - } - sort.Strings(holding) - m.ModuleDirs, m.ModuleDirsSaid = holding, true - return m, nil -} - -// mergeAskPrefix is what the id of a merge ask made by the retired module mesh-delivery begins with: the pull request's number -// and the head's first twelve digits (`merge---`). The teller's ids name neither (see findMergeWarrant). -func mergeAskPrefix(number int, head string) string { - if len(head) > 12 { - head = head[:12] - } - return fmt.Sprintf("merge-%d-%s-", number, head) -} - -// askersOfMerges are the askers a merge's warrant is read under: the teller alone (ADR 0437 §6), since R4e stopped taking -// mesh-delivery's (ADR 0450 §8; issue 1318). -func askersOfMerges() []string { return []string{sdkdelivery.TellerModule} } - -// caughtUpWarrant judges a chosen merge ask of the router as the warrant of a merge: the ask is the operator's, the warrant -// answers that very ask as it was published, and it is the operator's Approve proved by the channel it came through (ADR 0437 -// §6, as movedOnApprove judges an announced one). The heads the Approve was bound to are checked by the caller, where it can -// name them. -func caughtUpWarrant(asker string, ask *asks.Ask, w *asks.Warrant) (string, error) { - if err := askerAccepted(asker); err != nil { - return "", err - } - switch { - case ask == nil || w == nil: - return "", errors.New("the router holds no closed record of the ask") - case ask.Who != asks.Operator: - return "", fmt.Errorf("the ask %s is for %q to answer, not the operator", ask.ID, ask.Who) - } - option, err := w.For(asker, *ask) - if err != nil { - return "", err - } - if option.ID != optionApprove || w.Option != optionApprove { - return "", fmt.Errorf("the operator chose %q on the ask %s, not an Approve", w.Option, w.Ask) - } - if w.By == nil || w.By.Who != asks.Operator || w.By.Verified == "" { - return "", fmt.Errorf("the ask %s was not answered by the operator on a channel that proves who answered", w.Ask) - } - return w.Says() + " (ask " + w.Ask + ")", nil -} - -// routerAskIDs are the ids of the asks an asker has in the router's record. A variable so a test needs no bus. -var routerAskIDs = func(ctx context.Context, inv catalogueReader, asker string) ([]string, error) { - serving := servingBus.Load() - if serving == nil { - return nil, errors.New("only the serving controller reads the router's record") - } - bucket, err := asksRecords(ctx, inv) - if err != nil { - return nil, err - } - if bucket == "" { - return nil, errors.New("no module declares the operator channel's records") - } - api, err := jetstream.New(serving.Conn()) - if err != nil { - return nil, err - } - kv, err := api.KeyValue(ctx, bucket) - if err != nil { - return nil, err - } - return askIDsOf(ctx, kv, asker) -} - -// askIDsOf are the ids of the asks an asker has in the router's bucket. -func askIDsOf(ctx context.Context, kv jetstream.KeyValue, asker string) ([]string, error) { - lister, err := kv.ListKeysFiltered(ctx, asker+".>") - if err != nil { - return nil, err - } - var ids []string - for key := range lister.Keys() { - ids = append(ids, strings.TrimPrefix(key, asker+".")) - } - // The lister stops its own watch when it has sent the last key, so a stop of ours finds it stopped: nats answers that with - // "invalid subscription", which is no failure of the read (issue 1336). Any other error stays one. - if err := lister.Stop(); err != nil && !errors.Is(err, nats.ErrBadSubscription) { - return nil, err - } - return ids, nil -} - -// closedRecord is a router record of an ask that ended: it never changes, so it is read once. -type closedRecord struct { - ask *asks.Ask - w *asks.Warrant -} - -var closedRecords sync.Map // asker + "." + id -> closedRecord - -// closedRecordOf is the router's record of an ask, kept once it is closed. A nil ask is an ask still open, ended without a -// choice, or not there. -func closedRecordOf(ctx context.Context, inv catalogueReader, asker, id string) (*asks.Ask, *asks.Warrant, error) { - if v, ok := closedRecords.Load(asker + "." + id); ok { - r := v.(closedRecord) - return r.ask, r.w, nil - } - ask, w, err := routerAskOf(ctx, inv, asker, id) - if err != nil { - return nil, nil, err - } - if ask != nil && w != nil { - closedRecords.Store(asker+"."+id, closedRecord{ask, w}) - } - return ask, w, nil -} - -// deliveryIDOf is the id the gate gives a delivery, and so the key of its head in the heads an Approve binds: the repository as -// owner/name, lower-cased, `@` and the first twelve characters of the head. -func deliveryIDOf(owner, repo, head string) string { - if len(head) > 12 { - head = head[:12] - } - return strings.ToLower(owner+"/"+repo) + "@" + head -} - -// coversHead says an ask the operator approved is for this merge: its Approve is bound to the heads of the pull request alone -// (mergeAct over {delivery id: head}, which is how a pull request not in a group is asked), or, for a group's ask, whose other -// members' heads the router's record does not carry in full, the ask as published names this pull request at this head in its -// details ("# ... at :"), and the digest the warrant answers covers those words. -func coversHead(ask *asks.Ask, owner, repo string, number int, head string) (how string, ok bool) { - option, found := ask.Option(optionApprove) - if !found { - return "", false - } - heads := map[string]string{deliveryIDOf(owner, repo, head): head} - if option.Performs(mergeAct(ask.ID, optionApprove, heads)) == nil { - return "bound to the head of this pull request", true - } - short := head - if len(short) > 12 { - short = short[:12] - } - line := regexp.MustCompile(`(?m)^` + regexp.QuoteMeta(repo) + `#` + strconv.Itoa(number) + ` .* at ` + regexp.QuoteMeta(short) + `:`) - if line.MatchString(ask.Details) { - return "a group's ask naming this pull request at this head", true - } - return "", false -} - -// findMergeWarrant finds the warrant of a merge, with no read the controller does not make today: the router's chosen merge -// ask for it. Under the teller (releases 3e on) the id is `merge---`, which names neither the pull -// request nor the head, so the teller's closed asks are read and the one the operator approved whose heads cover the merge's -// second parent is the warrant (coversHead). Under mesh-delivery, which made the earlier merges, the id names the pull request -// and the head and is found by them. Why it holds; or found false with why not, for a person to read; or an error when the -// router could not be read, which is no answer. A variable so a test needs no bus. -var findMergeWarrant = func(ctx context.Context, inv catalogueReader, owner, repo string, number int, head string) (why string, found bool, reason string, err error) { - reason = "the router holds no chosen merge ask of " + strings.Join(askersOfMerges(), " or ") + " that covers the head " + head[:min(12, len(head))] - for _, asker := range askersOfMerges() { - ids, err := routerAskIDs(ctx, inv, asker) - if err != nil { - return "", false, "", err - } - for _, id := range ids { - named := number > 0 && strings.HasPrefix(id, mergeAskPrefix(number, head)) - if asker == sdkdelivery.TellerModule && !strings.HasPrefix(id, sdkdelivery.AskKindMerge+"-") { - continue - } - ask, w, err := closedRecordOf(ctx, inv, asker, id) - if err != nil { - return "", false, "", err - } - if ask == nil || w == nil { - continue - } - warrant, err := caughtUpWarrant(asker, ask, w) - if err != nil { - if named { - reason = fmt.Sprintf("%s's ask %s holds no operator's Approve: %v", asker, id, err) - } - continue - } - how, covers := coversHead(ask, owner, repo, number, head) - if !covers && !named { - continue - } - if !covers { - how = "the ask's id names this pull request and head" - } - return warrant + ", " + how, true, "", nil - } - } - return "", false, reason, nil -} - -// caughtTrunk is one trunk the catalogue builds from. -type caughtTrunk struct{ Owner, Repo, Branch string } - -func (t caughtTrunk) repository() string { return t.Owner + "/" + t.Repo } - -// trunksOfCatalogue are the trunks of the repositories on the mesh's forge the catalogue builds a module from. -func trunksOfCatalogue(forms *sourceForms, entries []inventory.Entry) []caughtTrunk { - var out []caughtTrunk - for _, e := range entries { - if e.Provided || e.Source.Repository == "" { - continue - } - owner, name, onForge := forms.onTheForge(e.Source.Repository, e.Source.Seat) - if !onForge { - continue - } - branch := followedBranch(e.Source.Ref) - if branch == "" { - branch = "main" - } - t := caughtTrunk{owner, name, branch} - if !slices.Contains(out, t) { - out = append(out, t) - } - } - return out -} - -// catchUpBeside runs a catch-up beside the hearing of the merge that asked for it, so the bus's acknowledgement of that merge -// waits for no forge. A variable so a test runs it where it stands. -var catchUpBeside = func(run func()) { go run() } - -// catchingUp makes one catch-up at a time. -var catchingUp sync.Mutex - -// settledMerges are the merges this process found nothing to plan for: heard, and no record kept, because nothing reads them. -// They are not read from the forge again. -var settledMerges sync.Map - -// catchUp hears the merges of the catalogue's trunks (only one, when only names it) that the controller holds no record of. -// It says whether a trunk was left for later: the forge, the router or the store did not answer. -func (f following) catchUp(ctx context.Context, only *caughtTrunk) (later bool) { - catchingUp.Lock() - defer catchingUp.Unlock() - inv := f.open.inventory - entries, err := inv.Catalogued(ctx) - if err != nil { - fmt.Printf("catch-up: the catalogue cannot be read, so no merge is caught up now: %v\n", err) - return true - } - kept, first, err := inv.EarliestKept(ctx) - if err != nil { - fmt.Printf("catch-up: what the controller kept cannot be read, so no merge is caught up now: %v\n", err) - return true - } - if kept { - fmt.Printf("catch-up: the controller's record of merges begins %s; a merge made before it, less %s, is history on every trunk\n", - first.UTC().Format(time.RFC3339), mergeGrace) - } else { - fmt.Println("catch-up: the controller holds no merge of any trunk yet, so none can have been missed") - } - trunks := trunksOfCatalogue(newSourceForms(ctx, inv), entries) - if only != nil { - trunks = slices.DeleteFunc(trunks, func(t caughtTrunk) bool { - return !strings.EqualFold(t.repository(), only.repository()) || t.Branch != only.Branch - }) - } - for _, t := range trunks { - if f.catchUpTrunk(ctx, t, kept, first) { - later = true - } - } - return later -} - -func (f following) catchUpTrunk(ctx context.Context, t caughtTrunk, kept bool, first time.Time) (later bool) { - inv := f.open.inventory - repository := t.repository() - merges, err := readTrunkMerges(ctx, t.Owner, t.Repo, t.Branch) - if err != nil { - // Not nothing to catch up: left for the next start or the next heard merge (ADR 0457 clause 3). - fmt.Printf("catch-up: the forge did not answer for the last merges of %s %s, so they are not caught up now; the next start "+ - "or the next merge heard there does it: %v\n", repository, t.Branch, err) - return true - } - // **A merge made before the controller's record of merges begins is history, never a gap** (novox/hq ADR 0463 §2). - read := len(merges) - merges = sinceTheRecord(merges, kept, first) - fmt.Printf("catch-up: %s %s: %d of the last %d pull request merges were made since the record began\n", repository, t.Branch, - len(merges), read) - unwarranted := map[string]bool{} - for _, tm := range merges { - id := fmt.Sprintf("%s_%.8s", repository, tm.Commit) - if _, done := settledMerges.Load(repository + "@" + tm.Commit); done { - continue - } - if _, known, err := inv.MergeOf(ctx, repository, tm.Commit); err != nil { - fmt.Printf("catch-up: the record of %s cannot be read, so %s is not caught up now: %v\n", repository, tm.Commit[:8], err) - return true - } else if known { - continue - } - m, err := readMergeEvent(ctx, t.Owner, t.Repo, t.Branch, tm) - if err != nil { - fmt.Printf("catch-up: the merge %.8s of %s cannot be read from the forge, so it is not caught up now: %v\n", tm.Commit, repository, err) - later = true - continue - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return true - } - read, err := readForPlanning(ctx, inv) - if err != nil { - return true - } - facts, err := factsOfMerge(ctx, inv, m, time.Now().UTC(), entries, read) - if err != nil { - fmt.Printf("catch-up: %s %.8s cannot be judged now: %v\n", repository, tm.Commit, err) - later = true - continue - } - // The very test hearMerge makes: a merge it would plan, or take as a first build, is a gap; any other it would say - // nothing reads, and is no gap for a person to decide. - if !facts.plans() && !facts.addsModuleToBuilds { - settledMerges.Store(repository+"@"+tm.Commit, true) - continue - } - why, found, reason, err := findMergeWarrant(ctx, inv, t.Owner, t.Repo, m.Number, tm.Head) - if err != nil { - fmt.Printf("catch-up: the router cannot be read, so %s %.8s is not caught up now: %v\n", repository, tm.Commit, err) - later = true - continue - } - if !found { - unwarranted[id] = true - f.raiseMergeNotHeard(ctx, repository, t.Branch, tm, m, reason) - continue - } - now := time.Now().UTC() - said := fmt.Sprintf("caught up: merged %s, not heard, planned %s", tm.At.Format(time.RFC3339), now.Format(time.RFC3339)) - fmt.Printf("catch-up: %s merged into %s (%.8s) was not heard; %s; on the warrant %s\n", repository, t.Branch, tm.Commit, said, why) - m.Title = strings.TrimSpace(m.Title + " (" + said + ")") - if err := f.hearMerge(ctx, m, now); err != nil { - fmt.Printf("catch-up: %s %.8s could not be heard now: %v\n", repository, tm.Commit, err) - later = true - continue - } - if _, known, err := inv.MergeOf(ctx, repository, tm.Commit); err == nil && !known { - settledMerges.Store(repository+"@"+tm.Commit, true) // heard, and nothing reads it - } - } - // Cleared only when every gap was judged: one left for later may still be open. - if !later { - f.clearMergesHeard(ctx, repository, unwarranted) - } - return later -} - -// raiseMergeNotHeard keeps open, and observes again, the condition of a merge with no warrant to plan it. One for each -// repository and commit, however many catch-ups find it. -func (f following) raiseMergeNotHeard(ctx context.Context, repository, branch string, tm trunkMerge, m link.SourceMoved, reason string) { - fmt.Printf("catch-up: %s merged into %s (%.8s) was not heard, and is not planned: %s\n", repository, branch, tm.Commit, reason) - if conditionsFrom == nil { - return - } - o := conditions.Observation{Scope: conditions.ScopeMerge, ID: fmt.Sprintf("%s_%.8s", repository, tm.Commit), Kind: kindMergeNotHeard, - Severity: conditions.Warning, Source: "catch-up", Resolver: conditions.ResolverOperator, - Summary: fmt.Sprintf("the merge %.8s of %s into %s was not heard by the controller, and no operator's Approve of it is on "+ - "record to plan it on: a person decides", tm.Commit, repository, branch), - Said: fmt.Sprintf("%s %.8s (pull request %d, head %.8s, merged %s): %s", repository, tm.Commit, m.Number, tm.Head, - tm.At.Format(time.RFC3339), reason), - Headline: fmt.Sprintf("A merge into %s was never delivered", repository), - Explanation: fmt.Sprintf("A merge into %s was made while the controller did not hear it, and no Approve of yours for it is on "+ - "record that the controller can plan it on. It stays open while the merge is among the latest %d merges of that repository. "+ - "The details say what was looked for.", repository, catchUpWindow), - Needs: "decide whether that merge should be delivered, and say so if it should.", - Resolved: "the merge is planned, or is no longer among the latest merges of its repository"} - if _, err := conditionsFrom.Observe(ctx, o); err != nil { - fmt.Printf("catch-up: the condition of %s %.8s could not be kept: %v\n", repository, tm.Commit, err) - } -} - -// clearMergesHeard clears the open conditions of a repository's merges not heard that this catch-up did not find again: the -// gap was planned, or left the window. -func (f following) clearMergesHeard(ctx context.Context, repository string, still map[string]bool) { - if conditionsFrom == nil { - return - } - open, err := conditionsFrom.Open(ctx) - if err != nil { - fmt.Printf("catch-up: the open conditions cannot be read, so none is cleared now: %v\n", err) - return - } - for _, c := range open { - if c.Kind != kindMergeNotHeard || c.Subject.Scope != conditions.ScopeMerge || - !strings.HasPrefix(c.Subject.ID, repository+"_") || still[c.Subject.ID] { - continue - } - if _, err := conditionsFrom.Clear(ctx, c.Key, "the merge is planned, or left the last merges of its trunk"); err != nil { - fmt.Printf("catch-up: the condition %s could not be cleared: %v\n", c.Key, err) - } - } -} - -// catchUpAtStart catches up every trunk once the controller serves, and again every minute while a trunk was left for later. -func catchUpAtStart(ctx context.Context, open *stores) { - select { - case <-ctx.Done(): - return - case <-time.After(30 * time.Second): - } - for try := 0; try < 20; try++ { - if !(following{open: open}).catchUp(ctx, nil) { - return - } - select { - case <-ctx.Done(): - return - case <-time.After(time.Minute): - } - } - fmt.Println("catch-up: a trunk is still left for later after twenty tries; the next merge heard there does it") -} diff --git a/cmd/mesh-controller/catch_up_integration_test.go b/cmd/mesh-controller/catch_up_integration_test.go deleted file mode 100644 index c0b698fc..00000000 --- a/cmd/mesh-controller/catch_up_integration_test.go +++ /dev/null @@ -1,352 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "errors" - "fmt" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" -) - -// The checks of ADR 0457 (issue 1336): what the controller does about merges it did not hear. - -// aTrunk is a forge's trunk as the catch-up reads it: its merges oldest first, the module each moves, and which of them the -// router holds an operator's Approve for. -type aTrunk struct { - merges []trunkMerge - module map[string]string - approved map[string]bool - read []string // the trunks asked of the forge, as owner/repo - events int // merges read in full from the forge - silent bool // the forge does not answer -} - -func aTrunkOf(t *testing.T, repo string, modules ...string) *aTrunk { - t.Helper() - tr := &aTrunk{module: map[string]string{}, approved: map[string]bool{}} - for i, m := range modules { - c := fmt.Sprintf("%s-c%02d", repo, i+1) - tr.merges = append(tr.merges, trunkMerge{Commit: c, Head: fmt.Sprintf("%shead%02d", repo, i+1), - Subject: fmt.Sprintf("Merge pull request 'change %d' (#%d) from pr/%s into main", i+1, 100+i, c), - At: t0.Add(time.Duration(i) * time.Minute)}) - tr.module[c] = m - tr.approved[c] = true - } - was, wasEvent, wasWarrant := trunkCommitsPage, readMergeEvent, findMergeWarrant - trunkCommitsPage = func(_ context.Context, owner, name, _ string, page int) ([]trunkCommit, error) { - tr.read = append(tr.read, owner+"/"+name) - if tr.silent { - return nil, errors.New("the forge did not answer") - } - if page > 1 || owner+"/"+name != "novox/"+repo { - return nil, nil - } - var newest []trunkCommit - for i := len(tr.merges) - 1; i >= 0; i-- { - m := tr.merges[i] - newest = append(newest, trunkCommit{Commit: m.Commit, SecondParent: m.Head, Subject: m.Subject, At: m.At}) - } - return newest, nil - } - readMergeEvent = func(_ context.Context, owner, name, branch string, tm trunkMerge) (link.SourceMoved, error) { - tr.events++ - m := catalogueMerge(tm.Commit, tr.module[tm.Commit], tm.At) - m.Owner, m.Repo, m.Base, m.HeadSHA = owner, name, branch, tm.Head - _, m.Number, _ = pullOfSubject(tm.Subject) - return m, nil - } - findMergeWarrant = func(_ context.Context, _ catalogueReader, _, _ string, number int, head string) (string, bool, string, error) { - for _, m := range tr.merges { - if strings.Contains(m.Subject, fmt.Sprintf("(#%d)", number)) && m.Head == head && tr.approved[m.Commit] { - return "the operator chose Approve (ask merge-" + fmt.Sprint(number) + ")", true, "", nil - } - } - return "", false, "the router holds no chosen merge ask for the pull request and head", nil - } - wasBeside := catchUpBeside - catchUpBeside = func(run func()) { run() } - t.Cleanup(func() { - trunkCommitsPage, readMergeEvent, findMergeWarrant, catchUpBeside = was, wasEvent, wasWarrant, wasBeside - settledMerges.Clear() - }) - return tr -} - -// aKeptMergeBefore is a merge the controller heard before the merges of a test: where its record begins, so what a trunk holds from -// then on can be a gap (novox/hq ADR 0463 §2). -func aKeptMergeBefore(t *testing.T, open *stores) { - t.Helper() - hear(t, open, catalogueMerge("mesh-catalog-seed", "app", t0.Add(-time.Hour)), t0.Add(-time.Hour)) -} - -func mergeKnown(t *testing.T, open *stores, commit string) bool { - t.Helper() - _, known, err := open.inventory.MergeOf(t.Context(), "novox/mesh-catalog", commit) - if err != nil { - t.Fatal(err) - } - return known -} - -// Clause 1: the trunk holds merges 1, 2 and 3 and the controller holds records of 1 and 3 only; at its start it plans 2, said -// as caught up; a second start plans nothing new. -func TestAGapBeforeANewerHeardMergeIsPlannedAtTheControllersStart(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksRecorded(t) - tr := aTrunkOf(t, "mesh-catalog", "app", "notes", "app") - hear(t, open, catalogueMerge("mesh-catalog-c01", "app", t0), t0.Add(time.Second)) - hear(t, open, catalogueMerge("mesh-catalog-c03", "app", t0.Add(2*time.Minute)), t0.Add(2*time.Minute+time.Second)) - if mergeKnown(t, open, "mesh-catalog-c02") { - t.Fatal("the merge to be caught up is known before the catch-up") - } - if (following{open: open}).catchUp(t.Context(), nil) { - t.Fatal("the catch-up says a trunk was left for later") - } - if !mergeKnown(t, open, "mesh-catalog-c02") { - t.Fatal("the merge the controller did not hear was not planned") - } - kept, _, _ := open.inventory.MergeOf(t.Context(), "novox/mesh-catalog", "mesh-catalog-c02") - if said := announcedOf(kept).Title; !strings.Contains(said, "caught up: merged ") || !strings.Contains(said, "not heard, planned ") { - t.Fatalf("the merge does not say it was caught up: %q", said) - } - cutAt(t, open, t0.Add(10*time.Minute)) - ws, _ := walks(t, open) - if len(ws) == 0 { - t.Fatal("no walk answers the caught-up merge") - } - if _, in := ws[0].Modules["notes"]; !in { - t.Fatalf("the walk does not build notes, which the caught-up merge moved: %v", ws[0].Modules) - } - before := tr.events - if (following{open: open}).catchUp(t.Context(), nil) || tr.events != before { - t.Fatalf("a second start read %d merge(s) in full again", tr.events-before) - } -} - -// Clause 1: a heard merge of a repository whose trunk holds earlier merges the controller has no record of plans them all, the -// earlier first, and the forge is read for that repository's trunk only. -func TestAHeardMergeCatchesUpTheEarlierOnesOfItsOwnTrunkOnly(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksRecorded(t) - tr := aTrunkOf(t, "mesh-catalog", "app", "notes", "app") - var order []string - inner := readMergeEvent - readMergeEvent = func(ctx context.Context, owner, name, branch string, tm trunkMerge) (link.SourceMoved, error) { - order = append(order, tm.Commit) - return inner(ctx, owner, name, branch, tm) - } - t.Cleanup(func() { readMergeEvent = inner }) - tr.read = nil - heard := catalogueMerge("mesh-catalog-c03", "app", t0.Add(2*time.Minute)) - if err := (following{open: open}).SourceMoved(t.Context(), heard); err != nil { - t.Fatal(err) - } - for _, c := range []string{"mesh-catalog-c01", "mesh-catalog-c02", "mesh-catalog-c03"} { - if !mergeKnown(t, open, c) { - t.Fatalf("%s is not planned: the heard merge and the earlier ones of its trunk are all wanted", c) - } - } - if len(order) != 2 || order[0] != "mesh-catalog-c01" || order[1] != "mesh-catalog-c02" { - t.Fatalf("the earlier merges were heard in the order %v, not the trunk's", order) - } - if len(tr.read) == 0 { - t.Fatal("the forge was not read for the trunk of the merge heard") - } - for _, r := range tr.read { - if r != "novox/mesh-catalog" { - t.Fatalf("the forge was read for %s, not only for the trunk of the merge heard: %v", r, tr.read) - } - } -} - -// Clause 1: a trunk of fifty-one merges whose oldest is unheard plans nothing: the oldest is outside the window. -func TestAMergeOlderThanTheLastFiftyIsNotRead(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksRecorded(t) - modules := make([]string, 51) - for i := range modules { - modules[i] = "app" - } - aTrunkOf(t, "mesh-catalog", modules...) - // Fifty are known; the oldest is not. - for i := 2; i <= 51; i++ { - c := fmt.Sprintf("mesh-catalog-c%02d", i) - hear(t, open, catalogueMerge(c, "app", t0.Add(time.Duration(i)*time.Minute)), t0.Add(time.Duration(i)*time.Minute+time.Second)) - } - // Paged as the forge pages: fifty a page, and a trunk of another repository is empty (a short page, or none). - pages := trunkCommitsPage - trunkCommitsPage = func(ctx context.Context, owner, name, branch string, page int) ([]trunkCommit, error) { - all, err := pages(ctx, owner, name, branch, 1) - if err != nil || page < 1 || (page-1)*50 >= len(all) { - return nil, err - } - return all[(page-1)*50 : min(page*50, len(all))], nil - } - t.Cleanup(func() { trunkCommitsPage = pages }) - (following{open: open}).catchUp(t.Context(), nil) - if mergeKnown(t, open, "mesh-catalog-c01") { - t.Fatal("the oldest of fifty-one merges was read and planned, outside the window") - } -} - -// Clause 2: a gap with no chosen Approve in the router is not planned and raises one condition naming its repository and -// commit; a second start leaves that one condition open, observed twice; planning the gap clears it. -func TestAGapWithNoApproveRaisesOneConditionKeptOpenAndClearedWhenPlanned(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksRecorded(t) - aKeptMergeBefore(t, open) - keeper, _ := withConditionsInMemory(t) - tr := aTrunkOf(t, "mesh-catalog", "notes") - tr.approved["mesh-catalog-c01"] = false - catch := func() { (following{open: open}).catchUp(t.Context(), nil) } - catch() - if mergeKnown(t, open, "mesh-catalog-c01") { - t.Fatal("a merge with no Approve on record was planned") - } - raised := func() []conditions.Condition { - t.Helper() - all, err := keeper.Open(t.Context()) - if err != nil { - t.Fatal(err) - } - var out []conditions.Condition - for _, c := range all { - if c.Kind == kindMergeNotHeard { - out = append(out, c) - } - } - return out - } - one := raised() - if len(one) != 1 || !strings.Contains(one[0].Subject.ID, "novox/mesh-catalog") || !strings.Contains(one[0].Subject.ID, "mesh-cata") { - t.Fatalf("the gap raised %+v, not one condition naming its repository and commit", one) - } - catch() - if again := raised(); len(again) != 1 || again[0].Observations < 2 { - t.Fatalf("a second catch-up left %+v: one condition, observed twice, was wanted", again) - } - tr.approved["mesh-catalog-c01"] = true - catch() - if !mergeKnown(t, open, "mesh-catalog-c01") { - t.Fatal("the gap was not planned once its Approve was on record") - } - if left := raised(); len(left) != 0 { - t.Fatalf("planning the gap left its condition open: %+v", left) - } -} - -// Clause 3: a forge that does not answer leaves the catch-up undone and says so, and the next start does it. -func TestAForgeThatDoesNotAnswerLeavesTheCatchUpForTheNextStart(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksRecorded(t) - aKeptMergeBefore(t, open) - tr := aTrunkOf(t, "mesh-catalog", "notes") - tr.silent = true - if later := (following{open: open}).catchUp(t.Context(), nil); !later { - t.Fatal("a forge that did not answer counted as nothing to catch up") - } - if mergeKnown(t, open, "mesh-catalog-c01") { - t.Fatal("a merge was planned without the forge's word") - } - tr.silent = false - if later := (following{open: open}).catchUp(t.Context(), nil); later || !mergeKnown(t, open, "mesh-catalog-c01") { - t.Fatalf("the next start did not catch up (left for later: %v)", later) - } -} - -// Finding 2 of the review: a gap that adds a module is a gap, judged as hearMerge judges it. With the operator's Approve on record it -// is heard (hearMerge asks the new module in its walk); without, it raises the condition and is not lost in silence. -func TestAGapThatAddsAModuleIsHeardOnItsWarrantAndRaisedWithout(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksRecorded(t) - aKeptMergeBefore(t, open) - keeper, _ := withConditionsInMemory(t) - tr := aTrunkOf(t, "mesh-catalog", "newmodule") - tr.approved["mesh-catalog-c01"] = false - catch := func() { (following{open: open}).catchUp(t.Context(), nil) } - catch() - if mergeKnown(t, open, "mesh-catalog-c01") { - t.Fatal("a merge adding a module was heard with no Approve on record") - } - all, err := keeper.Open(t.Context()) - if err != nil { - t.Fatal(err) - } - var raised int - for _, c := range all { - if c.Kind == kindMergeNotHeard { - raised++ - } - } - if raised != 1 { - t.Fatalf("a merge adding a module with no Approve raised %d condition(s), not one: it was lost in silence", raised) - } - tr.approved["mesh-catalog-c01"] = true - catch() - if !mergeKnown(t, open, "mesh-catalog-c01") { - t.Fatal("a merge adding a module was not heard once its Approve was on record") - } -} - -// A trunk with merges from before the controller began keeping merges, and merges since: only the later ones are gaps. -func TestAMergeFromBeforeTheRecordBeganIsHistoryAndNoCondition(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksRecorded(t) - keeper, _ := withConditionsInMemory(t) - tr := aTrunkOf(t, "mesh-catalog", "notes", "notes", "notes") - tr.approved["mesh-catalog-c01"], tr.approved["mesh-catalog-c02"], tr.approved["mesh-catalog-c03"] = false, false, false - // The record begins at the second merge: the first is two days older. - tr.merges[0].At = t0.Add(-48 * time.Hour) - hear(t, open, catalogueMerge("mesh-catalog-c02", "notes", t0.Add(time.Minute)), t0.Add(time.Minute)) - delete(tr.approved, "mesh-catalog-c02") - (following{open: open}).catchUp(t.Context(), nil) - // The first merge (two days before the record) was not even read from the forge: only the third, which is unknown, was. - if tr.events != 1 { - t.Fatalf("%d merge(s) were read from the forge: the one from before the record is history and is not read", tr.events) - } - all, err := keeper.Open(t.Context()) - if err != nil { - t.Fatal(err) - } - var raised int - for _, c := range all { - if c.Kind == kindMergeNotHeard { - raised++ - } - } - if raised != 1 { - t.Fatalf("%d condition(s) raised: only the merge since the record began (the third) is a gap", raised) - } -} - -// novox/hq ADR 0463 §2: the record begins at the earliest merge kept on ANY trunk. A trunk the controller kept no merge of has the -// merges made since as gaps (mesh-delivery #1 on novox), and the merges made before as history. -func TestATrunkWithNoKeptMergeHasItsLaterMergesAsGapsAndItsEarlierOnesAsHistory(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksRecorded(t) - // The record begins on another trunk, at t0 less an hour. - hear(t, open, repoMerge("one", "c-one", t0.Add(-time.Hour)), t0.Add(-time.Hour)) - tr := aTrunkOf(t, "mesh-catalog", "notes", "notes") - tr.merges[0].At = t0.Add(-72 * time.Hour) // before the record: history - (following{open: open}).catchUp(t.Context(), nil) - if mergeKnown(t, open, "mesh-catalog-c01") { - t.Fatal("a merge from before the record began was heard") - } - if !mergeKnown(t, open, "mesh-catalog-c02") { - t.Fatal("a merge since the record began, on a trunk with no merge kept, was not caught up") - } -} diff --git a/cmd/mesh-controller/catch_up_test.go b/cmd/mesh-controller/catch_up_test.go deleted file mode 100644 index eb7be1bb..00000000 --- a/cmd/mesh-controller/catch_up_test.go +++ /dev/null @@ -1,318 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/asks" - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" -) - -// The pieces of the catch-up that need no store (novox/hq ADR 0457, issue 1336). - -func TestAMergeCommitsSubjectNamesItsPullRequest(t *testing.T) { - title, number, ok := pullOfSubject("Merge pull request 'Fix keycloak's sign-in (issue 1332)' (#378) from fix/1332-sign-in into main") - if !ok || number != 378 || title != "Fix keycloak's sign-in (issue 1332)" { - t.Fatalf("read %q %d %v", title, number, ok) - } - if b := headBranchOfSubject("Merge pull request 'x' (#378) from fix/1332-sign-in into main"); b != "fix/1332-sign-in" { - t.Fatalf("the head branch read as %q", b) - } - if _, _, ok := pullOfSubject("Merge branch 'main' of somewhere"); ok { - t.Fatal("a merge that names no pull request read as one") - } -} - -func TestOnlyTheLastFiftyMergesOfATrunkAreRead(t *testing.T) { - pages := map[int][]trunkCommit{} - var all []trunkCommit // newest first: 51 merges, a plain commit between each pair - for i := 51; i >= 1; i-- { - all = append(all, trunkCommit{Commit: fmt.Sprintf("merge-%02d", i), SecondParent: fmt.Sprintf("head-%02d", i), - Subject: fmt.Sprintf("Merge pull request 'change %d' (#%d) from fix/x into main", i, i)}, - trunkCommit{Commit: fmt.Sprintf("plain-%02d", i)}) - } - for i := 0; i < len(all); i += 50 { - pages[i/50+1] = all[i:min(i+50, len(all))] - } - was := trunkCommitsPage - trunkCommitsPage = func(_ context.Context, _, _, _ string, page int) ([]trunkCommit, error) { return pages[page], nil } - t.Cleanup(func() { trunkCommitsPage = was }) - got, err := readTrunkMerges(t.Context(), "novox", "mesh-catalog", "main") - if err != nil { - t.Fatal(err) - } - if len(got) != catchUpWindow || got[0].Commit != "merge-02" || got[len(got)-1].Commit != "merge-51" { - t.Fatalf("read %d merges from %s to %s: the oldest of 51 is outside the window", len(got), got[0].Commit, got[len(got)-1].Commit) - } - if got[0].Head != "head-02" { - t.Fatalf("a merge's head is its second parent, not %q", got[0].Head) - } -} - -func TestAForgeThatDoesNotAnswerIsAnErrorNotAnEmptyTrunk(t *testing.T) { - was := trunkCommitsPage - trunkCommitsPage = func(context.Context, string, string, string, int) ([]trunkCommit, error) { - return nil, errors.New("the forge did not answer") - } - t.Cleanup(func() { trunkCommitsPage = was }) - if got, err := readTrunkMerges(t.Context(), "novox", "mesh-catalog", "main"); err == nil { - t.Fatalf("a forge that did not answer read as %d merges: nothing to catch up", len(got)) - } -} - -func TestACaughtUpMergeIsAcceptedOnTheSameWarrantAsAnAnnouncedOne(t *testing.T) { - ok := approvedMerge(t) - if _, err := caughtUpWarrant(sdkdelivery.TellerModule, &ok.ask, &ok.warrant); err != nil { - t.Errorf("the operator's Approve under the teller was refused: %v", err) - } - // 1318 (ADR 0450 §8): mesh-delivery's warrant of before the gate is no longer taken. - if _, err := caughtUpWarrant(theDelivery, &ok.ask, &ok.warrant); err == nil { - t.Error("a warrant under mesh-delivery was accepted") - } - if _, err := caughtUpWarrant("keycloak", &ok.ask, &ok.warrant); err == nil { - t.Error("a warrant of an asker the controller takes none from was accepted") - } - declined := ok.warrant - declined.Option, declined.Label = "decline", "Decline" - if _, err := caughtUpWarrant(theDelivery, &ok.ask, &declined); err == nil { - t.Error("a Decline was accepted as a warrant") - } - unproved := ok.warrant - by := *unproved.By - by.Verified = "" - unproved.By = &by - if _, err := caughtUpWarrant(theDelivery, &ok.ask, &unproved); err == nil { - t.Error("an answer whose channel proves nobody was accepted") - } - if _, err := caughtUpWarrant(theDelivery, nil, nil); err == nil { - t.Error("a router that holds nothing was accepted") - } -} - -// aTellersAsk is the ask the delivery gate composes for a pull request, as the router holds it once the operator chose Approve -// (the gate's composeMergeAsk: its id from sdkdelivery.AskingID of the group and the heads, its Approve bound to mergeAct over the -// heads, the pull request and its head's first twelve digits in the details), with the warrant the router wrote. -func aTellersAsk(t *testing.T, group string, members ...sdkdelivery.Head) (asks.Ask, asks.Warrant) { - t.Helper() - key, err := sdkdelivery.MergeAskKey(group, members) - if err != nil { - t.Fatal(err) - } - id := sdkdelivery.AskingID(key, 1) - heads := map[string]string{} - var details []string - for _, m := range members { - owner, repo, _ := strings.Cut(m.Repository, "/") - heads[deliveryIDOf(owner, repo, m.Head)] = m.Head - details = append(details, fmt.Sprintf("%s#%d %q at %s: issues: none named; review: approved; reaches: no delivery plan yet", repo, m.Number, "title", m.Head[:12])) - } - bind := func(option string) string { - d, err := asks.ActDigest(mergeAct(id, option, heads)) - if err != nil { - t.Fatal(err) - } - return d - } - at := time.Date(2026, 10, 20, 12, 2, 1, 0, time.UTC) - ask := asks.Ask{ID: id, Headline: "Merge into main", Explanation: "passed its checks.", Who: asks.Operator, Expires: at.Add(24 * time.Hour), - OnExpiry: "nothing is merged; the ask is made again", Details: strings.Join(details, "\n"), - Options: []asks.Option{ - {ID: optionApprove, Label: "Approve", Does: "merges", Level: asks.Approve, Binds: bind(optionApprove)}, - {ID: "decline", Label: "Decline", Does: "nothing is merged", Level: asks.Approve, Binds: bind("decline")}}} - w := asks.Warrant{Ask: id, Asker: sdkdelivery.TellerModule, Outcome: asks.OutcomeChosen, Option: optionApprove, Label: "Approve", - Level: asks.Approve, By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "1", Verified: "user id"}, - At: at.Add(time.Minute), AskDigest: ask.Digest()} - return ask, w -} - -// withRouter hands findMergeWarrant these asks as the router's records, by asker. -func withRouter(t *testing.T, records map[string]map[string]struct { - ask asks.Ask - w asks.Warrant -}) { - t.Helper() - wasIDs, wasAsk := routerAskIDs, routerAskOf - closedRecords.Range(func(k, _ any) bool { closedRecords.Delete(k); return true }) - routerAskIDs = func(_ context.Context, _ catalogueReader, asker string) ([]string, error) { - var ids []string - for id := range records[asker] { - ids = append(ids, id) - } - return ids, nil - } - routerAskOf = func(_ context.Context, _ catalogueReader, asker, id string) (*asks.Ask, *asks.Warrant, error) { - r, ok := records[asker][id] - if !ok { - return nil, nil, nil - } - return &r.ask, &r.w, nil - } - t.Cleanup(func() { - routerAskIDs, routerAskOf = wasIDs, wasAsk - closedRecords.Range(func(k, _ any) bool { closedRecords.Delete(k); return true }) - }) -} - -type aRouterRecord = struct { - ask asks.Ask - w asks.Warrant -} - -// The teller's asks, as the router held them on 2026-10-20: the id is the router's own key, rebuilt here from the group and the -// head the gate composed it for (mesh-controller#405 at 8a330afd, mesh-catalog#381 at 1c702422, mesh-host#124 at 52600ec7). It names -// neither the pull request nor the head, so the warrant is found by the heads the Approve is bound to. -func TestATellersWarrantIsFoundByTheHeadItsApproveIsBoundTo(t *testing.T) { - real := []struct { - group, key, owner, repo string - number int - head string - }{ - {"feat/r4b-sdk-users", "merge-feat_r4b_sdk_users-ba59b03220f8-989afe0a3bcf", "novox", "mesh-controller", 405, "8a330afd44ba852873339cf1210e90989dca46e1"}, - {"fix/1351-remove-todays-delivery-module", "merge-fix_1351_remove_todays_delivery_module-64a353-d93b97b9697b", "novox", "mesh-catalog", 381, "1c7024224dd77c5b603299378883e0993b2e2fdd"}, - {"feat/r4b-sdk-users", "merge-feat_r4b_sdk_users-6a59ba48da14-424e053ebc9c", "novox", "mesh-host", 124, "52600ec7bd61ee271a5cc7f4136bcf0a2a8f00fc"}, - } - teller := map[string]aRouterRecord{} - for _, r := range real { - ask, w := aTellersAsk(t, r.group, sdkdelivery.Head{Repository: r.owner + "/" + r.repo, Number: r.number, Head: r.head}) - if ask.ID != r.key { - t.Fatalf("the id of %s#%d is %q, and the router held it as %q: the fixture is not the router's", r.repo, r.number, ask.ID, r.key) - } - teller[ask.ID] = aRouterRecord{ask, w} - } - withRouter(t, map[string]map[string]aRouterRecord{sdkdelivery.TellerModule: teller}) - for _, r := range real { - why, found, reason, err := findMergeWarrant(t.Context(), nil, r.owner, r.repo, r.number, r.head) - if err != nil || !found || !strings.Contains(why, r.key) || !strings.Contains(why, "bound to the head of this pull request") { - t.Errorf("%s#%d: the warrant was not found: %q %v %q %v", r.repo, r.number, why, found, reason, err) - } - // Another head of the same pull request is another merge: the Approve was not given for it. - if _, found, _, _ := findMergeWarrant(t.Context(), nil, r.owner, r.repo, r.number, "ffffffffffffffffffffffffffffffffffffffff"); found { - t.Errorf("%s#%d: another head found the warrant of this one", r.repo, r.number) - } - } - // One pull request's Approve is none for another repository's merge at the same head digits. - if _, found, _, _ := findMergeWarrant(t.Context(), nil, "novox", "mesh-tools", 405, real[0].head); found { - t.Error("a merge of another repository found mesh-controller#405's warrant") - } -} - -// A group's ask binds several heads, and the router's record carries only twelve digits of the other members': the pull request is -// found in the ask as it was published, which the warrant's digest covers. -func TestAGroupsWarrantIsFoundByTheAsksWordsForThePullRequestAndItsHead(t *testing.T) { - a := sdkdelivery.Head{Repository: "novox/mesh-host", Number: 124, Head: "52600ec7bd61ee271a5cc7f4136bcf0a2a8f00fc"} - b := sdkdelivery.Head{Repository: "novox/mesh-controller", Number: 405, Head: "8a330afd44ba852873339cf1210e90989dca46e1"} - ask, w := aTellersAsk(t, "feat/r4b-sdk-users", a, b) - withRouter(t, map[string]map[string]aRouterRecord{sdkdelivery.TellerModule: {ask.ID: {ask, w}}}) - for _, h := range []sdkdelivery.Head{a, b} { - owner, repo, _ := strings.Cut(h.Repository, "/") - why, found, reason, err := findMergeWarrant(t.Context(), nil, owner, repo, h.Number, h.Head) - if err != nil || !found || !strings.Contains(why, "a group's ask naming this pull request at this head") { - t.Errorf("%s#%d: not found in the group's ask: %q %v %q %v", repo, h.Number, why, found, reason, err) - } - } - if _, found, _, _ := findMergeWarrant(t.Context(), nil, "novox", "mesh-catalog", 381, a.Head); found { - t.Error("a pull request the group's ask does not name found its warrant") - } - // A Decline is no warrant. - declined := w - declined.Option, declined.Label = "decline", "Decline" - withRouter(t, map[string]map[string]aRouterRecord{sdkdelivery.TellerModule: {ask.ID: {ask, declined}}}) - if _, found, _, _ := findMergeWarrant(t.Context(), nil, "novox", "mesh-host", 124, a.Head); found { - t.Error("a Decline found a warrant") - } -} - -// A trunk shorter than a page, or with none, is read without a fault: the pages end when the forge's do. -func TestAShortTrunkIsReadWholeAndAnEmptyOneIsNoMerges(t *testing.T) { - was := trunkCommitsPage - t.Cleanup(func() { trunkCommitsPage = was }) - trunkCommitsPage = func(_ context.Context, _, _, _ string, page int) ([]trunkCommit, error) { - if page > 1 { - return nil, nil - } - return []trunkCommit{{Commit: "m2", SecondParent: "h2", Subject: "Merge pull request 'two' (#2) from fix/b into main"}, - {Commit: "p"}, {Commit: "m1", SecondParent: "h1", Subject: "Merge pull request 'one' (#1) from fix/a into main"}}, nil - } - got, err := readTrunkMerges(t.Context(), "novox", "x", "main") - if err != nil || len(got) != 2 || got[0].Commit != "m1" || got[1].Commit != "m2" { - t.Fatalf("a short trunk read as %+v %v", got, err) - } - trunkCommitsPage = func(context.Context, string, string, string, int) ([]trunkCommit, error) { return nil, nil } - if got, err := readTrunkMerges(t.Context(), "novox", "x", "main"); err != nil || len(got) != 0 { - t.Fatalf("an empty trunk read as %+v %v", got, err) - } -} - -// ADR 0253: what the operator reads of a merge not heard is plain: no hash, no identifier, nothing to do "by hand". -func TestTheConditionOfAMergeNotHeardIsInPlainWords(t *testing.T) { - keeper, _ := withConditionsInMemory(t) - tm := trunkMerge{Commit: "c0880c4d1f2e3a4b5c6d7e8f9a0b1c2d3e4f5a6b", Head: "b9104454f9a5e0c4d3b2a1f0e9d8c7b6a5948372"} - (following{}).raiseMergeNotHeard(t.Context(), "novox/mesh-catalog", "main", tm, link.SourceMoved{Number: 378}, - "the router holds no chosen merge ask of mesh-delivery-teller or mesh-delivery that covers the head b9104454f9a5") - open, err := keeper.Open(t.Context()) - if err != nil || len(open) != 1 { - t.Fatalf("opened %d conditions, %v", len(open), err) - } - c := open[0] - if why, ok := conditions.PlainWords(conditions.Words{Headline: c.Headline, Explanation: c.Explanation, Needs: c.Needs, - Resolved: c.Resolved}); !ok { - t.Fatalf("the condition's words are not plain: %s\n%q\n%q\n%q", why, c.Headline, c.Explanation, c.Needs) - } -} - -// The trunk's history lists the merges made inside a pull request's branch too. On novox (2026-10-20) 15 of the 15 non-pull-request -// "gaps" of mesh-catalog and mesh-controller were such merges; only a pull request's merge into this trunk is a merge to catch up. -func TestAMergeMadeInsideABranchIsNoMergeOfTheTrunk(t *testing.T) { - was := trunkCommitsPage - t.Cleanup(func() { trunkCommitsPage = was }) - trunkCommitsPage = func(_ context.Context, _, _, _ string, page int) ([]trunkCommit, error) { - if page > 1 { - return nil, nil - } - return []trunkCommit{ - {Commit: "c0880c4d", SecondParent: "b9104454", Subject: "Merge pull request 'Read the grant file' (#378) from fix/1332 into main"}, - {Commit: "6394a62a", SecondParent: "x1", Subject: "Merge remote-tracking branch 'origin/main' into fix/1081-withdraw"}, - {Commit: "896f1d28", SecondParent: "x2", Subject: "Merge feat/agents-write-git-ssh into feat/agents-write-git-ssh-f (issue 955 F)"}, - {Commit: "4d03755c", SecondParent: "x3", Subject: "Bring the branch up to main after #354 (issue 1047)"}, - {Commit: "other", SecondParent: "x4", Subject: "Merge pull request 'Elsewhere' (#5) from fix/y into nox-mesh"}, - }, nil - } - got, err := readTrunkMerges(t.Context(), "novox", "mesh-catalog", "main") - if err != nil || len(got) != 1 || got[0].Commit != "c0880c4d" { - t.Fatalf("read %+v %v: only keycloak's pull request merge into main is a merge to catch up", got, err) - } -} - -// novox/hq ADR 0463 §2: the record of merges begins at the earliest merge the controller holds on any trunk. A merge before it is -// history on every trunk; a trunk with no merge kept of its own still has the merges made since as gaps (mesh-delivery #1 on novox). -func TestOnlyTheMergesSinceTheRecordBeganAreGapsOnEveryTrunk(t *testing.T) { - first := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC) // the first merge it heard, on some trunk of the mesh - trunk := func(prefix string) []trunkMerge { - return []trunkMerge{ - {Commit: prefix + "-old-1", At: first.Add(-30 * 24 * time.Hour)}, - {Commit: prefix + "-old-2", At: first.Add(-2 * time.Hour)}, - {Commit: prefix + "-within-grace", At: first.Add(-mergeGrace / 2)}, - {Commit: prefix + "-since", At: first.Add(24 * time.Hour)}, - } - } - // Two trunks, one of which the controller has kept no merge of: both read by the one starting point. - for _, prefix := range []string{"kept-trunk", "trunk-with-no-kept-merge"} { - var commits []string - for _, m := range sinceTheRecord(trunk(prefix), true, first) { - commits = append(commits, m.Commit) - } - if want := prefix + "-within-grace " + prefix + "-since"; strings.Join(commits, " ") != want { - t.Errorf("%s: the gaps are %v, want %q: earlier ones are history on every trunk, later ones are gaps", prefix, commits, want) - } - } - if none := sinceTheRecord(trunk("x"), false, time.Time{}); len(none) != 0 { - t.Errorf("with no merge kept anywhere there are %d gap(s), but no record to have one in", len(none)) - } -} diff --git a/cmd/mesh-controller/chain_integration_test.go b/cmd/mesh-controller/chain_integration_test.go deleted file mode 100644 index 57a72b45..00000000 --- a/cmd/mesh-controller/chain_integration_test.go +++ /dev/null @@ -1,232 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "fmt" - "slices" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/chain" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" -) - -// **Acceptance 10 of design 47: one read of an assignment names the commit, the build and the rollout that put it there, -// and one read of the rollout lists the assignments it changed** (novox/hq ADR 0408, issue 1095). It enters by the live -// trigger — a merge heard, its batch cut into a plan when the window closed, its build answered through the build seat's -// path, the plan advanced — and writes no key and no row by hand. Only the machine's end of the send is the test's: the send -// is recorded as the controller records one it made, and the gate's facts are the store's. -func TestOneReadOfAnAssignmentNamesItsCommitBuildAndRolloutAndOneReadOfTheRolloutListsItsAssignments(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asked := asksRecorded(t) - ctx := t.Context() - inv := open.inventory - if _, err := assign(ctx, open, "anchor", "notes"); err != nil { - t.Fatal(err) - } - - var sent [][]string - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - sent = append(sent, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - digest := fmt.Sprintf("d-%s-%d", node, len(sent)) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, - map[string]string{"notes": current["notes"].Commit}); err != nil { - return nil, err - } - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - wasGather := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return f, err - } - for _, r := range reports { - f.reports[r.Node] = r - } - return f, nil - } - t.Cleanup(func() { gatherGateFacts = wasGather }) - - // A merge of notes is heard, and its window closes: the batch is cut into the walk. - hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second)) - cutAt(t, open, t0.Add(time.Second+90*time.Second)) - ws, _ := walks(t, open) - if len(ws) != 1 { - t.Fatalf("%d walk(s) after the cut", len(ws)) - } - rollout := ws[0].ID - - // The walk asks its first tier (at the cut, or at its next step); the build seat answers, and the controller takes the - // outcome in as it does. - asking, err := inv.PlanByID(ctx, rollout) - if err != nil { - t.Fatal(err) - } - for try := 0; try < 6 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == "" || asking.Modules["notes"].State == "asked"); try++ { - advancePlans(ctx, open) - if asking, err = inv.PlanByID(ctx, rollout); err != nil { - t.Fatal(err) - } - } - buildID := asking.Modules["notes"].Build - if buildID == "" || len(*asked) != 1 || !strings.HasSuffix((*asked)[0], "#"+buildID) { - t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID) - } - - // The walk sends the tier to its first machine. - for try := 0; try < 3 && len(sent) == 0; try++ { - advancePlans(ctx, open) - } - if !slices.Equal(slices.Concat(sent...), []string{"anchor"}) { - t.Fatalf("sent %v", sent) - } - - // One read of the assignment. - on, err := inv.AssignmentsOn(ctx, "anchor") - a, ok := assignmentOf(on, "notes") - if err != nil || !ok { - t.Fatalf("the assignments of anchor: %+v %v", on, err) - } - // Beside it a module pulled in by the assignment, which no send of this walk put anywhere. - if wireguard, pulled := assignmentOf(on, "mesh-wireguard"); pulled && (wireguard.Build != "" || wireguard.Rollout != "") { - t.Fatalf("a module this rollout did not send names a build or a rollout: %+v", wireguard) - } - if a.Module != "notes" || a.Commit != "c-notes" || a.Build != buildID || a.Rollout != rollout || - a.Since.IsZero() || a.Assigned.IsZero() || !a.Stands() { - t.Fatalf("the assignment names %+v, not commit c-notes, build %s, rollout %s", a, buildID, rollout) - } - // One read of the rollout. - p, err := inv.PlanByID(ctx, rollout) - if err != nil { - t.Fatal(err) - } - wantChange := chain.Change{Node: "anchor", Module: "notes", After: chain.Ref{Build: buildID, Commit: "c-notes"}} - if !slices.Contains(p.Changed, wantChange) || !slices.Contains(p.Builds, buildID) || len(p.Steps) != 1 || - p.Steps[0].Node != "anchor" || p.Steps[0].Sent == nil { - t.Fatalf("the rollout lists changed %+v, builds %v, steps %+v; want %+v", p.Changed, p.Builds, p.Steps, wantChange) - } - // And one read of the build names the same commit. - built, found, err := inv.BuildByID(ctx, buildID) - if err != nil || !found || built.Commit != "c-notes" || built.Module != "notes" { - t.Fatalf("the build: %+v %v %v", built, found, err) - } - // The verbs say the links. - if lines := changedLines(p); len(lines) != 2 || !strings.Contains(lines[1], "anchor/notes") || - !strings.Contains(lines[1], buildID) { - t.Fatalf("`plans` says %v", lines) - } - if lines := runLines(on); len(lines) != 2 || !strings.Contains(lines[1], buildID) || !strings.Contains(lines[1], rollout) { - t.Fatalf("`node show` says %v", lines) - } -} - -// A machine sent again what it runs says nothing new, and a push by hand with no rollout fills what is missing and blanks -// nothing (issue 1095). -func TestASendOfWhatAMachineRunsAlreadyChangesNoLinkOfItsAssignment(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - register(t, open, catalogue.Manifest{Module: "notes", Version: "1"}) - if _, err := assign(ctx, open, "anchor", "notes"); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-1", Module: "notes", Commit: "c1", Repository: "novox/notes"}); err != nil { - t.Fatal(err) - } - node := nodeID(t, open, "anchor") - inRollout := inventory.WithRollout(ctx, "plan-1", map[string]inventory.RolloutModule{"notes": {Commit: "c1", Build: "build-1"}}) - if err := inv.RecordSent(inRollout, node, "d1", map[string]string{"notes": "c1"}); err != nil { - t.Fatal(err) - } - // Sent again by hand, with no rollout: the assignment still names the rollout that put the build there. - if err := inv.RecordSent(ctx, node, "d2", map[string]string{"notes": "c1"}); err != nil { - t.Fatal(err) - } - on, _ := inv.AssignmentsOn(ctx, "anchor") - if a, _ := assignmentOf(on, "notes"); a.Rollout != "plan-1" || a.Build != "build-1" || a.Commit != "c1" { - t.Fatalf("the assignment names %+v", on) - } - // A declaration sent by hand with no builds known makes what the machine runs unknown: a time on the key, no commit. - if err := inv.RecordSent(ctx, node, "d3", nil); err != nil { - t.Fatal(err) - } - if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "" || runOf(on, "notes").SentUnknown.IsZero() { - t.Fatalf("a send of unknown builds left %+v", on) - } - // A module sent a commit no build is kept of is named by its commit alone. - if err := inv.RecordSent(ctx, node, "d4", map[string]string{"notes": "c2"}); err != nil { - t.Fatal(err) - } - if on, _ = inv.AssignmentsOn(ctx, "anchor"); runOf(on, "notes").Commit != "c2" || runOf(on, "notes").Build != "" || - runOf(on, "notes").Rollout != "" { - t.Fatalf("the assignment after a push of c2 by hand names %+v", on) - } -} - -// assignmentOf is one module's assignment among those of a machine: assigning a module assigns what it requires beside it. -func assignmentOf(on []chain.Assignment, module string) (chain.Assignment, bool) { - for _, a := range on { - if a.Module == module { - return a, true - } - } - return chain.Assignment{}, false -} - -func runOf(on []chain.Assignment, module string) chain.Assignment { - a, _ := assignmentOf(on, module) - return a -} - -// **An assignment waiting for its build is one key of the assignments bucket** (ADR 0261, ADR 0408, issue 1094): entered -// through the `assign` verb for a module whose build is in flight, one read of `.` says state pending with the -// build it waits for, the build's outcome through the build seat's path makes the key stand with the wait carried as applied, -// and the table of pending assignments is read by nobody. No key is written by hand. -func TestAnAssignmentWaitingForItsBuildIsOneKeyOfTheAssignmentsBucket(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatalf("an assignment while its build runs was refused: %v", err) - } - waiting, found, err := inv.AssignmentValue(ctx, "laptop", "sensors") - if err != nil || !found || waiting.State != chain.StatePending || waiting.Stands() || !waiting.Assigned.IsZero() || - waiting.Pending == nil || waiting.Pending.State != chain.PendingWaiting || waiting.Pending.Build != "b-modules/sensors" || - !strings.HasPrefix(waiting.Pending.ID, "p-") { - t.Fatalf("one read of the key says %+v (found %v, %v)", waiting, found, err) - } - if got := assignedTo(t, open, "laptop"); slices.Contains(got, "sensors") { - t.Fatalf("a module only waiting is assigned: %v", got) - } - - if err := (builds{inv: inv, open: open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", "3da80a4b00aa", "sensors", "")); err != nil { - t.Fatal(err) - } - made, found, err := inv.AssignmentValue(ctx, "laptop", "sensors") - if err != nil || !found || !made.Stands() || made.State != "" || made.Assigned.IsZero() || made.Pending == nil || - made.Pending.State != chain.PendingApplied || made.Pending.ID != waiting.Pending.ID { - t.Fatalf("one read of the key after the build says %+v (found %v, %v)", made, found, err) - } - if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("the build registered sensors and it was not assigned") - } -} diff --git a/cmd/mesh-controller/chain_no_table_integration_test.go b/cmd/mesh-controller/chain_no_table_integration_test.go deleted file mode 100644 index aab6ca2d..00000000 --- a/cmd/mesh-controller/chain_no_table_integration_test.go +++ /dev/null @@ -1,77 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" -) - -// **Acceptance 12 of design 47: no table in the controller's store holds builds, rollouts or assignments** (novox/hq ADR -// 0408, issue 1111). A delivery is driven through the live trigger — a merge heard, its batch cut into a plan, the plan's -// build asked and answered, the assignment made — until a build, a rollout and an assignment exist; they are read from the -// chain, and then the store's own catalog is read: none of the tables of the chain is in it, nor the function that raised the generation from the assignment table; the table that stays, stays. -func TestNoTableOfTheControllersStoreHoldsBuildsRolloutsOrAssignments(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asked := asksRecorded(t) - ctx := t.Context() - inv := open.inventory - if _, err := assign(ctx, open, "anchor", "notes"); err != nil { - t.Fatal(err) - } - hear(t, open, catalogueMerge("c-notes", "notes", t0), t0.Add(time.Second)) - cutAt(t, open, t0.Add(91*time.Second)) - ws, _ := walks(t, open) - if len(ws) != 1 { - t.Fatalf("%d walk(s) after the cut", len(ws)) - } - rollout := ws[0].ID - asking, err := inv.PlanByID(ctx, rollout) - if err != nil { - t.Fatal(err) - } - for try := 0; try < 6 && (asking.Modules["notes"] == nil || asking.Modules["notes"].Build == "" || asking.Modules["notes"].State == "asked"); try++ { - advancePlans(ctx, open) - if asking, err = inv.PlanByID(ctx, rollout); err != nil { - t.Fatal(err) - } - } - buildID := asking.Modules["notes"].Build - if buildID == "" || len(*asked) != 1 { - t.Fatalf("the walk asked %v, and notes' build is %q", *asked, buildID) - } - - // What the live path made is in the chain. - if built, found, err := inv.BuildByID(ctx, buildID); err != nil || !found || built.Commit != "c-notes" { - t.Fatalf("the build is not read from the chain: %+v %v %v", built, found, err) - } - if p, err := inv.PlanByID(ctx, rollout); err != nil || p.ID != rollout { - t.Fatalf("the rollout is not read from the chain: %+v %v", p, err) - } - if on, err := inv.AssignmentsOn(ctx, "anchor"); err != nil { - t.Fatal(err) - } else if _, ok := assignmentOf(on, "notes"); !ok { - t.Fatalf("the assignment is not read from the chain: %+v", on) - } - - // And the catalog of the store holds none of them. - have := map[string]bool{} - for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) { - have[strings.Trim(row[0], `"`)] = true - } - for _, table := range []string{"build", "build_request", "release_plan", "batched_merge", "assignment", "pending_assignment", "chain_moved"} { - if have[table] { - t.Errorf("the table %s is in the controller's store", table) - } - } - for _, table := range []string{"build_gate", "assignment_generation"} { - if !have[table] { - t.Errorf("the table %s, which stays, is gone", table) - } - } - if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" { - t.Errorf("the function raise_assignment_generation is in the store: %v", rows) - } -} diff --git a/cmd/mesh-controller/changeplan.go b/cmd/mesh-controller/changeplan.go deleted file mode 100644 index da60adf4..00000000 --- a/cmd/mesh-controller/changeplan.go +++ /dev/null @@ -1,192 +0,0 @@ -package main - -import ( - "fmt" - "path" - "slices" - "sort" - "strings" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// The change plan (novox/hq ADR 0238): **one commit, one plan** — what a change does to the mesh, computed -// from its diffset (a repository, the branch it merges into, the commit at hand) by the planner, never by -// a mapping of its own. reachOfMerge answers what moves and what follows it; this adds where each module -// goes — the deploy plan, machine by machine in the build plan's order — and what is not an ordinary -// send. A pull request's check posts it with its verdict; the release a merge makes follows the same -// planner, so the two can be compared. - -// policyOf is a module's upgrade policy, as the controller holds it; false when it cannot be read. -type policyOf func(module string) (inventory.Upgrade, bool) - -// changePlanOf is the change plan of a reach: pure, so it is tested without a store. -func changePlanOf(repository, base, head string, r mergeReach, entries []inventory.Entry, policy policyOf) link.ChangePlan { - p := link.ChangePlan{Repository: repository, Base: base, Head: head, Moved: r.Moved(), Dependents: r.Dependents(), - New: r.Added, Unread: r.Unread, Tiers: r.Plan.Tiers} - byName := map[string]inventory.Entry{} - for _, e := range entries { - byName[e.Manifest.Module] = e - } - // The machine holding the bus, whose declaration carries every module's grants (novox/hq issue 490). - holder := "" - for _, e := range entries { - if e.Manifest.BusUsers != "" && e.Manifest.ClaimsSeat(catalogue.BrokerSeat) && len(e.On) > 0 { - holder = e.On[0] - } - } - var granting []string - machines := map[string]*link.MachinePlan{} - machine := func(name string) *link.MachinePlan { - if machines[name] == nil { - machines[name] = &link.MachinePlan{Machine: name} - } - return machines[name] - } - for _, tier := range r.Plan.Tiers { - for _, name := range tier { - e, held := byName[name] - if !held { - continue - } - u, known := policy(name) - waits := known && !u.RollOut - for _, on := range e.On { - if waits { - machine(on).Waits = append(machine(on).Waits, name) - } else { - machine(on).Receives = append(machine(on).Receives, name) - } - } - if !waits && holder != "" && !(len(e.On) == 1 && e.On[0] == holder) && len(e.On) > 0 && - !slices.Contains(granting, name) { - granting = append(granting, name) - } - switch { - case (name == "nats" || catalogue.ProvidesBus(e.Manifest)) && len(e.On) > 0: - // Said before the merge (novox/hq issue 336): a new bus build holds every send to the bus's machine - // until a person takes the step, so merging it is a promise to take it. - p.Steps = append(p.Steps, fmt.Sprintf("%s: %s is never sent by an ordinary send, and until %s runs, "+ - "nothing else is sent to %s either — unless the new build turns out the same as the one running there "+ - "(issue 280)", busUpgradeNeeded, name, busUpgradeVerb, strings.Join(e.On, ", "))) - case name == "nats" || catalogue.ProvidesBus(e.Manifest): - p.Steps = append(p.Steps, "a planned bus step: the bus is upgraded by `bus upgrade`, never by an ordinary send") - case waits && len(e.On) > 0 && (u.From == catalogue.FromModule || u.From == catalogue.FromData): - // Judged per machine by the walk, against the build each was last sent (novox/hq ADR 0377). - p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s) — unless on a machine "+ - "its move recreates none of its containers and moves none of its data, which the walk judges and then sends", - name, orNone(u.From))) - case waits && len(e.On) > 0: - p.Steps = append(p.Steps, fmt.Sprintf("%s waits for a person: its policy records (%s)", name, - orNone(u.From))) - } - if len(e.Manifest.Provides) > 0 && len(e.On) > 0 { - var offers []string - for _, o := range e.Manifest.Provides { - offers = append(offers, o.Name) - } - p.Steps = append(p.Steps, fmt.Sprintf("%s provides %s: its consumers are sent again after it", - name, strings.Join(offers, ", "))) - } - if e.Manifest.Data != nil && len(e.On) > 0 { - p.Steps = append(p.Steps, fmt.Sprintf("%s keeps data (ADR 0233): what it holds is backed up before it moves", name)) - } - } - } - if len(granting) > 0 { - // Said before the merge (novox/hq issue 490): what the walk does when the change alters the bus's - // user list. Whether it does is known only once the builds are registered, so it is said as a rule. - p.Steps = append(p.Steps, fmt.Sprintf("%s: if this changes what the bus's user list says (a new tool, "+ - "subject or user), %s is sent that list alone, every build there kept, before %s is judged on its first "+ - "machine", grantsStepWord, holder, strings.Join(granting, ", "))) - } - var names []string - for name := range machines { - names = append(names, name) - } - sort.Strings(names) - for _, name := range names { - p.Machines = append(p.Machines, *machines[name]) - } - p.Summary = summaryOf(p) - p.Class = changeClassOf(p, entries) - return p -} - -// classNone is the class of a change that moves no module: nothing for a walk to move (novox/hq ADR 0313 §2). -const classNone = "none" - -// changeClassOf is a change plan's class (novox/hq ADR 0313 §2): that of the slowest-budgeted module it moves, -// itself or as a dependent — core when one is on the controller's own path or holds the mesh's resolver, leaf -// otherwise — and none when it moves nothing. A module it adds a directory for counts by its own class: the -// catalogue names a module's directory by the module, so the directory's last element is its name. The graph holds -// no manifest of it yet, so a new module is core by its name alone (on the controller's own path), never by a -// resolver claim its manifest makes. -func changeClassOf(p link.ChangePlan, entries []inventory.Entry) string { - if len(p.Moved) == 0 && len(p.Dependents) == 0 && len(p.New) == 0 { - return classNone - } - modules := append(append([]string{}, p.Moved...), p.Dependents...) - for _, dir := range p.New { - if name := path.Base(strings.Trim(dir, "/")); name != "." && name != "/" && name != "" { - modules = append(modules, name) - } - } - return classOfModules(modules, entries) -} - -// grantsStepWord names the grants step (novox/hq issue 490): the bus's user list sent alone to the machine -// holding the bus ahead of a walk's gate. Not the bus step, which replaces the bus itself. -const grantsStepWord = "grants step" - -// busUpgradeNeeded is how a change plan says that merging it holds the bus's machine for a person's step. -const busUpgradeNeeded = "merging this needs a person's bus upgrade" - -// summaryOf is a change plan in one line: what it builds, where it goes, and whether the bus moves. -func summaryOf(p link.ChangePlan) string { - if len(p.Moved) == 0 && len(p.New) == 0 { - return "builds nothing: the change touches no module of the mesh's graph" - } - var parts []string - what := strings.Join(p.Moved, ", ") - if len(p.Dependents) > 0 { - what += fmt.Sprintf(" (+%d dependent(s))", len(p.Dependents)) - } - if len(p.New) > 0 { - if what != "" { - what += ", " - } - what += "new: " + strings.Join(p.New, ", ") - } - var to []string - for _, m := range p.Machines { - if len(m.Receives) > 0 { - to = append(to, m.Machine) - } - } - if len(to) > 0 { - parts = append(parts, "builds "+what+" → "+strings.Join(to, ", ")) - } else { - parts = append(parts, "builds "+what+", sent nowhere") - } - bus := "no bus step" - for _, s := range p.Steps { - switch { - case strings.HasPrefix(s, busUpgradeNeeded): - bus = busUpgradeNeeded - case strings.HasPrefix(s, "a planned bus step") && bus == "no bus step": - bus = "a bus step" - } - } - parts = append(parts, bus) - waiting := 0 - for _, m := range p.Machines { - waiting += len(m.Waits) - } - if waiting > 0 { - parts = append(parts, fmt.Sprintf("%d wait(s) for a person", waiting)) - } - return strings.Join(parts, "; ") -} diff --git a/cmd/mesh-controller/changeplan_test.go b/cmd/mesh-controller/changeplan_test.go deleted file mode 100644 index bb250865..00000000 --- a/cmd/mesh-controller/changeplan_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package main - -import ( - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// **One commit, one change plan** (novox/hq ADR 0238): the planner's reach, laid out machine by machine in -// the build plan's order, with what is not an ordinary send said — the bus step, a module that waits for a -// person, a provider whose consumers follow it. -func TestAChangePlanSaysWhatEachMachineReceives(t *testing.T) { - const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git" - entry := func(name, path string, on ...string) inventory.Entry { - e := fromRepo(name, catalogue_, path) - e.Source.BuiltFrom = "old" - e.On = on - return e - } - nats := entry("nats", "modules/nats", "anchor") - nats.Manifest.Provides = []catalogue.Offer{{Name: "mesh-bus"}} - gitea := entry("gitea", "modules/gitea", "anchor") - held := entry("photos", "modules/photos", "anchor", "laptop") - tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools") - tools.On = []string{"anchor", "laptop"} - entries := []inventory.Entry{nats, gitea, held, tools} - edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}} - policy := func(module string) (inventory.Upgrade, bool) { - if module == "photos" { - return inventory.Upgrade{RollOut: false, From: "a person"}, true - } - return inventory.Upgrade{RollOut: true}, true - } - plan := func(paths ...string) link.ChangePlan { - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths} - return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, edges), entries, policy) - } - - p := plan("modules/gitea/index.ts") - if p.Summary != "builds gitea → anchor; no bus step" { - t.Errorf("one module's change reads %q", p.Summary) - } - - p = plan("modules/nats/Dockerfile", "modules/photos/x.js") - if !strings.Contains(p.Summary, "needs a person's bus upgrade") || !strings.Contains(p.Summary, "2 wait(s) for a person") || - !strings.Contains(p.Summary, "(+1 dependent(s))") { - t.Errorf("the bus and a held module read %q", p.Summary) - } - got := map[string]string{} - for _, m := range p.Machines { - got[m.Machine] = strings.Join(m.Receives, ",") + "|" + strings.Join(m.Waits, ",") - } - // The bus first, what stands on it after: the build plan's order, per machine. - if got["anchor"] != "nats,node-tools|photos" || got["laptop"] != "node-tools|photos" { - t.Errorf("the deploy plan reads %v", got) - } - text := strings.Join(p.Steps, "\n") - // Said before the merge (novox/hq issue 336): merging it holds every send to the bus's machine until a - // person runs the bus's step, and the verb that does. - for _, want := range []string{"merging this needs a person's bus upgrade", "nothing else is sent to anchor", - "mesh_call mesh-controller.bus", "the same as the one running there", "photos waits for a person", - "nats provides mesh-bus"} { - if !strings.Contains(text, want) { - t.Errorf("the steps do not say %q:\n%s", want, text) - } - } - - p = plan("merge-check.sh") - if !strings.HasPrefix(p.Summary, "builds nothing") || len(p.Machines) != 0 || strings.Join(p.Unread, ",") != "merge-check.sh" { - t.Errorf("a root file's plan reads %+v", p) - } -} diff --git a/cmd/mesh-controller/checks.go b/cmd/mesh-controller/checks.go index f3d84c34..8b1d093e 100644 --- a/cmd/mesh-controller/checks.go +++ b/cmd/mesh-controller/checks.go @@ -38,10 +38,8 @@ type checkScope struct { New []string // Manifests are the moved modules' and the new ones' manifests in the change's tree. Manifests []string - // Width is how many modules a merge would build, in how many tiers; Unread the changed files no - // module's build reads. - Width, Tiers int - Unread []string + // Unread the changed files no module's build reads. + Unread []string // Mesh says the repository is the mesh's: modules are built from it on some branch, its owner is the // core's, or the change adds a module to it. Mesh bool @@ -62,8 +60,7 @@ func pullScope(p link.PullUpdated, entries []inventory.Entry, read map[string][] Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs, ModuleDirsSaid: p.ModuleDirsSaid} r := reachOfMerge(m, entries, read, edges) - s := checkScope{Modules: r.Moved(), Dependents: r.Dependents(), New: r.Added, Unread: r.Unread, - Width: len(r.Plan.Modules), Tiers: len(r.Plan.Tiers), Reach: r} + s := checkScope{Modules: r.Moved(), Dependents: r.Dependents(), New: r.Added, Unread: r.Unread, Reach: r} for _, e := range append(append([]inventory.Entry{}, r.Touched...), r.Deleted...) { s.Manifests = append(s.Manifests, path.Join(strings.Trim(e.Source.Path, "/"), moduleManifestFile)) } diff --git a/cmd/mesh-controller/checks_test.go b/cmd/mesh-controller/checks_test.go index 96d134cb..f35ea93f 100644 --- a/cmd/mesh-controller/checks_test.go +++ b/cmd/mesh-controller/checks_test.go @@ -93,60 +93,3 @@ func TestAPullRequestTouchesWhatPackagesItsRepository(t *testing.T) { t.Fatalf("a change to what node-tools packages touched %v (manifests %v)", s.Modules, s.Manifests) } } - -// **The check asks the planner, never a mapping of its own** (novox/hq ADR 0238): what a pull request -// reaches is reachOfMerge's answer — the same that plans a merge — so a file at the root touches no -// module in both, and what stands on a touched module is named as its dependents in both. -func TestAPullRequestReachesWhatAMergeOfItWouldPlan(t *testing.T) { - const catalogue = "http://forge.internal:20000/novox/mesh-catalog.git" - nats := fromRepo("nats", catalogue, "modules/nats") - nats.Source.BuiltFrom = "old" - gitea := fromRepo("gitea", catalogue, "modules/gitea") - gitea.Source.BuiltFrom = "old" - tools := fromRepo("node-tools", "http://forge.internal:20000/novox/mesh-tools.git", "node-tools") - entries := []inventory.Entry{nats, gitea, tools} - // node-tools stands on the bus's image; a code edge, so a plan takes it along. - edges := []inventory.Edge{{From: "node-tools", To: "nats", Kind: inventory.EdgeStandsOn}} - read := map[string][]inventory.ReadRepository{} - - for _, c := range []struct { - what string - paths []string - moved, dependents string - width int - unread string - }{ - {"a file at the root, read by no build", []string{"merge-check.sh"}, "", "", 0, "merge-check.sh"}, - {"the bus's own directory", []string{"modules/nats/Dockerfile"}, "nats", "node-tools", 2, ""}, - {"both, and a README", []string{"modules/gitea/index.ts", "modules/nats/x", "README.md"}, "gitea,nats", "node-tools", 3, "README.md"}, - } { - p := link.PullUpdated{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths} - s := pullScope(p, entries, read, edges) - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: c.paths} - r := reachOfMerge(m, entries, read, edges) - if got := strings.Join(s.Modules, ","); got != c.moved || got != strings.Join(r.Moved(), ",") { - t.Errorf("%s: the check reaches %q, the planner %v, wanted %q", c.what, got, r.Moved(), c.moved) - } - if got := strings.Join(s.Dependents, ","); got != c.dependents { - t.Errorf("%s: dependents %q, wanted %q", c.what, got, c.dependents) - } - if s.Width != c.width || s.Width != len(r.Plan.Modules) { - t.Errorf("%s: a merge would build %d, the planner says %d, wanted %d", c.what, s.Width, len(r.Plan.Modules), c.width) - } - if got := strings.Join(s.Unread, ","); got != c.unread { - t.Errorf("%s: unread %q, wanted %q", c.what, got, c.unread) - } - } - - // A repository whose build another module's recipe packages: that module is reached through the - // planner's `read`, and what stands on it after it. - read["gitea"] = []inventory.ReadRepository{{Repository: "http://forge.internal:20000/novox/mesh-sdk.git"}} - s := pullScope(link.PullUpdated{Owner: "novox", Repo: "mesh-sdk", Base: "main", Commit: "head", - Paths: []string{"src/index.ts"}}, entries, read, edges) - if strings.Join(s.Modules, ",") != "gitea" || len(s.Manifests) != 0 { - t.Fatalf("a change to the source gitea packages reaches %v (manifests %v)", s.Modules, s.Manifests) - } -} - -// A check stopped by its pull request's key moving on says no verdict (issue 974, review 2): an error said for it would name the -// live head, and the newer request is the one that judges. A check that failed for any other reason still says its error. diff --git a/cmd/mesh-controller/compose_input.go b/cmd/mesh-controller/compose_input.go index 7e176e7b..217ad7ae 100644 --- a/cmd/mesh-controller/compose_input.go +++ b/cmd/mesh-controller/compose_input.go @@ -267,19 +267,6 @@ func readNode(ctx context.Context, open *stores, n inventory.Node, place invento out.Runs[module] = compose.Run{Commit: ref.Commit, Build: ref.Build} } } - // A recorded module kept at the build this machine runs, in the context a send composes under (recorded_kept.go): its - // manifest and the commit, applied to this node's own resolution only. - // What cannot be kept refuses this machine alone (keepRecorded's refusal, as planFor returned it). - kept, err := recordedKept(ctx, open, n.Name) - if err != nil { - *refused = err - } else if len(kept) > 0 { - out.KeptBuilds = kept - if out.Kept, err = keptManifests(ctx, open, n.Name, kept); err != nil { - *refused = err - out.KeptBuilds, out.Kept = nil, nil - } - } stated, has, err := inv.HealthOf(ctx, n.Name) if err != nil { return out, err @@ -291,18 +278,6 @@ func readNode(ctx context.Context, open *stores, n inventory.Node, place invento return out, nil } -// keptManifests is the manifest of each module kept at the build a machine runs: keepRecorded's reads, and its refusals in its -// words, without putting any of it into a shelf (that is the node's own Kept, applied by the composition). -func keptManifests(ctx context.Context, open *stores, node string, kept map[string]string) (map[string]catalogue.Manifest, error) { - // applyKept puts each manifest into the shelf it is given and refuses a build that cannot be read; the shelf here is a - // scratch one, whose entries are the kept manifests. - scratch := map[string]catalogue.Manifest{} - if err := applyKept(ctx, open, node, kept, scratch); err != nil { - return nil, err - } - return scratch, nil -} - func readTunnels(ctx context.Context, inv *inventory.Inventory) (map[string]compose.FoundTunnel, error) { found, err := inv.Tunnels(ctx) if err != nil { diff --git a/cmd/mesh-controller/compose_input_integration_test.go b/cmd/mesh-controller/compose_input_integration_test.go index 6e4f190c..d26ac56b 100644 --- a/cmd/mesh-controller/compose_input_integration_test.go +++ b/cmd/mesh-controller/compose_input_integration_test.go @@ -6,11 +6,9 @@ import ( "reflect" "strings" "testing" - "time" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/compose" - "github.com/novox/mesh-controller/internal/link" ) // 1382: the adapter's Input names, for every node, what the inventory answers for it, each compared with the read named @@ -132,69 +130,6 @@ func nodePorts(in compose.Input, name string) map[string]map[int]int { return n.Ports } -// 1382/AC2: a node keeping a recorded module at the build it runs has that module in Kept and KeptBuilds, and the Input's -// Catalogue is the current shelf, not the kept manifest (R4a review round 1, finding 1). -func TestANodeKeepingARecordedBuildHasItInKeptAndTheCatalogueStaysCurrent(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - start := time.Now().Add(-time.Hour) - image := "registry.invalid:5000/postgres/server@sha256:" + strings.Repeat("a", 64) - for _, b := range []link.BuildResult{ - aContainerBuild(t, "postgres", "c1111111", catalogue.PolicyRecord, start, map[string][2]string{"server": {image, ""}}), - } { - if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil { - t.Fatal(err) - } - } - if _, err := inv.Assign(ctx, "anchor", "postgres"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"postgres": "c1111111"}); err != nil { - t.Fatal(err) - } - if _, _, err := takeIn(ctx, inv, aContainerBuild(t, "postgres", "c2222222", catalogue.PolicyRecord, start.Add(time.Minute), - map[string][2]string{"server": {image, "runtime"}})); err != nil { - t.Fatal(err) - } - - // A send that is not a person's push keeps the build the machine runs. - in, err := readComposeInput(keepingRecorded(ctx), open) - if err != nil { - t.Fatal(err) - } - anchor, _ := nodeOf(in.Input, "anchor") - if anchor.KeptBuilds["postgres"] != "c1111111" { - t.Fatalf("anchor keeps %v, want postgres at c1111111", anchor.KeptBuilds) - } - kept, found := anchor.Kept["postgres"] - if !found { - t.Fatalf("the kept manifest of postgres is not in anchor's Kept: %v", anchor.Kept) - } - current, err := inv.Catalogue(ctx) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(in.Catalogue["postgres"], current["postgres"]) { - t.Error("the Input's Catalogue is not the current shelf") - } - if reflect.DeepEqual(kept, current["postgres"]) { - t.Error("the kept manifest is the current one: nothing is kept") - } - if laptop, _ := nodeOf(in.Input, "laptop"); len(laptop.Kept) != 0 || len(laptop.KeptBuilds) != 0 { - t.Errorf("laptop keeps what only anchor runs: %v", laptop.KeptBuilds) - } - - // A person's push keeps nothing. - pushed, err := readComposeInput(ctx, open) - if err != nil { - t.Fatal(err) - } - if a, _ := nodeOf(pushed.Input, "anchor"); len(a.Kept) != 0 || len(a.KeptBuilds) != 0 { - t.Errorf("a person's push keeps %v", a.KeptBuilds) - } -} - // 1382/AC3: the seats on record, the tunnels, the adopted tunnel, its carried peers and the nodes joining equal the // inventory's answers; the range and the suffix are read from the environment once, here. func TestTheAdaptersInputCarriesTheNetworkAsTheInventoryHoldsIt(t *testing.T) { diff --git a/cmd/mesh-controller/compose_sdk.go b/cmd/mesh-controller/compose_sdk.go index 0a8f6b5e..82edab7d 100644 --- a/cmd/mesh-controller/compose_sdk.go +++ b/cmd/mesh-controller/compose_sdk.go @@ -48,15 +48,7 @@ func freshInput(ctx context.Context) context.Context { // keptSignature is what the Input read depends on in ctx: which recorded modules each machine keeps at the build it runs // (recorded_kept.go), so an Input read for a person's push is never taken for one read for a gated send. func keptSignature(ctx context.Context) string { - skip, on := keptExcept(ctx) - var names []string - for m := range skip { - names = append(names, m) - } - sort.Strings(names) - every, _ := ctx.Value(keepEveryBuildKey{}).(string) - settings, _ := ctx.Value(keepForSettingsKey{}).(string) - return fmt.Sprintf("kept=%v;except=%v;every=%s;settings=%s", on, names, every, settings) + return "" } // inputFor is the Input for ctx's operation. diff --git a/cmd/mesh-controller/compose_sdk_test.go b/cmd/mesh-controller/compose_sdk_test.go index 8d6a3cb8..f20caa22 100644 --- a/cmd/mesh-controller/compose_sdk_test.go +++ b/cmd/mesh-controller/compose_sdk_test.go @@ -1,7 +1,6 @@ package main import ( - "context" "reflect" "strings" "testing" @@ -86,26 +85,3 @@ func TestADeclarationThatStandsInForAValueIsStillForeseen(t *testing.T) { } } } - -// 1386: an Input read for a person's push is never the one read for a send that keeps recorded builds, and the cache is -// emptied by what a send made. -func TestAnInputIsCachedPerWhatItKeepsAndForgottenAfterASend(t *testing.T) { - ctx := withInputCache(context.Background()) - plain := keptSignature(ctx) - kept := keptSignature(keepingRecorded(ctx)) - bus := keptSignature(keepingRecorded(ctx, "nats")) - every := keptSignature(keepingEveryBuild(ctx, "anchor")) - for name, sig := range map[string]string{"a push": plain, "a kept send": kept, "the bus step": bus, "the grants step": every} { - for other, o := range map[string]string{"a push": plain, "a kept send": kept, "the bus step": bus, "the grants step": every} { - if name != other && sig == o { - t.Errorf("%s and %s read the same Input", name, other) - } - } - } - cache := ctx.Value(inputCacheKey{}).(*inputCache) - cache.by[plain] = &meshInput{} - forgetInput(ctx) - if len(cache.by) != 0 { - t.Fatal("what a send made left the Input it read in the cache") - } -} diff --git a/cmd/mesh-controller/conditions_brief_test.go b/cmd/mesh-controller/conditions_brief_test.go deleted file mode 100644 index 135c6336..00000000 --- a/cmd/mesh-controller/conditions_brief_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" -) - -// stalledConditions are n open conditions like those of 2026-10-08: one per delivery held past its bound, -// each keeping all its evidence. -func stalledConditions(n int) []conditions.Condition { - at := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC) - var out []conditions.Condition - for i := 0; i < n; i++ { - id := fmt.Sprintf("d-%012x", 0x51a11ed+i) - lines := stalledObservations([]stalledLine{{ID: id, State: "checking", For: "13h2m", Bound: "45m", - H2: "close: recheck", Says: "the check was asked and nothing has answered"}}) - o := lines[0] - c := conditions.Condition{Key: o.Key(), Kind: o.Kind, Subject: conditions.Subject{Scope: o.Scope, ID: o.ID}, - Severity: o.Severity, Summary: o.Summary, Source: probeDeliveriesID, Raised: at, LastObserved: at, - Observations: 160, Count: 1, Resolver: conditions.ResolverSelf} - for e := 0; e < conditions.KeptEvidence; e++ { - c.Evidence = append(c.Evidence, conditions.Evidence{At: at.Add(time.Duration(e) * 5 * time.Minute), - Said: "checking for " + (13*time.Hour + time.Duration(e)*time.Minute).String()}) - } - out = append(out, c) - } - return out -} - -// novox/hq issue 314: a list of hundreds of open conditions carries each one's newest evidence, with -// how much more it keeps and where to read it, and the verb's answer says it once — as data, not again -// as the text the command printed. On 2026-10-08, 689 stalled deliveries made that answer more than the -// bus carries in one message; now it is less than half of one. -func TestAListOfConditionsCarriesEachInBrief(t *testing.T) { - list := stalledConditions(689) - printed, _ := json.MarshalIndent(map[string]any{"conditions": inBrief(list), "open": len(list), - "counted": counted(list)}, "", " ") - var parsed any - if err := json.Unmarshal(printed, &parsed); err != nil { - t.Fatal(err) - } - wire, _ := json.Marshal(map[string]any{"result": verbAnswer{OK: true, Answer: parsed, - Output: "its answer, as data, is `answer`\n"}}) - before, _ := json.MarshalIndent(map[string]any{"conditions": list}, "", " ") - t.Logf("689 stalled deliveries: %d bytes on the wire, where the whole list printed was %d and was sent twice", - len(wire), len(before)) - if len(wire) >= 512<<10 { - t.Fatalf("the answer of 689 conditions is %d bytes, not under half of one message of the bus", len(wire)) - } - var doc struct { - Conditions []conditionInBrief `json:"conditions"` - Counted map[string]int `json:"counted"` - } - if err := json.Unmarshal(printed, &doc); err != nil { - t.Fatal(err) - } - c := doc.Conditions[0] - if len(doc.Conditions) != 689 || len(c.Evidence) != 1 || c.EvidenceKept != conditions.KeptEvidence || - c.Key != list[0].Key || c.Summary != list[0].Summary || !strings.Contains(c.More, "key="+c.Key) { - t.Fatalf("a condition in brief reads %+v", c) - } - if doc.Counted["D14 stalled"] != 689 { - t.Fatalf("counted %v", doc.Counted) - } - if one := inBrief(list[:1])[0]; len(list[0].Evidence) != conditions.KeptEvidence || one.Evidence[0] != list[0].Evidence[0] { - t.Fatal("the brief changed the condition it was made from, or kept other than the newest") - } -} - -// The self-check's verdict carries at most overviewFindings of one probe's findings, with their count and -// how to read them all; `doctor probe=` gives that probe whole (novox/hq issue 314). diff --git a/cmd/mesh-controller/core_box.go b/cmd/mesh-controller/core_box.go index 33c99192..a86d1f68 100644 --- a/cmd/mesh-controller/core_box.go +++ b/cmd/mesh-controller/core_box.go @@ -246,7 +246,7 @@ const kindVaultSilent = "vault-silent" // vaultSilentObservation is that, as a condition: `machine..vault-silent`. func vaultSilentObservation(node string, b coreBoxes) conditions.Observation { o := conditions.Observation{Scope: conditions.ScopeMachine, ID: node, Kind: kindVaultSilent, Token: kindVaultSilent, - Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource} + Machine: node, Severity: conditions.Warning, Source: "sends"} var said []string if len(b.notPlaced) > 0 { o.Severity = conditions.Urgent diff --git a/cmd/mesh-controller/core_downgrade.go b/cmd/mesh-controller/core_downgrade.go deleted file mode 100644 index cae82843..00000000 --- a/cmd/mesh-controller/core_downgrade.go +++ /dev/null @@ -1,215 +0,0 @@ -package main - -import ( - "context" - "fmt" - "sort" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" -) - -// **No send but a person's `push ` and a gate's own put-back moves the mesh's own core to a build older -// than the one its machine was last sent** (novox/hq issue 201). Every send made through sendToEach keeps it — a -// plan's, a release's, a healer's, a rotation's, and the bus step too, which is a person's act but composes under -// sendKeeps like the rest. On 2026-10-02 two merges a minute apart made two plans; -// the controller's plan rolled the new controller, which widened its seat's row, and the node-engine's plan then -// sent the control node the controller build before it. The older binary read the row it could not serve and the -// mesh lost its voice for ten minutes — the thing that would have corrected it was the thing that was down. ADR -// 0185 took the outage out of it (a controller behind its row serves what it can); the race is closed here, -// without needing its trigger: whatever made the mesh's record older, the send keeps the build. -// -// **Kept, not refused.** The send composes the controller or the node-engine at the build the machine was last -// sent, as a recorded module is kept (ADR 0245), and carries everything else: no other module, secret, healer or -// walk waits on it, and nothing is retried. The kept move is no move for a gate either: ungatedIn and gatedSend -// leave it out (withoutKept), so a gate judges the other moves of its send. A plan whose own build is the older one finds its gate superseded by -// the build kept there (issue 352) and ends. The mesh's record stays older, and a warning condition says so until -// a send carries the record — a person's `push ` accepts the older build — or the record moves forward: a -// newer merge, or `rebuild mesh-controller`. -// -// Older is by the builds' registration order — when each was asked (novox/hq issue 219) — never by the commits' -// dates. So a retry or a rebuild of an old commit, asked anew, counts as newer: it is registered as the module's -// current build by the same order, and the guard does not stop that move. A build that failed its gate is never -// ahead: the machine was sent it and the record was put back before it (gateFailed), and if the put-back's own -// send was refused, the next send completes it rather than keeping the failed build there. A gate's own put-back -// carries its modules back whatever the order (sendScope.puttingBack); the controller's is held to the store's -// schema already (issue 352). - -// kindCoreDowngrade is a core module kept at the build its machine was last sent, over an older record. -const kindCoreDowngrade = "core-downgrade-kept" - -// coreDowngradeSource is the condition's source: the sends, which raise and clear it. -const coreDowngradeSource = "sends" - -// downgradeGuarded is the core components a send may not move backwards: the controller, which reads the -// store and its seat's row, and the node-engine, which reads what the controller declares. -var downgradeGuarded = []string{lease.ComponentController, lease.ComponentEngine} - -// askedBuild is what the order needs of one build of a module: when it was asked, and whether it failed its gate. -type askedBuild struct { - asked time.Time - failedGate bool -} - -// buildAsked is the newest build of a module from a commit that worked — when it was asked (or recorded, when -// the request time is not known), and whether it failed its gate — and false when the records hold none. A -// variable, so a test reads its own order. -var buildAsked = func(ctx context.Context, inv *inventory.Inventory, module, commit string) (askedBuild, bool, error) { - builds, err := inv.Builds(ctx, module, askedLookBack) - if err != nil { - return askedBuild{}, false, err - } - for _, b := range builds { - if b.Worked() && sameCommit(b.Commit, commit) { - failed, err := inv.GateFailed(ctx, b.ID) - if err != nil { - return askedBuild{}, false, err - } - return askedBuild{asked: b.AskedOrAt(), failedGate: failed}, true, nil - } - } - return askedBuild{}, false, nil -} - -// askedLookBack is how many of a module's newest builds are read for when a build was asked. -const askedLookBack = 50 - -// coreDowngrade is one core module a machine was last sent a build of that is newer than the mesh's record. -type coreDowngrade struct { - module, node, component string - // sent is the build the machine was last sent, record the one the mesh holds. - sent, record string - sentAsked, recordAsked time.Time -} - -func (d coreDowngrade) String() string { - return fmt.Sprintf("%s was last sent %s %s, and the mesh holds the older %s (asked %s, before %s): a send there "+ - "keeps %s; `push %s` sends the older build, and a newer merge or `rebuild %s` moves the record forward "+ - "(novox/hq issue 201)", d.node, d.module, short(d.sent), short(d.record), - d.recordAsked.UTC().Format(time.RFC3339), d.sentAsked.UTC().Format(time.RFC3339), short(d.sent), d.node, - d.module) -} - -// coreDowngrades is, for one machine, each guarded core module whose record is older than the build the machine -// was last sent, leaving out what a gate's own put-back puts back and a sent build that failed its gate. A move -// whose either build the records do not hold is not proved older and is left to the gate that judges it. Pure -// but for asked. -func coreDowngrades(node string, sent map[string]string, record map[string]string, scope sendScope, - asked func(module, commit string) (askedBuild, bool, error)) ([]coreDowngrade, error) { - var out []coreDowngrade - for module, was := range sent { - component := coreComponent(module) - now := record[module] - if !guarded(component) || was == "" || now == "" || sameCommit(was, now) { - continue - } - if scope.puttingBack && scope.modules[module] { - continue - } - from, known, err := asked(module, was) - if err != nil { - return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(was), err) - } - if !known || from.failedGate { - continue - } - to, known, err := asked(module, now) - if err != nil { - return nil, fmt.Errorf("when %s's build %s was asked cannot be read: %w", module, short(now), err) - } - if known && to.asked.Before(from.asked) { - out = append(out, coreDowngrade{module: module, node: node, component: component, sent: was, record: now, - sentAsked: from.asked, recordAsked: to.asked}) - } - } - sort.Slice(out, func(i, j int) bool { return out[i].module < out[j].module }) - return out, nil -} - -func guarded(component string) bool { - for _, c := range downgradeGuarded { - if c == component { - return true - } - } - return false -} - -// coreKept is, for a send that keeps (sendKeeps), each guarded core module of a machine to keep at the build it -// was last sent, module → commit, each said and raised as a condition. -func coreKept(ctx context.Context, inv *inventory.Inventory, node string, sent map[string]string, - current map[string]inventory.CurrentBuild) (map[string]string, error) { - record := map[string]string{} - for m, c := range current { - record[m] = c.Commit - } - found, err := coreDowngrades(node, sent, record, scopeOf(ctx), func(module, commit string) (askedBuild, bool, error) { - return buildAsked(ctx, inv, module, commit) - }) - if err != nil || len(found) == 0 { - return nil, err - } - out := map[string]string{} - for _, d := range found { - out[d.module] = d.sent - sayCoreDowngrade(ctx, d) - } - return out, nil -} - -// coreDowngradeObservation is a kept core build as a condition: `core...`. -func coreDowngradeObservation(component, node, summary string) conditions.Observation { - return conditions.Observation{Scope: conditions.ScopeCore, ID: component + "." + node, Kind: kindCoreDowngrade, - Token: kindCoreDowngrade, Machine: node, Severity: conditions.Warning, Source: coreDowngradeSource, - Summary: summary} -} - -func sayCoreDowngrade(ctx context.Context, d coreDowngrade) { - fmt.Printf("KEPT: %s\n", d) - keeper := conditionsFrom - if keeper == nil { - return - } - if _, err := keeper.Observe(ctx, coreDowngradeObservation(d.component, d.node, d.String())); err != nil { - fmt.Printf("the condition for %s kept on %s could not be raised: %v\n", d.module, d.node, err) - } -} - -// settleCoreDowngrades clears, after a send to a machine is away, the condition of each guarded core module the -// send carried at the mesh's record — a person's push that accepted the older build, or a send after the record -// moved forward — or did not carry at all. A send that kept the newer build leaves it open. -func settleCoreDowngrades(ctx context.Context, inv *inventory.Inventory, node string, carried map[string]string) { - keeper := conditionsFrom - if keeper == nil || carried == nil { - return - } - var open []string - for _, c := range downgradeGuarded { - if _, found, err := keeper.Get(ctx, coreDowngradeObservation(c, node, "").Key()); err == nil && found { - open = append(open, c) - } - } - if len(open) == 0 { - return - } - current, err := inv.CurrentBuilds(ctx) - if err != nil { - fmt.Printf("whether %s still keeps a core build over the mesh's record cannot be read: %v\n", node, err) - return - } - for module, now := range current { - component := coreComponent(module) - if !guarded(component) { - continue - } - if was, has := carried[module]; has && !sameCommit(was, now.Commit) { - continue - } - key := coreDowngradeObservation(component, node, "").Key() - if _, err := keeper.Clear(ctx, key, node+" was sent the "+component+" the mesh holds"); err != nil { - fmt.Printf("the condition %s could not be cleared: %v\n", key, err) - } - } -} diff --git a/cmd/mesh-controller/delivery.go b/cmd/mesh-controller/delivery.go deleted file mode 100644 index cad30ba5..00000000 --- a/cmd/mesh-controller/delivery.go +++ /dev/null @@ -1,1213 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "flag" - "fmt" - "os" - "slices" - "sort" - "strings" - "sync" - "time" - - "github.com/nats-io/nats.go" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// The controller's part in a delivery (novox/hq ADR 0239, to-be 47). -// -// **A delivery is mesh-delivery's; the walk is the controller's.** A delivery is one commit in one -// repository from its pull request's head to every machine, and a delivery group a few of them sharing a -// branch name; their states, order, holds and record belong to the module holding the `mesh-delivery` -// seat. The controller keeps what it already owned — the planner, the gate, registration, sending, the -// rollback — and the **walk**: one trunk commit's plan, tier by tier across the machines, one machine -// first and judged at the gate (ADR 0236). What changes here is when a walk starts. -// -// - A walk that moves the controller, the node-engine, the node tools, the bus or mesh-delivery itself is -// **on the controller's own path**: started by the merge, as every plan was. mesh-delivery cannot gate -// or deliver itself, and nothing the core needs to be repaired may wait for it. -// - Any other walk, **while the seat has a holder on record**, is opened by the merge and waits — nothing -// asked, nothing registered, nothing sent — until mesh-delivery says `deliver`, or a person says -// `plans go`. Nothing of it is registered before then, so no other send can carry it to a machine -// before its turn (ADR 0236 §4a carries everything registered). -// - With no holder on record, every walk starts at the merge, exactly as before this existed. -// -// The verbs mesh-delivery asks with are here: `delivery plan` (the planner's one answer for a diffset), -// `delivery order` (a group's order from the graph), `delivery check` (a group's heads composed), `delivery -// go`, `delivery stop` and `delivery walks`. - -// onTheControllersPath are the modules, by name, whose walk never waits for the delivery's owner, and what each is. -// The delivery's owner is not among them: it is whichever module claims the seat (ownPathOf, novox/hq ADR 0437 §3). -var onTheControllersPath = map[string]string{ - "mesh-controller": "the controller", - "mesh-host": "the node-engine", - "node-tools": "the node tools", - "nats": "the bus", -} - -// claimantsOf are the modules among the catalogued entries that claim a seat, each once, in name order. -func claimantsOf(entries []inventory.Entry, seat string) []string { - var out []string - for _, e := range entries { - if e.Manifest.ClaimsSeat(seat) && !slices.Contains(out, e.Manifest.Module) { - out = append(out, e.Manifest.Module) - } - } - sort.Strings(out) - return out -} - -// ownPathOf is the controller's own path among these entries: the modules by name, and the modules that claim -// the delivery seat, which is how the delivery's owner is found (novox/hq ADR 0437 §3). -func ownPathOf(entries []inventory.Entry) map[string]string { - own := make(map[string]string, len(onTheControllersPath)+1) - for m, what := range onTheControllersPath { - own[m] = what - } - for _, m := range claimantsOf(entries, catalogue.DeliverySeat) { - own[m] = "the delivery's owner" - } - return own -} - -// onTheOwnPath is whether one module, with its manifest at hand, is on the controller's own path. -func onTheOwnPath(module string, m catalogue.Manifest) bool { - _, named := onTheControllersPath[module] - return named || m.ClaimsSeat(catalogue.DeliverySeat) -} - -// ownPathNow is ownPathOf the catalogue as it is. -func ownPathNow(ctx context.Context, inv *inventory.Inventory) (map[string]string, error) { - entries, err := inv.Catalogued(ctx) - if err != nil { - return nil, err - } - return ownPathOf(entries), nil -} - -// deliverySeatHeld is whether a module claiming the delivery seat is assigned somewhere: the seat has a -// holder on record. Read from the catalogue the merge handler already holds; never from whether the -// holder answers, so a walk does not start by itself because mesh-delivery is down — that wait is said. -func deliverySeatHeld(entries []inventory.Entry) bool { - for _, e := range entries { - if len(e.On) > 0 && e.Manifest.ClaimsSeat(catalogue.DeliverySeat) { - return true - } - } - return false -} - -// awaitsFor is what a new walk waits for: nil when it starts at once — no holder on record, or a module -// on the controller's own path among those it moves. -func awaitsFor(entries []inventory.Entry, modules []string) *inventory.PlanDelivery { - if !deliverySeatHeld(entries) { - return nil - } - onPath := ownPathOf(entries) - for _, m := range modules { - if _, own := onPath[m]; own { - return nil - } - } - return &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat} -} - -// waitingNote is what a walk waiting for its word says, wherever it is read. -func waitingNote(p inventory.Plan) string { - return fmt.Sprintf("published; its walk waits for %s's word — `plans go %s --why …` starts it by hand", - p.Delivery.Awaits, p.ID) -} - -// waitingSaid is what a walk waiting for its word says when the delivery's owner said what holds it (novox/hq -// issue 529): a delivery held for a person, its reason and the verb that releases it — the person's decision, not -// a hand start — or the deliveries not at their turn. Told nothing, it is waitingNote. -func waitingSaid(p inventory.Plan, lines []stalledLine) string { - if len(lines) == 0 { - return waitingNote(p) - } - var held, unheld, other []string - for _, l := range lines { - switch l.State { - case "held": - held = append(held, l.Says) - case "unheld": - unheld = append(unheld, l.Says) - default: - other = append(other, l.Says) - } - } - switch { - case len(held) > 0: - return fmt.Sprintf("published; its walk waits for %s's word, and a delivery it carries is held for a person: %s", - p.Delivery.Awaits, strings.Join(held, "; ")) - case len(other) > 0: - // Not at its turn: it goes when its turn comes. A hand start here would skip its group's order (review of - // novox/hq issue 529), so none is offered. - return fmt.Sprintf("published; its walk waits for %s's word: %s", p.Delivery.Awaits, strings.Join(other, "; ")) - } - // Held by nothing: nothing will give it its word, so the hand start is the way on. - return fmt.Sprintf("published; its walk waits for %s's word: %s — `plans go %s --why …` starts it by hand", - p.Delivery.Awaits, strings.Join(unheld, "; "), p.ID) -} - -// letGo gives a waiting walk its word: by the delivery's owner, or a person. The next advance asks its -// first tier. Refused for a walk that does not wait. -func letGo(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) { - release, err := inv.HoldPlans(ctx, true) - if err != nil { - return inventory.Plan{}, err - } - defer release() - p, err := inv.PlanByID(ctx, id) - if err != nil { - return inventory.Plan{}, err - } - switch { - case !p.Open(): - return p, fmt.Errorf("%s is %s: there is no walk to start", p.ID, p.State) - case p.Delivery == nil || p.Delivery.Awaits == "": - return p, fmt.Errorf("%s waits for nobody: it started at its merge", p.ID) - case p.Delivery.Go != nil: - return p, fmt.Errorf("%s was let go by %s at %s already", p.ID, p.Delivery.By, - p.Delivery.Go.Local().Format("15:04:05")) - } - // **One walk at a time beside a related one** (novox/hq ADR 0276, ADR 0407 decision 4, issue 973): the word is - // taken, and the walk starts once no started walk it is related to is open (advanceOnce), so the delivery's - // owner says it once and is not refused. - now := time.Now().UTC() - p.Delivery.Go, p.Delivery.By, p.Delivery.Why = &now, by, why - p.Note = "let go by " + by + "; its first tier is asked next" - if err := inv.SavePlan(ctx, &p); err != nil { - return p, err - } - return p, nil -} - -// stopWalk ends a walk on its delivery's word: failed, said as stopped by whom, why. What it asked still -// builds and registers; nothing further is asked or sent. -func stopWalk(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, error) { - p, _, err := stopWalkWithout(ctx, inv, id, by, why, nil, time.Now().UTC()) - return p, err -} - -// stopAnswer is what a stop that leaves merges out says, as data (novox/hq ADR 0299): the record stopped, the -// batch that walks its other merges again with those merges, every merge left out, the modules the stopped walk -// already sent that stay on the machines named, the modules the batch's walk withholds, and every merge elsewhere -// that will deliver a stopped change when it is walked — a later merge of its repository, or one moving its modules. -type stopAnswer struct { - Stopped string `json:"stopped"` - WalkedAgainBy string `json:"walked_again_by,omitempty"` - WalkedAgain []string `json:"walked_again,omitempty"` - LeftOut []inventory.PlanLeftOut `json:"left_out,omitempty"` - Kept []inventory.PlanKept `json:"kept,omitempty"` - Withheld []string `json:"withheld,omitempty"` - StillCarriedBy []string `json:"still_carried_by,omitempty"` -} - -// stopWalkWithout ends a walk on its delivery's word and, given merges to leave out, walks every other merge it -// answered again without them (novox/hq issue 459, ADR 0299): one walk carries every merge of its batch, so ending -// it for one group member ended every other delivery on it. -// -// - Left out are the merges named, each later merge of their repository's branch, which contains one, and each -// merge moving a module a left-out merge moves: a module is built at its branch, which holds the stopped merge. -// - The others join the open batch of their kind — merges heard while the walk ran wait there, and an older -// commit never walks after a newer one — or a new batch when none is open; its window is closed, and its walk -// withholds the left-out merges' modules where it would build them only as dependents. -// - Given a batch not yet cut, the merges left out leave it for a stopped record of their own, and the batch -// goes on without them; with nothing left in it, the batch itself is stopped. -// - The stopped record, the batch and every merge moved are written in one transaction (decision 7). -// -// With nothing left to walk, or no merge named, the walk is only ended. A merge named that the record does not -// answer refuses the stop whole. -func stopWalkWithout(ctx context.Context, inv *inventory.Inventory, id, by, why string, without []string, - now time.Time) (inventory.Plan, stopAnswer, error) { - said := stopAnswer{Stopped: id} - release, err := inv.HoldPlans(ctx, true) - if err != nil { - return inventory.Plan{}, said, err - } - defer release() - p, err := inv.PlanByID(ctx, id) - if err != nil { - return inventory.Plan{}, said, err - } - batch := p.Batch() - if !p.Open() && !(batch && len(without) > 0) { - return p, said, fmt.Errorf("%s is already %s", p.ID, p.State) - } - var merges, again []inventory.BatchedMerge - if len(without) > 0 { - if merges, err = inv.MergesOf(ctx, p.ID); err != nil { - return p, said, err - } - if again, said.LeftOut, err = leaveOut(merges, without); err != nil { - return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err) - } - } - stopped := func(r *inventory.Plan) { - if r.Delivery == nil { - r.Delivery = &inventory.PlanDelivery{} - } - r.Delivery.Stopped, r.Delivery.StoppedWhy, r.Delivery.LeftOut = by, why, said.LeftOut - r.State = inventory.PlanFailed - r.Note = fmt.Sprintf("stopped by %s at tier %d: %s", by, r.Tier, why) - } - withheld := movesOf(leftOutMerges(merges, said.LeftOut)) - said.Withheld = withheld - if said.StillCarriedBy, err = stillCarried(ctx, inv, p.ID, merges, said.LeftOut, withheld); err != nil { - return p, said, err - } - if batch { - return p, said, stopInBatch(ctx, inv, &p, merges, again, &said, stopped, now) - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return p, said, err - } - running := map[string][]string{} - for _, e := range entries { - running[e.Manifest.Module] = e.On - } - said.Kept = keptOf(p, movesOf(again), running) - stopped(&p) - p.Delivery.Kept = said.Kept - if len(again) == 0 { - if len(said.LeftOut) > 0 { - p.Note += fmt.Sprintf("; left out %s, nothing else to walk", mergesLeftOutWords(said.LeftOut)) - } - return p, said, inv.SavePlan(ctx, &p) - } - target, folded, foldMoves, err := batchToWalkAgain(ctx, inv, p.ID, again, now) - if err != nil { - return p, said, fmt.Errorf("%s is not stopped: %w", p.ID, err) - } - target.Delivery.Batch.Closed = true - target.Delivery.Batch.Withheld = union(target.Delivery.Batch.Withheld, withheld) - sort.Strings(target.Delivery.Batch.Withheld) - said.WalkedAgainBy = target.ID - var moves []inventory.MergeMove - for _, m := range again { - said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit) - moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: target.ID, Alone: m.Alone}) - } - p.Delivery.WalkedAgainBy = target.ID - p.Note += fmt.Sprintf("; its other merges are walked again by %s: %s", target.ID, strings.Join(said.WalkedAgain, ", ")) - if len(said.LeftOut) > 0 { - p.Note += "; left out " + mergesLeftOutWords(said.LeftOut) - } - if len(said.Kept) > 0 { - p.Note += "; " + keptWords(said.Kept) - } - // The batch first, so no merge names a plan the store does not hold; all of it, or nothing (decision 7) — the - // batches folded into it too. - if err := inv.SavePlansMoving(ctx, append([]*inventory.Plan{&target, &p}, folded...), - append(moves, foldMoves...)); err != nil { - return p, said, fmt.Errorf("%s is not stopped, and none of its merges moved: %w", p.ID, err) - } - // What the batch holds and names now: written again by the next look should this fail. - if err := keepBatch(ctx, inv, &target, now, ""); err != nil { - fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", target.ID, err) - } - fmt.Printf("%s stopped by %s; %s walks its other merges again: %s\n", p.ID, by, target.ID, - strings.Join(said.WalkedAgain, ", ")) - return p, said, nil -} - -// stopInBatch leaves merges out of a batch not yet cut (ADR 0299): they go to a stopped record of their own, which -// names the batch as walking the others, and the batch withholds their modules; a batch left with none is stopped -// itself. -func stopInBatch(ctx context.Context, inv *inventory.Inventory, b *inventory.Plan, merges, again []inventory.BatchedMerge, - said *stopAnswer, stopped func(*inventory.Plan), now time.Time) error { - if len(again) == 0 { - stopped(b) - b.Delivery.Batch = nil - b.Note = "a batch whose every merge was left out by a stop: " + b.Note - return inv.SavePlan(ctx, b) - } - left := leftOutMerges(merges, said.LeftOut) - first := left[0] - r := inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch, - Commit: first.Commit, Merged: first.Merged, Created: now, Tiers: [][]string{}, - Modules: map[string]*inventory.PlanModule{}} - if r.ID == b.ID { - r.ID += "-left" - } - stopped(&r) - r.Delivery.WalkedAgainBy = b.ID - r.Note += fmt.Sprintf("; left out of %s before it was walked: %s; %s walks the others", b.ID, - mergesLeftOutWords(said.LeftOut), b.ID) - b.Delivery.Batch.Withheld = union(b.Delivery.Batch.Withheld, said.Withheld) - sort.Strings(b.Delivery.Batch.Withheld) - var moves []inventory.MergeMove - for _, m := range left { - moves = append(moves, inventory.MergeMove{Repository: m.Repository, Commit: m.Commit, Plan: r.ID, Alone: m.Alone}) - } - for _, m := range again { - said.WalkedAgain = append(said.WalkedAgain, m.Repository+"@"+m.Commit) - } - said.Stopped, said.WalkedAgainBy = r.ID, b.ID - if err := inv.SavePlansMoving(ctx, []*inventory.Plan{&r, b}, moves); err != nil { - return fmt.Errorf("%s is not changed, and none of its merges moved: %w", b.ID, err) - } - if err := keepBatch(ctx, inv, b, now, ""); err != nil { - fmt.Printf(" %s holds its merges; its record is written again at the next look: %v\n", b.ID, err) - } - fmt.Printf("%s: %s left out by a stop, kept by %s\n", b.ID, mergesLeftOutWords(said.LeftOut), r.ID) - return nil -} - -// batchToWalkAgain is the batch a stopped walk's merges join (novox/hq ADR 0299): a batch of their kind not yet cut -// that they are related to (ADR 0304 decision 1, ADR 0306 decision 3), or a new one, not yet written. Related to -// two or more, the newer are folded into the oldest: the plans and merge moves that fold them, to be written in -// the stop's one act. The kind is read from what the merges moved; the stopped walk itself is no walk a cut folds. -func batchToWalkAgain(ctx context.Context, inv *inventory.Inventory, stopping string, again []inventory.BatchedMerge, - now time.Time) (inventory.Plan, []*inventory.Plan, []inventory.MergeMove, error) { - entries, read, edges, err := theGraph(ctx, inv) - if err != nil { - return inventory.Plan{}, nil, nil, err - } - own := false - onPath := ownPathOf(entries) - for _, n := range movesOf(again) { - if _, on := onPath[n]; on { - own = true - } - } - joins, _, _, err := relatedBatches(ctx, inv, again, own, stopping, entries, read, edges) - if err != nil { - return inventory.Plan{}, nil, nil, err - } - if len(joins) > 0 { - into := joins[0] - if len(joins) == 1 { - return into, nil, nil, nil - } - plans, moves, err := foldInto(ctx, inv, &into, joins[1:]) - if err != nil { - return inventory.Plan{}, nil, nil, err - } - return into, plans[1:], moves, nil - } - first := again[0] - return inventory.Plan{ID: fmt.Sprintf("plan-%d", now.UnixNano()), Repository: first.Repository, Branch: first.Branch, - Commit: first.Commit, Merged: first.Merged, Created: now, State: inventory.PlanAssembling, - Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}, - Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: own}}}, nil, nil, nil -} - -// leaveOut splits a walk's merges by the merges named to leave out, each `@` (a commit's -// first characters will do): the merges walked again, and those left out — each named one; each later merge of -// its repository's branch, which contains it; and each merge moving a module a left-out merge moves, which is -// built at its branch and so holds the stopped merge — until none more is. A name the walk does not answer is -// refused. -func leaveOut(merges []inventory.BatchedMerge, without []string) ([]inventory.BatchedMerge, []inventory.PlanLeftOut, error) { - var stopped []inventory.BatchedMerge - for _, w := range without { - repository, commit, ok := strings.Cut(strings.TrimSpace(w), "@") - if !ok || repository == "" || len(commit) < 7 { - return nil, nil, fmt.Errorf("%q names no merge: @", w) - } - i := slices.IndexFunc(merges, func(m inventory.BatchedMerge) bool { - return strings.EqualFold(m.Repository, repository) && strings.HasPrefix(m.Commit, commit) - }) - if i < 0 { - return nil, nil, fmt.Errorf("it answers no merge %s@%s", repository, commit) - } - stopped = append(stopped, merges[i]) - } - same := func(a, b inventory.BatchedMerge) bool { - return strings.EqualFold(a.Repository, b.Repository) && a.Commit == b.Commit - } - out := map[int]inventory.PlanLeftOut{} - for i, m := range merges { - if slices.ContainsFunc(stopped, func(s inventory.BatchedMerge) bool { return same(s, m) }) { - out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit} - } - } - for grew := true; grew; { - grew = false - for i, m := range merges { - if _, left := out[i]; left { - continue - } - for j, o := range out { - l := merges[j] - contains := o.Contains - if contains == "" { - contains = l.Commit - } - if strings.EqualFold(l.Repository, m.Repository) && l.Branch == m.Branch && laterMerge(m, l) { - out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains} - } else if shared := sharedMoves(m, l); len(shared) > 0 { - out[i] = inventory.PlanLeftOut{Repository: m.Repository, Commit: m.Commit, Contains: contains, - Shares: shared} - } else { - continue - } - grew = true - break - } - } - } - var again []inventory.BatchedMerge - var left []inventory.PlanLeftOut - for i, m := range merges { - if o, is := out[i]; is { - left = append(left, o) - } else { - again = append(again, m) - } - } - sort.SliceStable(left, func(i, j int) bool { return left[i].Contains == "" && left[j].Contains != "" }) - return again, left, nil -} - -// sharedMoves are the modules two kept merges both moved when they were heard. -func sharedMoves(a, b inventory.BatchedMerge) []string { - var out []string - theirs := announcedOf(b).Moves - for _, n := range announcedOf(a).Moves { - if slices.Contains(theirs, n) && !slices.Contains(out, n) { - out = append(out, n) - } - } - sort.Strings(out) - return out -} - -// leftOutMerges are the kept merges a stop left out. -func leftOutMerges(merges []inventory.BatchedMerge, out []inventory.PlanLeftOut) []inventory.BatchedMerge { - var left []inventory.BatchedMerge - for _, m := range merges { - if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool { - return strings.EqualFold(o.Repository, m.Repository) && o.Commit == m.Commit - }) { - left = append(left, m) - } - } - return left -} - -// movesOf is every module kept merges moved when they were heard, sorted. -func movesOf(merges []inventory.BatchedMerge) []string { - var out []string - for _, m := range merges { - out = union(out, announcedOf(m).Moves) - } - sort.Strings(out) - return out -} - -// keptOf are the modules a stopped walk sent that no merge walked again moves, and the machines each reached: -// those keep running the stopped walk's build. A module sent to its first machines reached those, and the rest it -// was sent to after them; one sent with no first machine (its policy sends it to all together, or a later tier is -// built by it) reached every machine running it. -func keptOf(p inventory.Plan, walkedAgain []string, running map[string][]string) []inventory.PlanKept { - var out []inventory.PlanKept - for name, m := range p.Modules { - if m == nil || slices.Contains(walkedAgain, name) { - continue - } - var machines []string - if m.FirstAt != nil || m.SentAt != nil { - machines = union(machines, m.First) - } - if m.SentAt != nil && len(m.First) == 0 { - machines = union(machines, running[name]) - } - if m.Gate != nil && m.Gate.Since != nil { - machines = union(machines, m.Gate.Machines) - } - for machine := range m.Rest { - machines = union(machines, []string{machine}) - } - if len(machines) == 0 { - continue - } - sort.Strings(machines) - out = append(out, inventory.PlanKept{Module: name, Machines: machines}) - } - sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) - return out -} - -// keptWords is what a stopped walk leaves on the machines, as a person reads it. -func keptWords(kept []inventory.PlanKept) string { - var words []string - for _, k := range kept { - words = append(words, k.Module+" on "+strings.Join(k.Machines, ", ")) - } - return "left running the stopped walk's build, which nothing walks again: " + strings.Join(words, "; ") -} - -// stillCarried are the merges outside the stopped record, in a batch or an open walk, that will deliver a stopped -// change when they are walked: a later merge of its repository's branch, which contains it, or one that moves a -// module it moves. Holding a merged change back for good is a revert's (ADR 0299). -func stillCarried(ctx context.Context, inv *inventory.Inventory, id string, merges []inventory.BatchedMerge, - out []inventory.PlanLeftOut, withheld []string) ([]string, error) { - var named []inventory.BatchedMerge - for _, m := range leftOutMerges(merges, out) { - if slices.ContainsFunc(out, func(o inventory.PlanLeftOut) bool { return o.Commit == m.Commit && o.Contains == "" }) { - named = append(named, m) - } - } - if len(named) == 0 { - return nil, nil - } - open, err := inv.OpenPlans(ctx) - if err != nil { - return nil, err - } - batches, err := inv.Batches(ctx) - if err != nil { - return nil, err - } - var carried []string - for _, p := range append(open, batches...) { - if p.ID == id { - continue - } - ms, err := inv.MergesOf(ctx, p.ID) - if err != nil { - return nil, err - } - for _, m := range ms { - later := slices.ContainsFunc(named, func(s inventory.BatchedMerge) bool { - return strings.EqualFold(s.Repository, m.Repository) && s.Branch == m.Branch && laterMerge(m, s) - }) - shares := slices.ContainsFunc(announcedOf(m).Moves, func(n string) bool { return slices.Contains(withheld, n) }) - if later || shares { - carried = append(carried, fmt.Sprintf("%s@%s (in %s)", m.Repository, m.Commit, p.ID)) - } - } - } - sort.Strings(carried) - return carried, nil -} - -// mergesLeftOutWords is the merges a stop left out, as a person reads them: each that contains a stopped one, or -// shares its modules, says so. -func mergesLeftOutWords(out []inventory.PlanLeftOut) string { - var words []string - for _, o := range out { - s := o.Repository + "@" + short(o.Commit) - switch { - case len(o.Shares) > 0: - s += " (which moves " + strings.Join(o.Shares, ", ") + " as " + short(o.Contains) + " does)" - case o.Contains != "": - s += " (which contains " + short(o.Contains) + ")" - } - words = append(words, s) - } - return strings.Join(words, ", ") -} - -// orderMember is one member of a group, as mesh-delivery says it. -type orderMember struct { - ID string `json:"id"` - Repository string `json:"repository"` - Base string `json:"base,omitempty"` - Head string `json:"head,omitempty"` - Number int `json:"number,omitempty"` - Paths []string `json:"paths,omitempty"` - PathsTruncated bool `json:"paths_truncated,omitempty"` - Removed []string `json:"removed,omitempty"` - ModuleDirs []string `json:"module_dirs,omitempty"` - ModuleDirsSaid bool `json:"module_dirs_said,omitempty"` - CloneURL string `json:"clone_url,omitempty"` - // BaseSHA is the base branch's head the forge last announced the head with (novox/hq ADR 0323): the trunk commit - // a verdict no build judges names when the head is checked again through mesh-delivery. - BaseSHA string `json:"base_sha,omitempty"` - // Ready says mesh-delivery wants the member's check queued before every other check (novox/hq ADR 0406 §3, issue - // 974). There is no ready mark any more (issue 1022): mesh-delivery says it when it asks for the Approve. - Ready bool `json:"ready,omitempty"` -} - -// pull is the member as the forge announced it. -func (m orderMember) pull() link.PullUpdated { - owner, repo, _ := strings.Cut(m.Repository, "/") - base := m.Base - if base == "" { - base = "main" - } - return link.PullUpdated{Owner: owner, Repo: repo, Number: m.Number, Base: base, Commit: m.Head, BaseSHA: m.BaseSHA, - CloneURL: m.CloneURL, Paths: m.Paths, PathsTruncated: m.PathsTruncated, Removed: m.Removed, - ModuleDirs: m.ModuleDirs, ModuleDirsSaid: m.ModuleDirsSaid, Ready: m.Ready} -} - -// orderPair is one "before" among a group's members, why, and the rules it won over. -type orderPair struct { - Before string `json:"before"` - After string `json:"after"` - Why string `json:"why"` - // Over is the rules that ordered the pair the other way and lost to Why, by precedence (novox/hq ADR - // 0249): said, so a person reading the order sees which rule won. - Over []string `json:"over,omitempty"` -} - -// orderReach is what a member moves, as the planner says. -type orderReach struct { - Moved []string `json:"moved,omitempty"` - Dependents []string `json:"dependents,omitempty"` - New []string `json:"new,omitempty"` - Manifests []string `json:"manifests,omitempty"` -} - -// groupOrder is a group's order: the members in it, every pair and why, and the cycle when there is one. The -// order is read from the module graph and the manifests' change alone (novox/hq ADR 0419): a cycle is the only -// refusal, and CycleSaid names every module and edge of it. Edges are the module edges the pairs come from. -type groupOrder struct { - Order []string `json:"order"` - Pairs []orderPair `json:"pairs,omitempty"` - Cycle []string `json:"cycle,omitempty"` - CycleSaid string `json:"cycle_said,omitempty"` - Edges []inventory.Edge `json:"edges,omitempty"` - Reach map[string]orderReach `json:"reach,omitempty"` -} - -// orderBuiltBy is the reason a member goes before another that moves a module built by, standing on, packaging or -// declaring one of its modules: the module graph's own edge. -const orderBuiltBy = "built by" - -// orderOf is a group's order (novox/hq ADR 0419), pure: a member goes before another when a module it moves goes -// before a module the other moves by an edge of the module graph (built by), or by a rule of delivery_edges.go -// (verb before caller, version skew, engine before controller), which changes names the manifests' added verbs. -// No `after:` line is read. Ties break by repository then id, so every reading gives one order. A cycle through -// two members or more is named by its members, and none of them is ordered. -func orderOf(members []orderMember, reach map[string]orderReach, edges []inventory.Edge, changes map[string]moduleChange) groupOrder { - out := groupOrder{Reach: reach} - movedBy := map[string][]string{} - var all []string - for _, m := range members { - for _, n := range reach[m.ID].Moved { - movedBy[n] = append(movedBy[n], m.ID) - all = appendNew(all, n) - } - } - add := func(before, after, why string) { - if before == after { - return - } - for _, p := range out.Pairs { - if p.Before == before && p.After == after && p.Why == why { - return - } - } - out.Pairs = append(out.Pairs, orderPair{Before: before, After: after, Why: why}) - } - for _, e := range edges { - if !orders(e) { - continue - } - for _, before := range movedBy[e.To] { - for _, after := range movedBy[e.From] { - add(before, after, orderBuiltBy) - } - } - } - order := orderEdges(all, changes, edges) - out.Edges = order.Edges - out.CycleSaid = order.CycleSaid() - for _, e := range order.Edges { - for _, before := range movedBy[e.To] { - for _, after := range movedBy[e.From] { - add(before, after, e.Why) - } - } - } - sort.Slice(out.Pairs, func(i, j int) bool { - if out.Pairs[i].Before != out.Pairs[j].Before { - return out.Pairs[i].Before < out.Pairs[j].Before - } - if out.Pairs[i].After != out.Pairs[j].After { - return out.Pairs[i].After < out.Pairs[j].After - } - return out.Pairs[i].Why < out.Pairs[j].Why - }) - // Kahn's walk, the next always the first by repository and id among those with nothing before them. - waiting := map[string]int{} - for _, m := range members { - waiting[m.ID] = 0 - } - for _, p := range out.Pairs { - waiting[p.After]++ - } - done := map[string]bool{} - for len(done) < len(members) { - var ready []orderMember - for _, m := range members { - if !done[m.ID] && waiting[m.ID] == 0 { - ready = append(ready, m) - } - } - if len(ready) == 0 { - for _, m := range members { - if !done[m.ID] { - out.Cycle = append(out.Cycle, m.ID) - } - } - sort.Strings(out.Cycle) - return out - } - sort.Slice(ready, func(i, j int) bool { - if ready[i].Repository != ready[j].Repository { - return ready[i].Repository < ready[j].Repository - } - return ready[i].ID < ready[j].ID - }) - next := ready[0] - done[next.ID] = true - out.Order = append(out.Order, next.ID) - for _, p := range out.Pairs { - if p.Before == next.ID { - waiting[p.After]-- - } - } - } - return out -} - -// reachOfMembers is each member's reach, as the planner says it. -func reachOfMembers(members []orderMember, entries []inventory.Entry, read map[string][]inventory.ReadRepository, - edges []inventory.Edge) map[string]orderReach { - out := map[string]orderReach{} - for _, m := range members { - s := pullScope(m.pull(), entries, read, edges) - out[m.ID] = orderReach{Moved: s.Modules, Dependents: s.Dependents, New: s.New, Manifests: s.Manifests} - } - return out -} - -// changeMember is one change as the planner is asked about it, the same for `delivery plan` and `plans --what-if`. -func changeMember(repository, base, head string, paths, removed []string, moduleDirs *[]string) orderMember { - m := orderMember{ID: repository + "@" + head, Repository: repository, Base: base, Head: head, - Paths: paths, Removed: removed} - if moduleDirs != nil { - m.ModuleDirs, m.ModuleDirsSaid = *moduleDirs, true - } - return m -} - -// deliveryPlanDocument is the answer of `delivery plan` and of `plans --what-if … --data`: one change's -// change plan with its reach, the mesh's own part in it and whether the gate would run. One function, so -// the two verbs cannot differ (novox/hq ADR 0450 §2, issue 1324). -func deliveryPlanDocument(m orderMember, entries []inventory.Entry, read map[string][]inventory.ReadRepository, - edges []inventory.Edge, policy policyOf) map[string]any { - s := pullScope(m.pull(), entries, read, edges) - plan := changePlanOf(m.Repository, m.Base, m.Head, s.Reach, entries, policy) - return map[string]any{"plan": plan, "reach": orderReach{Moved: s.Modules, Dependents: s.Dependents, - New: s.New, Manifests: s.Manifests}, "mesh": s.Mesh, "gated": s.gated()} -} - -// deliveryCommand is `delivery`, the controller's verbs for the delivery's owner: -// -// delivery plan --repository owner/repo --head [--base main] --paths a,b [--module-dirs …] [--removed …] -// delivery order --members -// delivery check --group --members -// delivery go [--by ] [--why ] -// delivery stop --why [--by ] -// delivery walks [-n 50] [--plan ] -func deliveryCommand(ctx context.Context, args []string) error { - if len(args) == 0 { - return errors.New("delivery order|go|stop|walks") - } - sub, rest := args[0], args[1:] - if sub == "plan" { - // Removed (ADR 0450 §8; issue 1318): refused by name before any flag is read, naming its replacement. - return removedByName("delivery-plan") - } - if sub == "check" { - // Removed (ADR 0503 §3, ADR 0527; issue 1318): the checker judges every head, so the controller asks nothing. - return removedByName("delivery-check") - } - set := flag.NewFlagSet("delivery "+sub, flag.ContinueOnError) - membersJSON := set.String("members", "", "a group's members, as JSON") - membersFile := set.String("members-file", "", "a file of a group's members, as JSON: the same as --members, for a list "+ - "too long for an argument (novox/hq issue 1272)") - ready := set.Bool("ready", false, "the head's pull request is marked ready: its check is queued before every other check "+ - "(novox/hq ADR 0406 §3)") - by := set.String("by", catalogue.DeliverySeat, "who says it") - why := set.String("why", "", "why") - without := set.String("without", "", "merges a stop leaves out, @, comma-separated: the walk's "+ - "other merges are walked again without them (novox/hq ADR 0299)") - limit := set.Int("n", 50, "how many ended walks to answer beside the open ones") - planID := set.String("plan", "", "one walk") - positionals, err := parseAround(set, rest) - if err != nil { - return err - } - var members []orderMember - if *membersFile != "" { - raw, err := os.ReadFile(*membersFile) - if err != nil { - return fmt.Errorf("--members-file: %w", err) - } - *membersJSON = string(raw) - } - if *membersJSON != "" { - if err := json.Unmarshal([]byte(*membersJSON), &members); err != nil { - return fmt.Errorf("the members are not readable: %w", err) - } - // Marked ready by the verb's word for every member it names, besides any member that says so itself. - if *ready { - for i := range members { - members[i].Ready = true - } - } - } - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - inv := open.inventory - answer := func(v any) error { - enc := json.NewEncoder(os.Stdout) - enc.SetIndent("", " ") - return enc.Encode(v) - } - switch sub { - case "order": - if len(members) == 0 { - return errors.New("delivery order --members : a group has members") - } - entries, read, edges, err := theGraph(ctx, inv) - if err != nil { - return err - } - changes, err := changesOf(ctx, changeSourcesOf(members, entries, reachOfMembers(members, entries, read, edges))) - if err != nil { - return fmt.Errorf("what the members say of their manifests cannot be read: %w", err) - } - return answer(orderOf(members, reachOfMembers(members, entries, read, edges), edges, changes)) - case "go": - if len(positionals) != 1 { - return errors.New("delivery go [--by ] [--why ]") - } - var p inventory.Plan - if err := sayingOnTheBus(ctx, inv, func() (err error) { - p, err = letGo(ctx, inv, positionals[0], *by, strings.TrimSpace(*why)) - return err - }); err != nil { - return err - } - fmt.Printf("%s (%s at %s) is let go by %s; its first tier is asked when the controller next looks at the plans, which it does as this ends\n", p.ID, p.Repository, - short(p.Commit), *by) - return nil - case "stop": - if len(positionals) != 1 || strings.TrimSpace(*why) == "" { - return errors.New("delivery stop --why [--by ] [--without @,…]") - } - var p inventory.Plan - var said stopAnswer - if err := sayingOnTheBus(ctx, inv, func() (err error) { - p, said, err = stopWalkWithout(ctx, inv, positionals[0], *by, strings.TrimSpace(*why), splitList(*without), - time.Now().UTC()) - return err - }); err != nil { - return err - } - if *without != "" { - // What it left out and what it walks again, as data: the delivery's owner reads it (ADR 0299). - return answer(said) - } - fmt.Printf("%s stopped by %s at tier %d of %d; what was asked still builds and registers, nothing further "+ - "is asked or sent\n", p.ID, *by, p.Tier, len(p.Tiers)) - return nil - case "backups-again": - if len(positionals) != 1 || strings.TrimSpace(*why) == "" { - return errors.New("delivery backups-again --why [--by ]") - } - var p inventory.Plan - var again []inventory.PlanBackup - if err := sayingOnTheBus(ctx, inv, func() (err error) { - p, again, err = backupsAgain(ctx, inv, positionals[0], *by, strings.TrimSpace(*why)) - return err - }); err != nil { - return err - } - return answer(map[string]any{"plan": p.ID, "asked_again": again}) - case "walks": - var walks []inventory.Plan - if *planID != "" { - p, err := inv.PlanByID(ctx, *planID) - if err != nil { - return err - } - walks = []inventory.Plan{p} - } else { - if walks, err = inv.OpenPlans(ctx); err != nil { - return err - } - // And the batch being assembled (novox/hq ADR 0276): never a walk to link or let go, shown. - batches, err := inv.Batches(ctx) - if err != nil { - return err - } - walks = append(walks, batches...) - recent, err := inv.RecentPlans(ctx, *limit) - if err != nil { - return err - } - for _, p := range recent { - if !slices.ContainsFunc(walks, func(w inventory.Plan) bool { return w.ID == p.ID }) { - walks = append(walks, p) - } - } - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return err - } - // Each walk's phases, with the rest's reports (novox/hq ADR 0282 decision 6). - now := time.Now() - for i := range walks { - walks[i].Phases = walks[i].WalkPhases(now, appliedFrom(ctx, inv)) - } - return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks, - "own-path": sortedKeysOf(ownPathOf(entries))}) - } - return fmt.Errorf("delivery %s: plan, order, check, go, stop, backups-again or walks", sub) -} - -// appliedFrom answers a machine's own report of the send a walk made to it from the controller's `apply` durations -// (novox/hq issue 582); a lookup that fails is a report not read, which leaves the walk's end unknown rather than wrong. -func appliedFrom(ctx context.Context, inv *inventory.Inventory) inventory.AppliedLookup { - return func(node string, send inventory.SentDeclaration, walkSent time.Time) (inventory.AppliedReport, bool) { - r, ok, err := inv.FirstAppliedAfter(ctx, node, send, walkSent) - if err != nil { - fmt.Fprintf(os.Stderr, "the report of %s after %s could not be read: %v\n", node, walkSent.Format(time.RFC3339), err) - return inventory.AppliedReport{}, false - } - // And, for each module the send handed to the provisioner, its client's report on that build (novox/hq ADR - // 0362): applied only once both said it, at the later; failed if the client failed. - return inventory.ProvisionedSend(node, send, r, ok, provLookup(ctx)) - } -} - -// recordWalkPhases keeps, once, each phase of every walk ended lately whose end is known, as a duration of kind -// walk-phase per class (novox/hq ADR 0282 decision 6): what `durations` summarises. A walk whose end is unknown -// past ApplySilentAfter keeps its measured phases without its total. -func recordWalkPhases(ctx context.Context, inv *inventory.Inventory, now time.Time) error { - recent, err := inv.RecentPlans(ctx, 30) - if err != nil { - return err - } - for _, p := range recent { - if p.State != inventory.PlanDone || p.Release != nil || now.Sub(p.Updated) > 2*time.Hour { - continue - } - anyKept, totalKept, err := inv.WalkPhasesKept(ctx, p.ID) - if err != nil { - return err - } - if totalKept { - continue - } - ph := p.WalkPhases(now, appliedFrom(ctx, inv)) - if ph != nil && ph.End == nil && anyKept { - continue // kept without its end; kept again only once its end is known - } - if ph == nil || (ph.End == nil && now.Sub(p.Updated) < inventory.ApplySilentAfter+time.Minute) { - continue - } - class := ph.Class - if class == "" { - class = "unclassed" - } - for _, x := range ph.Phases { - if x.State != inventory.PhaseMeasured || x.Start == nil { - continue - } - if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationWalkPhase, - Subject: class + "/" + x.Name, Ref: fmt.Sprintf("%s/%s/%d", p.ID, x.Name, x.Tier), Started: *x.Start, - Took: time.Duration(x.TookMS) * time.Millisecond, Detail: p.Named()}); err != nil { - return err - } - } - if ph.End != nil && ph.From != nil { - if err := inv.RecordDuration(ctx, inventory.Duration{Kind: inventory.DurationWalkPhase, - Subject: class + "/total", Ref: p.ID + "/total", Started: *ph.From, - Took: time.Duration(ph.TotalMS) * time.Millisecond, Detail: ph.Said}); err != nil { - return err - } - } - } - return nil -} - -// theGraph is what the planner reads. -func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry, map[string][]inventory.ReadRepository, - []inventory.Edge, error) { - entries, err := inv.Catalogued(ctx) - if err != nil { - return nil, nil, nil, err - } - read, err := readForPlanning(ctx, inv) - if err != nil { - return nil, nil, nil, err - } - edges, err := inv.Dependencies(ctx) - if err != nil { - return nil, nil, nil, err - } - return entries, read, edges, nil -} - -// appendNew appends what is not there yet. -func appendNew(to []string, items ...string) []string { - for _, i := range items { - if !slices.Contains(to, i) { - to = append(to, i) - } - } - return to -} - -// sayPlanMoved says a walk kept in a new state as the controller's `plan-moved`, the plan whole: what the -// delivery it walks reads its steps from. Never in the way of the walk: said beside it, and a save that could -// not be said is logged — the delivery's owner, which also asks for the walks it follows, finds it by -// comparison. Records arriving out of order are told apart by the plan's revision. -// -// On the save that starts the walk and the one that ends it, the walk also says what it touches, its bound and, -// at the end, its outcome (novox/hq ADR 0308 decision 1, walk_said.go), read from facts; nil facts say only what -// the walk recorded. -// -// The plan the announcement is made from is a copy that shares nothing with the walk, taken before the goroutine -// starts: the walk goes on writing its steps' maps and pointers while the announcement is marshalled and its -// touches read (novox/hq issue 1260), and it says the plan as it was when this was called. -func sayPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inventory.Plan) { - kept, err := copyOfPlan(p) - if err != nil { - fmt.Fprintf(os.Stderr, "%s: kept, and could not be said as plan-moved: %v\n", p.ID, err) - return - } - go publishPlanMoved(ctx, bus, facts, kept) -} - -// copyOfPlan is the plan with nothing shared with the original: its JSON round trip, which is all of it but what -// the plan never keeps (WalkSaid), carried over. -func copyOfPlan(p inventory.Plan) (inventory.Plan, error) { - raw, err := json.Marshal(p) - if err != nil { - return inventory.Plan{}, err - } - var kept inventory.Plan - if err := json.Unmarshal(raw, &kept); err != nil { - return inventory.Plan{}, err - } - kept.WalkSaid = p.WalkSaid - return kept, nil -} - -func publishPlanMoved(ctx context.Context, bus link.Bus, facts walkFacts, p inventory.Plan) { - // Its phases so far (novox/hq ADR 0282 decision 6): the rest's reports come after the walk ends, and are - // read by whoever asks for the walk (`delivery walks`). - p.Phases = p.WalkPhases(time.Now(), nil) - body, err := json.Marshal(planMovedBody{Plan: p, walkSaid: sayWalk(ctx, facts, p)}) - if err != nil { - return - } - stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) - defer stop() - if err := bus.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyPlanMoved, body); err != nil { - fmt.Fprintf(os.Stderr, "%s: kept, and could not be said as plan-moved: %v\n", p.ID, err) - } -} - -// sayingOnTheBus runs a command that keeps walks or says verdicts with the bus to say them on: the serving -// controller's, or a connection of the command's own — a verb runs as a command of its own, and what it kept -// would otherwise be said by nobody until the delivery's owner read the walks back. Without a bus the command -// still runs; what it did is found by comparison. -func sayingOnTheBus(ctx context.Context, facts walkFacts, f func() error) error { - if checkEvents != nil { - return f() - } - ran := false - err := onTheBus(func(conn *nats.Conn) error { - js, err := conn.JetStream() - if err != nil { - return err - } - bus := link.OverNATS{Conn: conn, JS: js} - was := inventory.PlanSaved - checkEvents = bus - inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { publishPlanMoved(ctx, bus, facts, p) } - defer func() { checkEvents, inventory.PlanSaved = nil, was }() - ran = true - return f() - }) - if !ran { - fmt.Fprintf(os.Stderr, "the bus cannot be reached (%v): what this does is not said, and is found by comparison\n", err) - return f() - } - return err -} - -// sayingWhatAVerbKept is PlanSaved for a process that is not the serving controller: each walk the verb keeps is -// said as `plan-moved` before the save returns, with the facts of the inventory that kept it, on the bus the verb -// reaches. It reaches the bus on the first save and never for a verb that keeps no walk. When the bus cannot be -// reached, a save that started or ended a walk says so on the verb's error output: its mark is kept and no later -// save says it, so the router holds what the walk touched until its bound, or not at all for a start. done lets -// the bus go and takes the hook back. -func sayingWhatAVerbKept(ctx context.Context) (said func(*inventory.Inventory, inventory.Plan), done func()) { - var ( - mu sync.Mutex - reached bool - bus link.Bus - closing func() - failed error - ) - said = func(inv *inventory.Inventory, p inventory.Plan) { - mu.Lock() - defer mu.Unlock() - if !reached { - reached = true - if handActConn != nil { - bus, failed = overNATS(handActConn) - } else if js, err := aBus(); err != nil { - failed = err - } else if bus, failed = overNATS(js.Conn()); failed == nil { - closing = js.Close - } else { - js.Close() - } - } - if bus == nil { - switch p.WalkSaid { - case inventory.WalkStarted: - fmt.Fprintf(os.Stderr, "%s: its start is kept and could NOT be said as plan-moved (%v): the router "+ - "does not hold what it touches\n", p.ID, failed) - case inventory.WalkEnded: - fmt.Fprintf(os.Stderr, "%s: its end is kept and could NOT be said as plan-moved (%v): the router holds "+ - "what it touched until its bound\n", p.ID, failed) - } - return - } - publishPlanMoved(ctx, bus, inv, p) - } - done = func() { - mu.Lock() - defer mu.Unlock() - inventory.PlanSaved = nil - if closing != nil { - closing() - } - } - return said, done -} - -// overNATS is the bus a connection carries, with its JetStream. -func overNATS(conn *nats.Conn) (link.Bus, error) { - js, err := conn.JetStream() - if err != nil { - return nil, err - } - return link.OverNATS{Conn: conn, JS: js}, nil -} diff --git a/cmd/mesh-controller/delivery_conditions.go b/cmd/mesh-controller/delivery_conditions.go deleted file mode 100644 index c5de9415..00000000 --- a/cmd/mesh-controller/delivery_conditions.go +++ /dev/null @@ -1,262 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "strings" - "time" - - "github.com/nats-io/nats.go" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" -) - -// A delivery held past its bound, said by the controller and healed by H2 from mesh-delivery's own table -// (novox/hq ADR 0239 decision 9, to-be 47 Phase B). The conditions are the controller's, one owner: the -// self-check reads the delivery owner's `stalled` (probe D14) and raises `delivery..stalled`; healer -// H2 calls the owner's `close`, which takes only the transition the table names for that state, and only -// the next observation says whether it worked (ADR 0231). - -// probeDeliveriesID is the probe that reads the delivery's owner. -const probeDeliveriesID = "D14" - -// kindDeliveryStalled is what a delivery held past its bound raises: H2's kind, in the delivery scope. -const kindDeliveryStalled = "stalled" - -// deliveryOwnerWithin is how long the owner is given to answer. -var deliveryOwnerWithin = 10 * time.Second - -// stalledLine is one delivery past its bound, as mesh-delivery's `stalled` says it. -type stalledLine struct { - ID string `json:"id"` - // Number is the pull request's, for a line of a head the forge never announced (novox/hq issue 347). - Number int `json:"number,omitempty"` - State string `json:"state"` - // Walk is the walk a line is about: a waiting walk no live delivery holds, or, asked with `walks`, each walk - // waiting for the owner's word and what holds it (novox/hq issues 362, 529). - Walk string `json:"walk,omitempty"` - // Waiting are the merge checks a line of the state `unanswered` waits on, as mesh-delivery says each: - // "mesh/check pending since ", "mesh/check never set" (novox/hq issue 438). - Waiting []string `json:"waiting,omitempty"` - // Checks are the same merge checks as data, which the controller words in the operator's own time (novox/hq - // issue 443): a time inside a finished sentence cannot be said again in another zone. A mesh-delivery from before - // says none, and Waiting is said as it reads. - Checks []waitingCheck `json:"checks,omitempty"` - // Held is the state of the delivery of a line of the state `unanswered`, ready or rejected, when one is held - // (novox/hq issues 441, 451): its check answered once, so a recheck is the step. Empty for a head with no - // delivery, and from a mesh-delivery that does not say it yet; the words are never read for it. - Held string `json:"held,omitempty"` - For string `json:"for"` - Bound string `json:"bound"` - H2 string `json:"h2"` - Says string `json:"says"` - // Severity is "urgent" for a group held or half merged and a refused member (novox/hq issues 1275, 1278); empty - // or anything else is a warning, as before. - Severity string `json:"severity,omitempty"` - // Needs is "new-head" for a delivery whose cure is its author's new head (novox/hq ADR 0505 §5), Author that author - // as the operator reads it ("the author of feat/x", with the session that pushed it when its pull request names one) - // and Branch its branch. A mesh-delivery from before says none, and the words are as before. - Needs string `json:"needs,omitempty"` - Author string `json:"author,omitempty"` - Branch string `json:"branch,omitempty"` -} - -// needsNewHead is what a stalled line needs when its cure is its author's new head (ADR 0505 §5). -const needsNewHead = "new-head" - -// severity is the condition's severity a stalled line asks: urgent when it says so, a warning otherwise. -func (l stalledLine) severity() conditions.Severity { - if l.Severity == string(conditions.Urgent) { - return conditions.Urgent - } - return conditions.Warning -} - -// waitingCheck is one merge check a stalled line waits on, as mesh-delivery gives it: its context, its state — -// "pending", or "never-set" — and, for a pending one, since when in RFC 3339, empty when the forge did not say. -type waitingCheck struct { - Context string `json:"context"` - State string `json:"state"` - Since string `json:"since,omitempty"` -} - -// wordsNow is the time the words are said at, which says whether a time needs its day; a seam a test replaces. -var wordsNow = time.Now - -// wordsZone is the zone a stalled line's times are said in: the controller's local zone, as every other time in its -// messages. A seam a test replaces, so that no test writes time.Local, which every goroutine of the package reads. -var wordsZone = func() *time.Location { return time.Local } - -// operatorsOnly is whether the table leaves H2 nothing to do for the line: the state is the operator's. -func (l stalledLine) operatorsOnly() bool { return l.H2 == "" || strings.HasPrefix(l.H2, "none") } - -// askDeliveryOwner asks the holder of the mesh-delivery seat one of the verbs the controller is granted; -// a seam a test replaces. The holder's own refusal is an error naming it. -var askDeliveryOwner = func(ctx context.Context, conn *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) { - granted := false - for _, v := range broker.VerbsTheControllerAsksTheDeliveryOwner { - granted = granted || v.Verb == verb - } - if !granted { - return nil, fmt.Errorf("the controller asks %s.%s, which its grant does not name", catalogue.DeliverySeat, verb) - } - if conn == nil { - return nil, errors.New("this controller is not on the bus") - } - answer, err := link.AskMeshSeatTool(ctx, conn, catalogue.DeliverySeat, verb, args, deliveryOwnerWithin) - if err != nil { - return nil, err - } - if answer.Error != "" { - return nil, fmt.Errorf("%s.%s refused: %s", catalogue.DeliverySeat, verb, answer.Error) - } - return unwrapToolResult(answer.Result), nil -} - -// unwrapToolResult is a tool's answer whatever the runtime wrapped it in: the JSON itself, or the -// protocol's content list holding it as text. -func unwrapToolResult(raw json.RawMessage) json.RawMessage { - var wrapped struct { - Content []struct { - Text string `json:"text"` - } `json:"content"` - } - if json.Unmarshal(raw, &wrapped) == nil && len(wrapped.Content) > 0 && json.Valid([]byte(wrapped.Content[0].Text)) { - return json.RawMessage(wrapped.Content[0].Text) - } - return raw -} - -// deliveriesStalled is what the owner says is held past its bound; nothing when no holder is on record, -// or none answers (D3 says that one). -func deliveriesStalled(ctx context.Context, conn *nats.Conn, held bool) ([]stalledLine, error) { - if !held { - return nil, nil - } - raw, err := askDeliveryOwner(ctx, conn, "stalled", map[string]any{}) - if errors.Is(err, link.ErrNothingServes) { - return nil, nil // the holder not running is D3's holder-silent, said once there - } - if err != nil { - return nil, err - } - var lines []stalledLine - if err := json.Unmarshal(raw, &lines); err != nil { - return nil, fmt.Errorf("%s.stalled answered something unreadable: %w", catalogue.DeliverySeat, err) - } - return lines, nil -} - -// walkWaits are, by walk, the lines mesh-delivery says of a walk waiting for its word: what holds it. -type walkWaits map[string][]stalledLine - -// walksWaiting asks the delivery's owner what holds each walk waiting for its word (novox/hq issue 529): its -// `stalled` asked with `walks`, so a line of `plans` names a held delivery, its reason and the verb that releases -// it, instead of pointing at a hand start. Nil when no holder is on record or none answers: the line then reads as -// before. An owner from before this answers its stalled lines, of which only those naming a walk are kept. -func walksWaiting(ctx context.Context, conn *nats.Conn, held bool) walkWaits { - if !held { - return nil - } - raw, err := askDeliveryOwner(ctx, conn, "stalled", map[string]any{"walks": true}) - if err != nil { - return nil - } - var lines []stalledLine - if json.Unmarshal(raw, &lines) != nil { - return nil - } - out := walkWaits{} - for _, l := range lines { - if l.Walk != "" { - out[l.Walk] = append(out[l.Walk], l) - } - } - return out -} - -// stalledObservations are the conditions of what is stalled. -func stalledObservations(lines []stalledLine) []conditions.Observation { - out := make([]conditions.Observation, 0, len(lines)) - for _, l := range lines { - o := conditions.Observation{Scope: conditions.ScopeDelivery, ID: l.ID, Kind: kindDeliveryStalled, - Severity: l.severity(), - Summary: fmt.Sprintf("the delivery %s has been %s for %s, past its bound of %s (%s): healer H2 may %s — "+ - "`mesh-delivery.show %s`", l.ID, l.State, l.For, l.Bound, l.Says, l.H2, l.ID), - Said: fmt.Sprintf("%s for %s", l.State, l.For)} - if l.operatorsOnly() { - o.Resolver = conditions.ResolverOperator - } - o.Headline, o.Explanation, o.Resolved, o.Needs, o.Actions = stalledWords(l, o) - out = append(out, o) - } - return out -} - -// probeDeliveries is D14: every delivery held past its state's bound is said. -func probeDeliveries(ctx context.Context, d *doctor) ([]conditions.Observation, error) { - entries, err := d.open.inventory.Catalogued(ctx) - if err != nil { - return nil, err - } - var conn *nats.Conn - if d.js != nil { - conn = d.js.Conn() - } - lines, err := deliveriesStalled(ctx, conn, deliverySeatHeld(entries)) - if err != nil { - return nil, err - } - return stalledObservations(lines), nil -} - -// --- H2, for a delivery --------------------------------------------------------------------------------- - -// connOf is the bus a healer asks over. -func (h *healing) connOf() *nats.Conn { - if h.js == nil { - return nil - } - return h.js.Conn() -} - -// appliesToAStalledDelivery is H2's for a delivery: the owner still lists it, with a transition the table -// lets H2 take; never one whose state is the operator's. -func appliesToAStalledDelivery(ctx context.Context, h *healing, c conditions.Condition) (string, bool, string, error) { - lines, err := deliveriesStalled(ctx, h.connOf(), true) - if err != nil { - return "", false, "", err - } - for _, l := range lines { - if l.ID != c.Subject.ID { - continue - } - if l.operatorsOnly() { - return "", false, "the delivery is " + l.State + ", a state the table leaves to the operator", nil - } - return c.Key, true, "", nil - } - return "", false, "the delivery's owner no longer lists it as stalled: its condition clears on the next look", nil -} - -// repairDelivery is H2 for a delivery: the owner's `close`, which reads the walk again and takes only the -// transition its table names. A refusal is no repair, said; the next observation says whether it worked. -func repairDelivery(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) { - raw, err := askDeliveryOwner(ctx, h.connOf(), "close", map[string]any{"id": c.Subject.ID, "why": c.Key}) - if err != nil { - if errors.Is(err, link.ErrNothingServes) { - return "", "", err - } - return "closed nothing", oneLine(err.Error()), nil - } - var said string - if json.Unmarshal(raw, &said) != nil { - said = string(raw) - } - return "asked " + catalogue.DeliverySeat + " to close " + c.Subject.ID, said, nil -} diff --git a/cmd/mesh-controller/delivery_conditions_test.go b/cmd/mesh-controller/delivery_conditions_test.go deleted file mode 100644 index c9533fac..00000000 --- a/cmd/mesh-controller/delivery_conditions_test.go +++ /dev/null @@ -1,174 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "errors" - "slices" - "strings" - "testing" - "time" - - "github.com/nats-io/nats.go" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -// novox/hq ADR 0239 decision 9, to-be 47 Phase B: a delivery held past its bound is the controller's -// condition, read from mesh-delivery's `stalled`; H2 takes the table's transition through its `close`. - -// ownerAnswers replaces the delivery owner with one that answers stalled with these lines and records -// what close was asked. -func ownerAnswers(t *testing.T, lines []stalledLine, closeErr error) *[]string { - t.Helper() - var closed []string - was := askDeliveryOwner - askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) { - switch verb { - case "stalled": - raw, _ := json.Marshal(lines) - return raw, nil - case "close": - closed = append(closed, args["id"].(string)) - if closeErr != nil { - return nil, closeErr - } - return json.RawMessage(`"` + args["id"].(string) + `: delivering → delivered, as its walk's record says"`), nil - } - t.Fatalf("asked the owner %s", verb) - return nil, nil - } - t.Cleanup(func() { askDeliveryOwner = was }) - return &closed -} - -func holdTheDeliverySeat(t *testing.T, open *stores) { - t.Helper() - m := catalogue.Manifest{Module: "mesh-delivery", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}} - if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{Repository: "novox/mesh-catalog", - Path: "modules/mesh-delivery", BuiltFrom: "c0"}); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.Assign(t.Context(), "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } -} - -var twoStalled = []stalledLine{ - {ID: "novox/app@aaaaaaaaaaaa", State: "delivering", For: "3h0m0s", Bound: "2h0m0s", - H2: "close, by done or superseded or failed, when the walk's record says so", Says: "its walk runs"}, - {ID: "novox/lab@bbbbbbbbbbbb", State: "held", For: "49h0m0s", Bound: "24h0m0s", - H2: "none: the state is the operator's", Says: "it waits for the operator"}, -} - -func TestD14SaysEveryDeliveryHeldPastItsBound(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - ownerAnswers(t, twoStalled, nil) - d := &doctor{open: open} - got, err := probeDeliveries(ctx, d) - if err != nil || len(got) != 0 { - t.Fatalf("with no holder on record D14 said %+v %v", got, err) - } - holdTheDeliverySeat(t, open) - got, err = probeDeliveries(ctx, d) - if err != nil || len(got) != 2 { - t.Fatalf("D14 said %+v %v", got, err) - } - if got[0].Key() != "delivery.novox/app_aaaaaaaaaaaa.stalled" || got[0].Severity != conditions.Warning || - got[0].Resolver != "" || !strings.Contains(got[0].Summary, "mesh-delivery.show novox/app@aaaaaaaaaaaa") { - t.Fatalf("the first is %+v", got[0]) - } - if got[1].Resolver != conditions.ResolverOperator { - t.Fatalf("a held delivery is not the operator's: %+v", got[1]) - } - // The holder not running is D3's to say, not D14's. - was := askDeliveryOwner - askDeliveryOwner = func(context.Context, *nats.Conn, string, map[string]any) (json.RawMessage, error) { - return nil, link.ErrNothingServes - } - defer func() { askDeliveryOwner = was }() - if got, err := probeDeliveries(ctx, d); err != nil || len(got) != 0 { - t.Fatalf("an owner that is down was said by D14: %+v %v", got, err) - } -} - -// H2 for a delivery: close asked for the one whose state the table gives H2, never for the operator's; a -// refusal is no repair. -func TestH2ClosesAStalledDeliveryThroughItsOwnerOnly(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - h, told, _ := healingOn(t, open) - closed := ownerAnswers(t, twoStalled, nil) - for _, o := range stalledObservations(twoStalled) { - o.Source = probeDeliveriesID - if _, err := conditionsFrom.Observe(ctx, o); err != nil { - t.Fatal(err) - } - } - h.tick(ctx) - if !slices.Equal(*closed, []string{"novox/app@aaaaaaaaaaaa"}) { - t.Fatalf("close was asked for %v", *closed) - } - acts := told.said() - if len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "delivery.novox/app_aaaaaaaaaaaa.stalled" { - t.Fatalf("said %+v", acts) - } - // A refusal: closed nothing, said so. - c, _, _ := conditionsFrom.Get(ctx, "delivery.novox/app_aaaaaaaaaaaa.stalled") - ownerAnswers(t, twoStalled, errors.New("mesh-delivery.close refused: still delivering")) - act, said, err := repairDelivery(ctx, h, c) - if err != nil || act != "closed nothing" || !strings.Contains(said, "still delivering") { - t.Fatalf("a refused close reads %q %q %v", act, said, err) - } -} - -// The controller may ask the delivery's owner what D14 and H2 ask, on its flat subjects, and nothing else -// of it; and over a real bus the answer — wrapped as the runtime wraps it — is read. -func TestTheDeliveryOwnerIsAskedOverTheBus(t *testing.T) { - granted, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController, PasswordHash: "x"}) - if err != nil { - t.Fatal(err) - } - // And release and stop, which the operator's warrant chooses (novox/hq ADR 0259). - for _, verb := range []string{"stalled", "close", "release", "stop"} { - if !slices.Contains(granted.Publish, link.SeatToolSubject(catalogue.DeliverySeat, verb)) { - t.Errorf("the controller may not ask %s.%s", catalogue.DeliverySeat, verb) - } - } - if _, err := askDeliveryOwner(t.Context(), nil, "retire-history", nil); err == nil || !strings.Contains(err.Error(), "grant") { - t.Fatalf("a verb the grant does not name was asked: %v", err) - } - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - if _, err := deliveriesStalled(t.Context(), conn, true); err != nil { - t.Fatalf("an owner not running is D3's, not an error: %v", err) - } - lines, _ := json.Marshal(twoStalled) - wrapped, _ := json.Marshal(map[string]any{"content": []map[string]any{{"type": "text", "text": string(lines)}}}) - sub, err := conn.Subscribe(link.SeatToolSubject(catalogue.DeliverySeat, "stalled"), func(m *nats.Msg) { - body, _ := json.Marshal(link.Answer{Result: wrapped}) - _ = m.Respond(body) - }) - if err != nil { - t.Fatal(err) - } - defer func() { _ = sub.Unsubscribe() }() - ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) - defer cancel() - got, err := deliveriesStalled(ctx, conn, true) - if err != nil || len(got) != 2 || got[0].ID != "novox/app@aaaaaaaaaaaa" { - t.Fatalf("over the bus: %+v %v", got, err) - } -} diff --git a/cmd/mesh-controller/delivery_edges.go b/cmd/mesh-controller/delivery_edges.go deleted file mode 100644 index 8126201f..00000000 --- a/cmd/mesh-controller/delivery_edges.go +++ /dev/null @@ -1,170 +0,0 @@ -package main - -import ( - "fmt" - "slices" - "sort" - "strings" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// The order of a plan's modules is read from the module graph and from what the change says of the manifests -// (novox/hq ADR 0419), per module and never per repository: no `after:` line is read, and no member of a group -// is ordered against another. Pure, so a test needs no store. - -// The rules that draw an edge of a plan's order, in the words the plan shows. -const ( - ruleVerbBeforeCaller = "verb before caller" - ruleVersionSkew = "version skew" - ruleEngineFirst = "engine before controller" -) - -// The two modules the rollout rules name. -const ( - theController = "mesh-controller" - theEngine = "mesh-host" -) - -// moduleChange is what a change says of one module's manifest: the seat verbs (".") it adds to what the -// module serves, the seat verbs it adds to what the module invokes, and whether the manifest changes at all. -type moduleChange struct { - Serves []string - Invokes []string - Manifest bool -} - -// planOrder is the edges the rules draw among a plan's modules, and the cycle they make when they make one. -type planOrder struct { - Edges []inventory.Edge - // Cycle is every edge of a cycle among the order's edges and the graph's, the module graph's included. - Cycle []inventory.Edge -} - -// CycleSaid names every module and edge of the cycle, for a check to fail with (ADR 0419 clause 6). -func (o planOrder) CycleSaid() string { - if len(o.Cycle) == 0 { - return "" - } - var parts []string - for _, e := range o.Cycle { - parts = append(parts, edgeSaid(e)) - } - return "the modules of the plan are ordered in a cycle: " + strings.Join(parts, "; ") -} - -// edgeSaid is an edge as a person reads it: the module after, the module before, and why. -func edgeSaid(e inventory.Edge) string { - why := e.Why - if why == "" { - why = e.Kind - } - return fmt.Sprintf("%s after %s (%s)", e.From, e.To, why) -} - -// orders is whether an edge of the module graph orders a plan's tiers (tiersOf). -func orders(e inventory.Edge) bool { - return e.Kind != inventory.EdgePackages && e.Kind != inventory.EdgeBuiltBy && e.From != e.To -} - -// orderEdges draws the edges of a plan's order among its moved modules (ADR 0419 clause 3, 4): a module whose change -// adds a verb to a seat goes before one whose change adds a call of that verb; the controller, when the plan moves -// it, goes before every module whose manifest the plan changes, the node-engine excepted; and the node-engine goes -// before the controller by default, when the plan moves both and no edge orders them either way, transitively. An -// edge is "From after To". graph is the module graph, whose edges order too and are never drawn again. -func orderEdges(moved []string, changes map[string]moduleChange, graph []inventory.Edge) planOrder { - var out planOrder - sorted := append([]string(nil), moved...) - sort.Strings(sorted) - for _, server := range sorted { - for _, caller := range sorted { - if server == caller { - continue - } - for _, verb := range changes[server].Serves { - if slices.Contains(changes[caller].Invokes, verb) { - out.Edges = append(out.Edges, inventory.Edge{From: caller, To: server, Kind: edgeChangeOrder, - Why: ruleVerbBeforeCaller + ": " + verb}) - } - } - } - } - if slices.Contains(sorted, theController) { - for _, m := range sorted { - if m != theController && m != theEngine && changes[m].Manifest { - out.Edges = append(out.Edges, inventory.Edge{From: m, To: theController, Kind: edgeChangeOrder, - Why: ruleVersionSkew}) - } - } - } - if slices.Contains(sorted, theController) && slices.Contains(sorted, theEngine) { - all := orderingEdges(graph, out.Edges) - switch { - case pathBetween(all, theController, theEngine) != nil: - // An edge already puts the controller after the engine: the default says the same. - case pathBetween(all, theEngine, theController) != nil: - // An edge puts the engine after the controller: it overrules the default, and says so. - path := pathBetween(all, theEngine, theController) - for i := range out.Edges { - if out.Edges[i] == path[0] { - out.Edges[i].Why += " (overrules " + ruleEngineFirst + ")" - break - } - } - default: - out.Edges = append(out.Edges, inventory.Edge{From: theController, To: theEngine, Kind: edgeChangeOrder, - Why: ruleEngineFirst}) - } - } - all := orderingEdges(graph, out.Edges) - for _, e := range out.Edges { - if back := pathBetween(all, e.To, e.From); back != nil { - out.Cycle = append(out.Cycle, e) - out.Cycle = append(out.Cycle, back...) - break - } - } - return out -} - -// orderingEdges is the graph's edges that order and the edges drawn, together. -func orderingEdges(graph, drawn []inventory.Edge) []inventory.Edge { - var out []inventory.Edge - for _, e := range graph { - if orders(e) { - out = append(out, e) - } - } - return append(out, drawn...) -} - -// pathBetween is the edges that lead from a module to another along "depends on" (From to To), the shortest, or nil -// when the first does not depend on the second. -func pathBetween(edges []inventory.Edge, from, to string) []inventory.Edge { - if from == to { - return nil - } - via := map[string]inventory.Edge{} - seen := map[string]bool{from: true} - queue := []string{from} - for len(queue) > 0 { - at := queue[0] - queue = queue[1:] - for _, e := range edges { - if e.From != at || seen[e.To] { - continue - } - seen[e.To] = true - via[e.To] = e - if e.To == to { - var path []inventory.Edge - for m := to; m != from; m = via[m].From { - path = append([]inventory.Edge{via[m]}, path...) - } - return path - } - queue = append(queue, e.To) - } - } - return nil -} diff --git a/cmd/mesh-controller/delivery_edges_test.go b/cmd/mesh-controller/delivery_edges_test.go deleted file mode 100644 index 96da775d..00000000 --- a/cmd/mesh-controller/delivery_edges_test.go +++ /dev/null @@ -1,183 +0,0 @@ -package main - -import ( - "context" - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// The order of a plan's modules is read from the module graph and the manifests' change alone (novox/hq ADR 0419): -// no `after:` line, no member of a group. Pure, like orderOf was. - -func hasEdge(edges []inventory.Edge, from, to string) (inventory.Edge, bool) { - for _, e := range edges { - if e.From == from && e.To == to { - return e, true - } - } - return inventory.Edge{}, false -} - -// A module whose change adds a verb to a seat goes before the module whose change adds a call of it: the caller -// depends on the server, the plan puts them in two tiers, and the edge says why. -func TestAVerbIsServedBeforeItsCallerIsSent(t *testing.T) { - changes := map[string]moduleChange{ - "server": {Serves: []string{"mesh-delivery.groups"}, Manifest: true}, - "caller": {Invokes: []string{"mesh-delivery.groups"}, Manifest: true}, - } - got := orderEdges([]string{"caller", "server"}, changes, nil) - e, ok := hasEdge(got.Edges, "caller", "server") - if !ok || !strings.Contains(e.Why, "verb before caller") || !strings.Contains(e.Why, "mesh-delivery.groups") { - t.Fatalf("the caller does not follow the server by the verb: %+v", got.Edges) - } - if len(got.Cycle) != 0 { - t.Fatalf("a cycle: %+v", got.Cycle) - } - if tiers := tiersOf([]string{"caller", "server"}, got.Edges); len(tiers) != 2 || tiers[0][0] != "server" { - t.Fatalf("tiers %v, want the server first", tiers) - } -} - -// Edges of one rule both ways are a cycle: every module and edge is named. -func TestEdgesBothWaysAreACycleNamingEveryModuleAndEdge(t *testing.T) { - changes := map[string]moduleChange{ - "x": {Serves: []string{"s.a"}, Invokes: []string{"s.b"}}, - "y": {Serves: []string{"s.b"}, Invokes: []string{"s.a"}}, - } - got := orderEdges([]string{"x", "y"}, changes, nil) - if len(got.Cycle) < 2 { - t.Fatalf("no cycle found: %+v", got) - } - said := got.CycleSaid() - for _, want := range []string{"x", "y", "s.a", "s.b"} { - if !strings.Contains(said, want) { - t.Fatalf("the cycle is said as %q, without %s", said, want) - } - } -} - -// The node-engine goes before the controller by default, and says so; an edge of the manifests the other way -// overrules it, and the edge that did is said. -func TestTheEngineGoesBeforeTheControllerUnlessAnEdgeSaysOtherwise(t *testing.T) { - got := orderEdges([]string{"mesh-controller", "mesh-host"}, nil, nil) - e, ok := hasEdge(got.Edges, "mesh-controller", "mesh-host") - if !ok || e.Why != "engine before controller" { - t.Fatalf("the engine does not go first: %+v", got.Edges) - } - changes := map[string]moduleChange{ - "mesh-host": {Serves: []string{"s.v"}}, - "mesh-controller": {Invokes: []string{"s.v"}}, - } - got = orderEdges([]string{"mesh-controller", "mesh-host"}, changes, nil) - if _, ok := hasEdge(got.Edges, "mesh-controller", "mesh-host"); !ok || len(got.Edges) != 1 || - !strings.Contains(got.Edges[0].Why, "verb before caller") { - t.Fatalf("the verb edge does not place the controller after the engine: %+v", got.Edges) - } - // The other way: the engine calls what the controller brings. - changes = map[string]moduleChange{ - "mesh-controller": {Serves: []string{"s.v"}}, - "mesh-host": {Invokes: []string{"s.v"}}, - } - got = orderEdges([]string{"mesh-controller", "mesh-host"}, changes, nil) - e, ok = hasEdge(got.Edges, "mesh-host", "mesh-controller") - if !ok || len(got.Edges) != 1 || !strings.Contains(e.Why, "overrules engine before controller") { - t.Fatalf("the verb edge does not say it overruled the default: %+v", got.Edges) - } -} - -// The controller goes before every module whose manifest the plan changes (version skew), the node-engine excepted. -func TestTheControllerGoesBeforeEveryChangedManifest(t *testing.T) { - changes := map[string]moduleChange{"app": {Manifest: true}, "other": {}} - got := orderEdges([]string{"mesh-controller", "app", "other"}, changes, nil) - if e, ok := hasEdge(got.Edges, "app", "mesh-controller"); !ok || e.Why != "version skew" { - t.Fatalf("the changed manifest does not follow the controller: %+v", got.Edges) - } - if _, ok := hasEdge(got.Edges, "other", "mesh-controller"); ok { - t.Fatalf("a module with an unchanged manifest was ordered: %+v", got.Edges) - } - if got := orderEdges([]string{"app", "other"}, changes, nil); len(got.Edges) != 0 { - t.Fatalf("version skew ordered a plan that does not move the controller: %+v", got.Edges) - } -} - -// The order is per module, not per repository: of two modules of one repository only the one with the edge moves. -func TestTheOrderIsPerModuleNotPerRepository(t *testing.T) { - changes := map[string]moduleChange{ - "a": {Serves: []string{"s.v"}}, - "c": {Invokes: []string{"s.v"}}, - } - got := orderEdges([]string{"a", "b", "c"}, changes, nil) - tiers := tiersOf([]string{"a", "b", "c"}, got.Edges) - if len(tiers) != 2 || strings.Join(tiers[0], ",") != "a,b" || strings.Join(tiers[1], ",") != "c" { - t.Fatalf("tiers %v, want [a b] then [c]", tiers) - } -} - -// The edges a plan's order was drawn with are kept on the plan, and its gates read them: a later tier waits for the -// earlier to be applied although the catalogue's graph holds no such edge (ADR 0419, the lead's settlement Q4). -func TestAPlansOwnOrderGatesItsTiersWhateverTheStoreGraphHolds(t *testing.T) { - p := inventory.Plan{Tiers: [][]string{{"server"}, {"caller"}}, Tier: 0, - Delivery: &inventory.PlanDelivery{Order: []inventory.Edge{{From: "caller", To: "server", Kind: edgeChangeOrder, - Why: ruleVerbBeforeCaller + ": s.v"}}}} - if got := gates(p, nil, func(string) bool { return true }); len(got) != 1 || got[0] != "server" { - t.Fatalf("the caller's tier waits for %v; want the server", got) - } - p.Delivery.Order = nil - if got := gates(p, nil, func(string) bool { return true }); len(got) != 0 { - t.Fatalf("a plan with no order waits for %v", got) - } -} - -// A verb an earlier merge of the same repository in the batch added is added by the batch: the change is compared with -// what the repository held before the batch, not with the first parent of its latest merge. -func TestAVerbAnEarlierMergeOfTheBatchAddedDrawsItsEdge(t *testing.T) { - was, wasParent := manifestAt, parentOf - t.Cleanup(func() { manifestAt, parentOf = was, wasParent }) - manifestsReadMu.Lock() - clear(manifestsRead) - manifestsReadMu.Unlock() - const first, second, before = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222", "0000000000000000000000000000000000000000" - parentOf = func(_ context.Context, _, _, commit string) (string, error) { - if commit == first { - return before, nil - } - return first, nil // the latest merge's parent holds the verb already - } - manifestAt = func(_ context.Context, _, _, _, ref string) (catalogue.Manifest, bool, error) { - m := catalogue.Manifest{Module: "server"} - if ref != before { - m.DefinesSeats = []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "v"}}}} - } - return m, true, nil - } - src := changeSource{Owner: "novox", Repo: "r", New: second, Paths: []string{"module.json"}} - if got, err := changesOf(context.Background(), []changeSource{src}); err != nil || len(got["server"].Serves) != 0 { - t.Fatalf("against the first parent of the latest merge: %+v %v", got, err) - } - src.Since = first - got, err := changesOf(context.Background(), []changeSource{src}) - if err != nil || len(got["server"].Serves) != 1 || got["server"].Serves[0] != "s.v" { - t.Fatalf("against what the batch found: %+v %v", got, err) - } -} - -// A change that names only a beginning of its files is read as changing the manifests of the modules it moves. -func TestATruncatedChangeIsReadAsChangingTheManifestsOfWhatItMoves(t *testing.T) { - entries := []inventory.Entry{ - {Manifest: catalogue.Manifest{Module: "a"}, Source: inventory.Source{Repository: "novox/mesh-catalog", Path: "modules/a"}}, - {Manifest: catalogue.Manifest{Module: "b"}, Source: inventory.Source{Repository: "novox/mesh-catalog", Path: "modules/b"}}, - {Manifest: catalogue.Manifest{Module: "c"}, Source: inventory.Source{Repository: "novox/other", Path: "modules/c"}}, - } - got := truncatedPaths(entries, "novox", "mesh-catalog", []string{"a", "c"}) - if len(got) != 1 || got[0] != "modules/a/module.json" { - t.Fatalf("read as changed: %v", got) - } - members := []orderMember{{ID: "m", Repository: "novox/mesh-catalog", Head: "h", PathsTruncated: true, Paths: []string{"x"}}} - src := changeSourcesOf(members, entries, map[string]orderReach{"m": {Moved: []string{"a"}}}) - if len(src) != 1 || src[0].Truncated || len(src[0].Paths) != 1 || src[0].Paths[0] != "modules/a/module.json" { - t.Fatalf("the member's change is read as %+v", src) - } -} diff --git a/cmd/mesh-controller/delivery_integration_test.go b/cmd/mesh-controller/delivery_integration_test.go deleted file mode 100644 index fa0c04cf..00000000 --- a/cmd/mesh-controller/delivery_integration_test.go +++ /dev/null @@ -1,216 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := asksRecorded(t) - withConditionsInMemory(t) - for _, name := range []string{"app", "mesh-delivery"} { - m := catalogue.Manifest{Module: name, Version: "1"} - if name == "mesh-delivery" { - m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}} - } - if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", - Path: "modules/" + name, Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - } - merge := func(commit string, paths ...string) inventory.Plan { - t.Helper() - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, Paths: paths, - ModuleDirs: []string{"modules/app", "modules/mesh-delivery"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - recent, err := inv.RecentPlans(ctx, 1) - if err != nil || len(recent) != 1 || recent[0].Commit != commit { - t.Fatalf("no plan for %s: %v %v", commit, recent, err) - } - return recent[0] - } - - // finish ends a walk as done, so the next merge has nothing of it to take over. - finish := func(id string) { - t.Helper() - w, err := inv.PlanByID(ctx, id) - if err != nil { - t.Fatal(err) - } - w.State = inventory.PlanDone - if err := inv.SavePlan(ctx, &w); err != nil { - t.Fatal(err) - } - } - - // heardOf has this mesh's own view of the plans take in the plan as the bucket holds it. A verb of the - // controller (deliveryCommand, plansCommand) opens an inventory of its own and writes the plan there; this - // mesh's inventory learns of the write from the bucket's watch, a moment later, and a loaded machine makes the - // moment long enough for advanceHeld to read the plan as it was, still waiting for its word, and ask nothing - // (issue 1399). Reading the plan from the bucket puts it into the view at once, with no wait to bound. - heardOf := func(id string) { - t.Helper() - if _, err := inv.PlanByID(ctx, id); err != nil { - t.Fatal(err) - } - } - - // No holder on record: the merge starts its walk, as before. - // A cut takes its first tier from the builder's keys and a step of the walk records the take (the walk asks no - // build), so each walk that starts is stepped once before its takes are counted. - p := merge("c1aaaaaaaa", "modules/app/index.ts") - advanceHeld(ctx, open) - if p.Waiting() || len(*asked) != 1 { - t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked) - } - - // The holder on record: the next merge waits, asking nothing, and an advance asks nothing either. One walk - // is open at a time (novox/hq ADR 0276): the first ends before the next merge's batch is cut. - finish(p.ID) - if _, err := inv.Assign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - p = merge("c2bbbbbbbb", "modules/app/index.ts") - if !p.Waiting() || len(*asked) != 1 || !strings.Contains(p.Note, "plans go "+p.ID) { - t.Fatalf("the walk did not wait for its word: waiting %v, asked %v, note %q", p.Waiting(), *asked, p.Note) - } - advanceHeld(ctx, open) - if len(*asked) != 1 { - t.Fatalf("a waiting walk was advanced into asking: %v", *asked) - } - if line := planLine(p, p.Created); strings.Contains(line, "LATE") || !strings.Contains(line, "waits for") { - t.Errorf("a waiting walk reads %q", line) - } - - // The delivery's word starts it. - if err := deliveryCommand(ctx, []string{"go", p.ID, "--by", catalogue.DeliverySeat, "--why", "its turn"}); err != nil { - t.Fatal(err) - } - if err := deliveryCommand(ctx, []string{"go", p.ID}); err == nil { - t.Fatal("a walk was let go twice") - } - heardOf(p.ID) - advanceHeld(ctx, open) - if len(*asked) != 2 { - t.Fatalf("the word did not start the walk: %v", *asked) - } - got, err := inv.PlanByID(ctx, p.ID) - if err != nil || got.Delivery == nil || got.Delivery.By != catalogue.DeliverySeat || got.Delivery.Why != "its turn" { - t.Fatalf("the word was not kept: %+v %v", got.Delivery, err) - } - - // The delivery's owner's own merge never waits for it. A walk that started is never taken over (novox/hq - // ADR 0276): the merge's batch is cut once it ended. - finish(p.ID) - p = merge("c3cccccccc", "modules/mesh-delivery/main.go") - advanceHeld(ctx, open) - if p.Waiting() || len(*asked) != 3 { - t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked) - } - - // The owner down: a person starts a waiting walk, with why. - finish(p.ID) - p = merge("c4dddddddd", "modules/app/index.ts") - if !p.Waiting() { - t.Fatal("the walk did not wait") - } - if err := plansCommand(ctx, []string{"go", p.ID}); err == nil || !strings.Contains(err.Error(), "--why") { - t.Fatalf("a walk was started by hand without why: %v", err) - } - if err := plansCommand(ctx, []string{"go", p.ID, "--why", "mesh-delivery is down"}); err != nil { - t.Fatal(err) - } - heardOf(p.ID) - advanceHeld(ctx, open) - got, _ = inv.PlanByID(ctx, p.ID) - if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 4 { - t.Fatalf("a person's word did not start the walk: %+v, asked %v, state %q, tier %d, note %q", got.Delivery, *asked, got.State, got.Tier, got.Note) - } - - // The owner's stop: failed, said as stopped by it. - finish(p.ID) - p = merge("c5eeeeeeee", "modules/app/index.ts") - if err := deliveryCommand(ctx, []string{"stop", p.ID, "--why", "the operator stopped it", "--by", - "mesh-delivery for jochen"}); err != nil { - t.Fatal(err) - } - got, _ = inv.PlanByID(ctx, p.ID) - if got.State != inventory.PlanFailed || got.Delivery.Stopped != "mesh-delivery for jochen" || - !strings.Contains(got.Note, "the operator stopped it") { - t.Fatalf("the stop was kept as %s %+v %q", got.State, got.Delivery, got.Note) - } - - // Unassigned: the mesh is back on the controller's own path. - if err := inv.Unassign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - asks := len(*asked) - p = merge("c6ffffffff", "modules/app/index.ts") - advanceHeld(ctx, open) - if p.Waiting() || len(*asked) != asks+1 { - t.Fatalf("with the holder gone the walk waited: %v", p.Waiting()) - } -} - -func TestAWalkLetGoByAVerbIsSaidAsPlanMoved(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - url := testbus.URL(t) - js, err := broker.Dial(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - before := handActConn - handActConn = js.Conn() - t.Cleanup(func() { handActConn = before }) - heard := make(chan *nats.Msg, 4) - sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - - waiting := inventory.Plan{ID: "plan-waits", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c7c7c7c7", - Created: time.Now().UTC(), State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {}}, - Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}} - if err := open.inventory.SavePlan(ctx, &waiting); err != nil { - t.Fatal(err) - } - if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil { - t.Fatal(err) - } - select { - case m := <-heard: - var said inventory.Plan - if err := json.Unmarshal(m.Data, &said); err != nil || said.ID != waiting.ID || said.Delivery == nil || - said.Delivery.Go == nil { - t.Fatalf("plan-moved said %s (%v)", m.Data, err) - } - case <-time.After(5 * time.Second): - t.Fatal("a walk let go by a verb was not said") - } - if checkEvents != nil || inventory.PlanSaved != nil { - t.Fatal("the command's own bus was left in place") - } -} diff --git a/cmd/mesh-controller/delivery_order_test.go b/cmd/mesh-controller/delivery_order_test.go deleted file mode 100644 index e5776e50..00000000 --- a/cmd/mesh-controller/delivery_order_test.go +++ /dev/null @@ -1,152 +0,0 @@ -package main - -import ( - "context" - "errors" - "reflect" - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/link" -) - -// recordingDelivery is a delivery that writes down what was done, in order, and fails where told. -type recordingDelivery struct { - did []string - grantErr error - declareErr error -} - -func (r *recordingDelivery) grant(_ context.Context, sending []readyNode) error { - for _, s := range sending { - r.did = append(r.did, "grant "+s.node) - } - return r.grantErr -} - -func (r *recordingDelivery) declare(_ context.Context, s readyNode, _ []byte) (string, error) { - if r.declareErr != nil { - return "", r.declareErr - } - r.did = append(r.did, "declare "+s.node) - return "digest-" + s.node, nil -} - -func ready(names ...string) []readyNode { - var out []readyNode - for _, n := range names { - out = append(out, readyNode{node: n, declared: sendable{Resources: []map[string]any{{"id": "x"}}}}) - } - return out -} - -// novox/hq issue 249: the grants that come with a module's new declarations are issued before any -// machine is sent the code that uses them — except the machine holding the bus, whose declaration -// carries the controller's own right to issue them, and goes first. -func TestGrantsAreIssuedBeforeTheDeclarations(t *testing.T) { - d := &recordingDelivery{} - digests, err := deliver(t.Context(), d, "", ready("anchor", "laptop")) - if err != nil { - t.Fatal(err) - } - want := []string{"grant anchor", "grant laptop", "declare anchor", "declare laptop"} - if !reflect.DeepEqual(d.did, want) { - t.Fatalf("delivered in the order %v, wanted %v", d.did, want) - } - if digests["anchor"] != "digest-anchor" || digests["laptop"] != "digest-laptop" { - t.Fatalf("the digests sent were not answered: %v", digests) - } - - held := &recordingDelivery{} - if _, err := deliver(t.Context(), held, "broker", ready("broker", "anchor")); err != nil { - t.Fatal(err) - } - want = []string{"declare broker", "grant broker", "grant anchor", "declare anchor"} - if !reflect.DeepEqual(held.did, want) { - t.Fatalf("with the bus's machine in the send: %v, wanted %v", held.did, want) - } -} - -// A grant that cannot be issued holds back the machines it concerns and is an error the caller -// retries on — never "until the next push" — and the bus's own machine is sent regardless, so the -// grant that would let the controller issue memberships is never held behind them. -func TestAGrantThatFailsHoldsBackWhatItConcerns(t *testing.T) { - // A failure naming no machine (the buckets): everything but the bus's machine. - d := &recordingDelivery{grantErr: errors.New("the bus refused the bucket")} - _, err := deliver(t.Context(), d, "broker", ready("broker", "anchor", "laptop")) - if err == nil || !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "the bus refused the bucket") || - !strings.Contains(err.Error(), "anchor, laptop not sent") { - t.Fatalf("a failed grant was not said as the send's failure: %v", err) - } - if want := []string{"declare broker", "grant broker", "grant anchor", "grant laptop"}; !reflect.DeepEqual(d.did, want) { - t.Fatalf("delivered %v, wanted the bus's machine alone", d.did) - } - - // A membership that failed for one machine: that machine alone. - one := &recordingDelivery{grantErr: &grantsRefused{nodes: map[string]error{"laptop": errors.New("no")}}} - _, err = deliver(t.Context(), one, "", ready("anchor", "laptop")) - if !errors.Is(err, errGrants) || !strings.Contains(err.Error(), "laptop not sent") { - t.Fatalf("one machine's refused membership was not said: %v", err) - } - if want := []string{"grant anchor", "grant laptop", "declare anchor"}; !reflect.DeepEqual(one.did, want) { - t.Fatalf("delivered %v, wanted anchor sent and laptop held back", one.did) - } - - // The announced upgrade that hit it is asked again. - if !errors.Is(askAgainOnGrants(err), link.ErrTryAgain) { - t.Fatal("an announcement whose send stopped at its grants is not asked again") - } - if other := errors.New("laptop could not be resolved"); errors.Is(askAgainOnGrants(other), link.ErrTryAgain) { - t.Fatal("any failure is asked again, not only a grant's") - } - - // Nothing to send is nothing granted either. - none := &recordingDelivery{grantErr: errors.New("never asked")} - if _, err := deliver(t.Context(), none, "", nil); err != nil || len(none.did) != 0 { - t.Fatalf("an empty send granted or failed: %v %v", none.did, err) - } -} - -// Whether the bus's machine goes first is read from the user list alone, by its digest. -func TestTheBusMachineIsBehindByItsUserListAlone(t *testing.T) { - list := "users: [a, b]" - if userListBehind(list, digestOf([]byte(list))) { - t.Fatal("the list it was sent reads as behind") - } - if !userListBehind(list, digestOf([]byte("users: [a]"))) || !userListBehind(list, "") { - t.Fatal("a changed or never-sent list reads as current") - } - if userListBehind("", "") { - t.Fatal("a machine sent no list reads as behind") - } - // A list that could not be composed is behind: not known is never "not behind" (the review of hq issue 353). - if !listBehind("", errors.New("the plan cannot be worked out"), digestOf([]byte(list))) { - t.Fatal("a list nobody could compose reads as current") - } - if listBehind(list, nil, digestOf([]byte(list))) || !listBehind(list, nil, "") { - t.Fatal("listBehind does not read the digest as userListBehind does") - } -} - -// The machine holding the bus goes first: its declaration carries the user list the new grants are -// checked against. Among the machines it is moved to the front; not among them it is added only -// when it is behind. -func TestTheMachineHoldingTheBusIsSentFirst(t *testing.T) { - for _, c := range []struct { - what string - names []string - holder string - behind bool - want []string - }{ - {"among them", []string{"ace", "g14", "novox"}, "novox", false, []string{"novox", "ace", "g14"}}, - {"not among them, behind", []string{"ace", "g14"}, "novox", true, []string{"novox", "ace", "g14"}}, - {"not among them, current", []string{"ace", "g14"}, "novox", false, []string{"ace", "g14"}}, - {"nothing holds the bus", []string{"ace", "g14"}, "", true, []string{"ace", "g14"}}, - {"only it", []string{"novox"}, "novox", false, []string{"novox"}}, - } { - if got := brokerFirst(c.names, c.holder, c.behind); !reflect.DeepEqual(got, c.want) { - t.Errorf("%s: sent in the order %v, wanted %v", c.what, got, c.want) - } - } -} diff --git a/cmd/mesh-controller/delivery_stalled_severity_test.go b/cmd/mesh-controller/delivery_stalled_severity_test.go deleted file mode 100644 index d59eb346..00000000 --- a/cmd/mesh-controller/delivery_stalled_severity_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package main - -import ( - "encoding/json" - "testing" - - "github.com/novox/mesh-controller/internal/conditions" -) - -func severityRaisedFor(t *testing.T, line string) conditions.Severity { - t.Helper() - var l stalledLine - if err := json.Unmarshal([]byte(line), &l); err != nil { - t.Fatal(err) - } - got := stalledObservations([]stalledLine{l}) - if len(got) != 1 { - t.Fatalf("%d conditions", len(got)) - } - return got[0].Severity -} - -func TestAnUrgentStalledLineRaisesAnUrgentCondition(t *testing.T) { // 1278/AC1 - line := `{"id":"novox/mesh-delivery#5","state":"held","for":"3m0s","bound":"0s","h2":"close","says":"held","severity":"urgent"}` - if got := severityRaisedFor(t, line); got != conditions.Urgent { - t.Errorf("an urgent line raised %q", got) - } -} - -func TestAStalledLineWithoutOrWithAnUnknownSeverityRaisesAWarning(t *testing.T) { // 1278/AC2 - for _, extra := range []string{``, `,"severity":""`, `,"severity":"critical"`, `,"severity":"warning"`} { - line := `{"id":"novox/a#1","state":"unanswered","for":"3m0s","bound":"2m0s","h2":"close","says":"x"` + extra + `}` - if got := severityRaisedFor(t, line); got != conditions.Warning { - t.Errorf("the line %s raised %q", extra, got) - } - } -} diff --git a/cmd/mesh-controller/delivery_test.go b/cmd/mesh-controller/delivery_test.go deleted file mode 100644 index 05cfc7a3..00000000 --- a/cmd/mesh-controller/delivery_test.go +++ /dev/null @@ -1,155 +0,0 @@ -package main - -import ( - "reflect" - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq ADR 0419: a group's order, from the module graph and the manifests' change alone, the same on every -// reading — what a module stands on before it, the controller before a manifest that needs it, the node-engine -// before the controller, a verb's server before its caller — and no `after:` line read. -func TestAGroupIsOrderedByTheGraphAndTheManifestsChange(t *testing.T) { - members := []orderMember{ - {ID: "cat", Repository: "novox/mesh-catalog"}, - {ID: "ctl", Repository: "novox/mesh-controller"}, - {ID: "host", Repository: "novox/mesh-host"}, - {ID: "base", Repository: "novox/base"}, - {ID: "app", Repository: "novox/app"}, - {ID: "lab", Repository: "novox/lab"}, - } - reach := map[string]orderReach{ - "cat": {Moved: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}}, - "ctl": {Moved: []string{"mesh-controller"}, Manifests: []string{"module.json"}}, - "host": {Moved: []string{"mesh-host"}, Manifests: []string{"module.json"}}, - "base": {Moved: []string{"base"}}, - "app": {Moved: []string{"app"}}, - "lab": {Moved: []string{"lab"}}, - } - edges := []inventory.Edge{{From: "app", To: "base", Kind: inventory.EdgeStandsOn}, - {From: "gitea", To: "postgres", Kind: inventory.EdgeDeclared}} - changes := map[string]moduleChange{"gitea": {Manifest: true}, "mesh-controller": {Manifest: true}, - "mesh-host": {Manifest: true}, "lab": {Serves: []string{"seat.v"}}, "app": {Invokes: []string{"seat.v"}}} - got := orderOf(members, reach, edges, changes) - if len(got.Cycle) > 0 { - t.Fatalf("a cycle where there is none: %v", got.Cycle) - } - // By repository among those with nothing before them: app waits for base and lab; gitea's catalogue for the controller. - want := []string{"base", "lab", "app", "host", "ctl", "cat"} - if !reflect.DeepEqual(got.Order, want) { - t.Fatalf("ordered %v, wanted %v; pairs %+v", got.Order, want, got.Pairs) - } - why := map[string]string{} - for _, p := range got.Pairs { - why[p.Before+">"+p.After] = p.Why - } - for pair, reason := range map[string]string{"host>ctl": ruleEngineFirst, "ctl>cat": ruleVersionSkew, - "ctl>host": "", "base>app": orderBuiltBy, "lab>app": ruleVerbBeforeCaller + ": seat.v"} { - if why[pair] != reason { - t.Errorf("%s is ordered %q, wanted %q", pair, why[pair], reason) - } - } - // The same members in another order read the same. - shuffled := []orderMember{members[5], members[3], members[0], members[4], members[2], members[1]} - if again := orderOf(shuffled, reach, edges, changes); !reflect.DeepEqual(again.Order, want) { - t.Fatalf("another reading ordered %v", again.Order) - } -} - -// A cycle is the one refusal, and it names every module and edge. A description's `after:` line changes nothing. -func TestAGroupInACycleIsNamedAndNothingElseIsRefused(t *testing.T) { - members := []orderMember{ - {ID: "x", Repository: "novox/x", Number: 7}, - {ID: "y", Repository: "novox/y", Number: 9}, - } - reach := map[string]orderReach{"x": {Moved: []string{"mx"}}, "y": {Moved: []string{"my"}}} - changes := map[string]moduleChange{ - "mx": {Serves: []string{"s.a"}, Invokes: []string{"s.b"}}, - "my": {Serves: []string{"s.b"}, Invokes: []string{"s.a"}}, - } - got := orderOf(members, reach, nil, changes) - if !reflect.DeepEqual(got.Cycle, []string{"x", "y"}) || len(got.Order) > 0 { - t.Fatalf("a cycle was ordered: %v, cycle %v", got.Order, got.Cycle) - } - for _, w := range []string{"mx", "my", "s.a", "s.b"} { - if !strings.Contains(got.CycleSaid, w) { - t.Errorf("the cycle does not say %q: %s", w, got.CycleSaid) - } - } - // Without the edges the same members are ordered by repository. - got = orderOf(members, reach, nil, nil) - if !reflect.DeepEqual(got.Order, []string{"x", "y"}) || len(got.Cycle) > 0 || len(got.Pairs) > 0 { - t.Fatalf("members with no edge: %v %v %+v", got.Order, got.Cycle, got.Pairs) - } -} - -// novox/hq ADR 0239 decision 8: a walk waits for the delivery's owner only while the seat has a holder on -// record, and never one that moves a module on the controller's own path. -func TestAWalkWaitsOnlyWhileTheDeliverySeatIsHeldAndNeverForTheCore(t *testing.T) { - holder := inventory.Entry{Manifest: catalogue.Manifest{Module: "mesh-delivery", - Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}} - unassigned := holder - unassigned.On = nil - for _, c := range []struct { - entries []inventory.Entry - moved []string - waits bool - }{ - {nil, []string{"gitea"}, false}, - {[]inventory.Entry{unassigned}, []string{"gitea"}, false}, - {[]inventory.Entry{holder}, []string{"gitea", "plex"}, true}, - {[]inventory.Entry{holder}, []string{"gitea", "mesh-controller"}, false}, - {[]inventory.Entry{holder}, []string{"mesh-host"}, false}, - {[]inventory.Entry{holder}, []string{"node-tools"}, false}, - {[]inventory.Entry{holder}, []string{"nats"}, false}, - {[]inventory.Entry{holder}, []string{"mesh-delivery", "gitea"}, false}, - } { - d := awaitsFor(c.entries, c.moved) - if (d != nil) != c.waits { - t.Errorf("held %v, moving %v: waits %v, wanted %v", len(c.entries) > 0 && len(c.entries[0].On) > 0, - c.moved, d != nil, c.waits) - } - if d != nil && d.Awaits != catalogue.DeliverySeat { - t.Errorf("waits for %q", d.Awaits) - } - } -} - -// novox/hq ADR 0239: with mesh-delivery on record, a merge opens its walk and asks nothing until the -// delivery's word; the word starts it; the core's own merge never waits; a person starts a waiting walk by -// hand when the owner is down, and the owner's stop ends one as stopped. - -// The verbs mesh-delivery asks with become the commands they name, and nothing a caller sends is passed over. -func TestTheDeliveryVerbsComposeTheirCommands(t *testing.T) { - for _, c := range []struct { - verb string - args map[string]any - want []string - }{ - {"deliver", map[string]any{"plan": "plan-1", "why": "its turn"}, - []string{"delivery", "go", "plan-1", "--by", "mesh-delivery", "--why", "its turn"}}, - {"delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen"}, - []string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen"}}, - {"delivery-walks", map[string]any{}, []string{"delivery", "walks"}}, - {"delivery-walks", map[string]any{"plan": "plan-1"}, []string{"delivery", "walks", "--plan", "plan-1"}}, - {"delivery-order", map[string]any{"members": "[]"}, []string{"delivery", "order", "--members", "[]"}}, - {"plans", map[string]any{"go": "plan-1", "why": "down"}, []string{"plans", "go", "plan-1", "--why", "down"}}, - } { - got, err := argvFor(c.verb, c.args) - if err != nil || !reflect.DeepEqual(got, c.want) { - t.Errorf("%s %v → %v %v, wanted %v", c.verb, c.args, got, err, c.want) - } - } - if _, err := argvFor("deliver", map[string]any{}); err == nil { - t.Error("deliver without a walk was composed") - } - if _, err := argvFor("delivery-stop", map[string]any{"plan": "p"}); err == nil { - t.Error("a stop without why was composed") - } -} - -// A verb runs as a command of its own: what it keeps of a walk is still said as plan-moved, on a bus of the -// command's own, so the delivery's owner hears it at once and not only when it reads the walks back. diff --git a/cmd/mesh-controller/doctor.go b/cmd/mesh-controller/doctor.go index 475d33fb..45659067 100644 --- a/cmd/mesh-controller/doctor.go +++ b/cmd/mesh-controller/doctor.go @@ -115,26 +115,11 @@ var probeRegistry = []probe{ kindDataUnmeasured, kindDataMissing, kindArrayDegraded, kindProtectionMissing, kindCleanupWaiting}, Phase: 2, run: probeData, Asks: []broker.SeatVerb{{Seat: "node-backup", Verb: "backed-up"}}}, - // The delivery's owner (novox/hq ADR 0239): what it holds past a bound of its own table is said here, by - // the controller, and H2 works it through the owner's `close`. - {ID: probeDeliveriesID, Asserts: "no delivery is held past its state's bound unsaid: mesh-delivery's " + - "`stalled`, each with the transition its table lets healer H2 take", From: "ADR 0239", - Kind: kindDeliveryStalled, Phase: 3, run: probeDeliveries}, // What the watchman says stopped moving (ADR 0433 §2, issue 1156): each key of its stuck bucket not moved, but a // node's presence, which the silent check takes. {ID: probeStuckID, Asserts: "no key the watchman says stopped moving is left unsaid: each key of its stuck bucket " + "not moved is a condition of the delivery scope until it says moved", From: "ADR 0433 §2, issue 1156", Kind: kindStuck, Phase: 3, run: probeStuck}, - // A first merge into a repository the mesh builds nothing from, kept waiting for mesh-delivery's word on the - // operator's Approve (novox/hq ADR 0432, issue 1146): said past a bound, never only in a log line (ADR 0426 §3, §8). - {ID: probeFirstBuildsID, Asserts: "no first merge into a repository the mesh builds nothing from waits more than ten " + - "minutes for mesh-delivery's announcement of the operator's Approve of its head", From: "ADR 0432, issue 1146", - Kind: kindFirstBuildUnasked, Phase: 3, run: probeFirstBuilds}, - // The delivery budgets (novox/hq ADR 0282 decision 7): the newest delivery of each class within its budget, - // read from mesh-delivery's `times`; a measurement said, never a delivery held. - {ID: probeBudgetsID, Asserts: "the newest delivery of a leaf module ran on every machine within three minutes of " + - "its merge, and of a core module within ten: mesh-delivery's `times`", From: "ADR 0282, ADR 0304", - Kind: kindOverBudget, Phase: 3, run: probeBudgets}, // A client of the bus reconnecting in a loop (novox/hq issue 327), from the server's record of closed // connections, which the bus's own module reads. {ID: probeReconnectsID, Asserts: "no user of the bus had its connection dropped more than twelve times in the " + @@ -168,30 +153,13 @@ var probeRegistry = []probe{ {ID: "H-controller", Asserts: "the controller lease is held, renewed in time, by a controller that says it is " + "ready: its self-check ran and status answered in full within ten seconds", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeControllerHealth}, - {ID: "H-engine", Asserts: "every machine heard from has reported its current declaration, under a node-engine " + - "build it names", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeEngineHealth}, - {ID: "H-tools", Asserts: "every machine heard from that runs the node tools has them answering the bus", - From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeToolsHealth}, {ID: "H-bus", Asserts: "every stream and durable consumer the mesh defines is on the bus, and a request crosses " + "it to the machines' node tools and back", From: "ADR 0236, to-be 45 §8", Kind: kindCoreUnhealthy, Phase: 4, run: probeBusHealth}, - // The gate's verdicts and the witnesses' rollbacks (ADR 0236): each build that failed its gate keeps its - // condition until a newer build passes; each rollback a witness stands by is said. - {ID: gateProbe, Asserts: "no build that failed its gate, and no core component a witness put back, goes unsaid; " + - "a newer build that passes its gate clears it", From: "ADR 0236, to-be 45 §8", Kind: kindRolledBack, - Raises: []string{kindRollbackFailed}, Phase: 4, run: probeGates}, // The bus's planned step (ADR 0236): open while a person's bus upgrade runs, then checked by H-bus. {ID: busStepProbe, Asserts: "a bus upgrade a person started is said while it runs, and is followed by the bus's " + "health within its bound — or is said failed, with its snapshot as the way back", From: "ADR 0236, to-be 45 §8", Kind: kindBusMaintenance, Raises: []string{kindBusUpgradeFailed}, Phase: 4, run: probeBusStep}, - // A delivery plan held for a backup that failed or stayed silent (ADR 0404 decision 4): one condition per plan. - {ID: backupHeldProbe, Asserts: "no delivery plan is held for a backup before its send that failed or did not " + - "answer in ten minutes", From: "ADR 0404 decision 4, issue 1066", Kind: kindBackupHeld, Phase: 1, - run: probeBackupHeld}, - // A plan's send key the bus refused (ADR 0433 §4, issue 1157): in place of the timer that stood in for it. - {ID: sendKeyRefusedID, Asserts: "no delivery plan's key that watches a machine for its report was refused by the " + - "bus at the newest look", From: "ADR 0433 §4, issue 1157", Kind: kindSendKeyRefused, Phase: 1, - run: probeSendKeyRefused}, } // probeVerdict is one probe's outcome in a run. diff --git a/cmd/mesh-controller/epoch_send_test.go b/cmd/mesh-controller/epoch_send_test.go deleted file mode 100644 index 4fc2e5e4..00000000 --- a/cmd/mesh-controller/epoch_send_test.go +++ /dev/null @@ -1,109 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "errors" - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/link" -) - -// A declaration carries the lease's epoch (novox/hq to-be 45 §6) — to a machine whose node-engine said -// it reads one, and to no other: an older node-engine refuses a key it does not know, whole. - -// bodiesDelivery records each send as the mesh does, and keeps the bodies. -type bodiesDelivery struct { - recordedDelivery - bodies map[string][]byte -} - -func (b *bodiesDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) { - b.bodies[s.node] = body - return b.recordedDelivery.declare(ctx, s, body) -} - -func TestAMachineIsSentTheEpochOnlyOnceItSaysItReadsOne(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - epoch := uint64(57) - was := epochForActs - epochForActs = func(context.Context) (uint64, error) { return epoch, nil } - t.Cleanup(func() { epochForActs = was }) - - anchor, err := inv.NodeByName(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - if err := inv.RecordReadsEpoch(ctx, anchor.ID, true); err != nil { - t.Fatal(err) - } - d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: inv}, bodies: map[string][]byte{}} - if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, composeForPush(open), d, "", nil); err != nil { - t.Fatal(err) - } - carried := func(node string) (epoch float64, has bool) { - var envelope map[string]any - if err := json.Unmarshal(d.bodies[node], &envelope); err != nil { - t.Fatal(err) - } - epoch, has = envelope["epoch"].(float64) - return epoch, has - } - if e, has := carried("anchor"); !has || e != 57 { - t.Fatalf("the machine that reads an epoch was sent %v: %s", e, d.bodies["anchor"]) - } - if _, has := carried("laptop"); has { - t.Fatalf("a machine that never said it reads an epoch was sent one: %s", d.bodies["laptop"]) - } - - // A new holder of the lease is not a change of the machine: neither reads as behind. - epoch = 58 - would, err := wouldSend(ctx, open, mustNodes(t, open)) - if err != nil { - t.Fatal(err) - } - for _, node := range []string{"anchor", "laptop"} { - sent, err := inv.Outstanding(ctx, node) - if err != nil { - t.Fatal(err) - } - if would[node] != sent { - t.Fatalf("%s reads as behind after the lease changed hands, with nothing else changed", node) - } - } -} - -// A process that may not act composes nothing and sends nothing: its number is not taken. -func TestNothingIsComposedOrSentWithoutTheLease(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - was := epochForActs - epochForActs = func(context.Context) (uint64, error) { return 0, errors.New("this controller lost the lease") } - t.Cleanup(func() { epochForActs = was }) - d := &bodiesDelivery{recordedDelivery: recordedDelivery{inv: open.inventory}, bodies: map[string][]byte{}} - refused, err := sendRound(ctx, open, []string{"anchor"}, composeForPush(open), d, "", nil) - if err != nil { - t.Fatal(err) - } - if len(d.bodies) != 0 || len(refused) != 1 || !strings.Contains(refused[0], "lost the lease") { - t.Fatalf("a controller without the lease composed %d and refused %v", len(d.bodies), refused) - } - anchor, _ := open.inventory.NodeByName(ctx, "anchor") - if seq, _ := open.inventory.Sequence(ctx, anchor.ID); seq != 0 { - t.Fatalf("a controller without the lease took sequence %d", seq) - } - - // And at the send itself: the gate every declaration passes. - gate := link.ActingGate - link.ActingGate = func(context.Context) error { return errors.New("this controller lost the lease") } - t.Cleanup(func() { link.ActingGate = gate }) - if err := link.Declare(ctx, nil, nil, "anchor", []byte(`{"declaration":1}`), 0); err == nil || - !strings.Contains(err.Error(), "lost the lease") { - t.Fatalf("a declaration was let through the gate: %v", err) - } -} diff --git a/cmd/mesh-controller/failed_wait_provider_test.go b/cmd/mesh-controller/failed_wait_provider_test.go deleted file mode 100644 index 2790e340..00000000 --- a/cmd/mesh-controller/failed_wait_provider_test.go +++ /dev/null @@ -1,68 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A provider whose wait fails the controller's check lists who waits on it (novox/hq issue 450). -// -// A wait that does not check out is judged unhealthy (ADR 0283 decision 3), so the provider raises its unhealthy -// condition and its consumers are held under it (ADR 0240 rule 5). What is stored is what the machine said, the -// wait unchecked: read as said, the provider seems to wait for the operator, and the held consumers were never -// listed on the condition that is open. - -// refusedWait is a wait for a secret the database's manifest does not declare: it fails the check. -var refusedWait = inventory.Wait{Part: "postgres-database", Secret: "certificate", What: "the database's certificate"} - -// judgedRefused is the provider's resource as the controller judges it: unhealthy, saying why. -func judgedRefused() inventory.ResourceHealth { - r := waitingDatabaseFor(refusedWait) - r.State, r.Waits = link.StateUnhealthy, nil - r.Reason = "says it waits for the secret certificate, which db does not declare" - return r -} - -// The consumer's statement arrives after the provider's, so it is the consumer's judging (sayWaiters) that must list -// it at the provider's unhealthy condition, made urgent by who waits on it. -func TestAProviderWhoseWaitFailedItsCheckListsWhoWaitsOnIt(t *testing.T) { - k, _ := withConditionsInMemory(t) - stored := waitingDatabaseFor(refusedWait) - open := judgeBoth(t, k, []inventory.ResourceHealth{judgedRefused()}, - map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{stored}}}, - shopFailingBeside(stored)) - provider := conditionOf(open, moduleUnhealthyKey("db", "anchor")) - if len(open) != 1 || provider == nil { - t.Fatalf("a provider whose wait failed its check and a consumer of it raised %v; want the provider's "+ - "unhealthy alone", openKeysOf(open)) - } - if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") { - t.Fatalf("the provider's unhealthy condition does not list shop on laptop as waiting on it: %s", said) - } - if provider.Severity != conditions.Urgent { - t.Fatalf("the provider's unhealthy condition is %s with a consumer waiting on it; want urgent", provider.Severity) - } -} - -// A provider that waits for a secret already given is unhealthy to its consumers too, before its own condition opens: -// they are held under it as under any unhealthy provider, never as waiting for the operator, and its condition, when -// open, is said as unhealthy with who waits on it. -func TestAProviderWaitingForASecretAlreadyGivenIsUnhealthyToItsConsumers(t *testing.T) { - given := holdingOf(nil, shopFailingBeside(waitingDatabase()), shopOnTheDatabase) - given.waits.given["db@anchor"] = map[string]time.Time{"licence": time.Now().Add(-time.Hour)} - by, held := given.heldWith("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")}) - if !held || by.provider != theDatabase || len(by.waits) > 0 { - t.Fatalf("shop under a database waiting for a licence already given: held %v under %v with waits %v; want "+ - "held under db on anchor as unhealthy, no waits", held, by.provider, by.waits) - } - if _, uncovered := given.waitingUncovered("laptop", "shop", []inventory.ResourceHealth{failingConsumer("shop")}); uncovered { - t.Fatalf("a provider whose wait failed its check is said as waiting for another part") - } -} diff --git a/cmd/mesh-controller/first_builds.go b/cmd/mesh-controller/first_builds.go deleted file mode 100644 index b6c06bfc..00000000 --- a/cmd/mesh-controller/first_builds.go +++ /dev/null @@ -1,277 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "regexp" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// firstBuilds asks the first build of every module a merge adds to a repository no registered module is built from, when -// the merge was announced as made on the operator's Approve of that head (novox/hq ADR 0432, issue 1146): the modules' -// directories are the ones the announcement says hold a module.json at the merge commit, as touchedBy reads them. The -// merge is kept, so the build's outcome is judged by the rule that registers it (mayRegisterFrom, approvedMove). -// -// **Complete in either order, with no timer** (ADR 0426): when mesh-delivery's word is kept first, it is found here; when -// the forge's merge is heard first, the merge is kept as a first merge waiting for the word, and keepingDeliveryMerges -// runs this again when the word is kept (resumeFirstBuilds). A word that says no (a Decline, another head) asks nothing and -// says why. Answers whether the merge was dealt with here. -func (f following) firstBuilds(ctx context.Context, m link.SourceMoved, entries []inventory.Entry, - read map[string][]inventory.ReadRepository, now time.Time) (bool, error) { - inv := f.open.inventory - repository := m.Owner + "/" + m.Repo - added, why, err := firstBuildsOf(ctx, inv, m, entries, read) - switch { - case errors.Is(err, errFirstNotApproved): - fmt.Printf("%s merged into %s (%.8s) adds %s, and no build is asked: %v\n", repository, m.Base, m.Commit, - strings.Join(addedDirs(m, entries, read), ", "), err) - return true, nil - case errors.Is(err, errNotAnnounced): - // Kept, waiting for mesh-delivery's word: a restart keeps it, and the word's arrival asks the builds. - if kerr := keepFirstMerge(ctx, inv, m, now, true); kerr != nil { - return true, notNow(kerr) - } - fmt.Printf("%s merged into %s (%.8s) adds %s; it is kept, and the first build is asked when mesh-delivery "+ - "announces the merge on the operator's Approve\n", repository, m.Base, m.Commit, - strings.Join(addedDirs(m, entries, read), ", ")) - return true, nil - case err != nil: - return true, notNow(err) - case len(added) == 0: - return false, nil - } - if err := askFirstBuilds(ctx, f.open, m, now, added, why); err != nil { - return true, notNow(err) - } - return true, nil -} - -// keepFirstMerge keeps a first merge as the controller heard it, waiting (no plan answers it). -func keepFirstMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, now time.Time, waitingForWord bool) error { - merged, err := time.Parse(time.RFC3339Nano, m.MergedAt) - if err != nil { - merged = now - } - event, err := json.Marshal(keptMerge{SourceMoved: m, FirstBuild: waitingForWord}) - if err != nil { - return err - } - _, err = inv.AddMerge(ctx, inventory.BatchedMerge{Repository: m.Owner + "/" + m.Repo, Branch: m.Base, Commit: m.Commit, - Merged: merged.UTC(), Heard: now, Event: event}) - return err -} - -// askFirstBuilds keeps the merge as answered and registers its modules from the builder's done keys (novox/hq ADR 0501 -// §3, §6): nothing asks their first builds. The merge kept is the registration's warrant (approvedMove reads it by the -// commit the indexer's trunk value names); a module whose key is not done yet is registered when the key's write wakes -// the take-in (new_modules.go). -func askFirstBuilds(ctx context.Context, open *stores, m link.SourceMoved, now time.Time, added []string, - why string) error { - inv := open.inventory - repository := m.Owner + "/" + m.Repo - if err := keepFirstMerge(ctx, inv, m, now, false); err != nil { - return err - } - fmt.Printf("%s merged into %s (%.8s) on the operator's Approve (%s): the mesh builds nothing from it yet, so %s "+ - "are registered from the builder's done keys, asking no build\n", repository, m.Base, m.Commit, why, - strings.Join(added, ", ")) - // Answered: it no longer waits, so the word's arrival again does nothing twice. - if err := inv.AnswerMerge(ctx, repository, m.Commit, inventory.WaitingRecord, false); err != nil { - return err - } - modules, err := newModulesOf(ctx, repository) - switch { - case err != nil: - // Said, not failed: the merge is answered, and the look at every module when a controller starts, or the next - // write of the builder or the indexer for one of them, registers them. - fmt.Printf(" the indexer's modules of %s cannot be read now (%v): they are registered when the builder's next "+ - "write of one wakes the take-in\n", repository, err) - case len(modules) > 0: - inBackground(func() { takeNewModules(context.WithoutCancel(ctx), open, modules...) }) - } - return nil -} - -// resumeFirstBuilds runs firstBuilds for the first merges waiting for the word mesh-delivery just announced for a pull -// request's head: the other order of the same two events. -func (f following) resumeFirstBuilds(ctx context.Context, d deliveryMerge) error { - actingOnMerges.Lock() - defer actingOnMerges.Unlock() - inv := f.open.inventory - waiting, err := inv.WaitingMerges(ctx) - if err != nil { - return notNow(err) - } - for _, w := range waiting { - k := announcedOf(w) - if !k.FirstBuild || len(k.Held) > 0 || !strings.EqualFold(k.Owner+"/"+k.Repo, d.Owner+"/"+d.Repo) || k.Number != d.Number || - k.HeadSHA != d.HeadSHA { - continue - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return notNow(err) - } - read, err := readForPlanning(ctx, inv) - if err != nil { - return notNow(err) - } - added, why, err := firstBuildsOf(ctx, inv, k.SourceMoved, entries, read) - switch { - case errors.Is(err, errFirstNotApproved): - fmt.Printf("%s/%s merged into %s (%.8s) was waiting for mesh-delivery's word, and no build is asked: %v\n", - k.Owner, k.Repo, k.Base, k.Commit, err) - if aerr := inv.AnswerMerge(ctx, w.Repository, w.Commit, inventory.WaitingRecord, false); aerr != nil { - return notNow(aerr) - } - case err != nil: - return notNow(err) - case len(added) > 0: - if err := askFirstBuilds(ctx, f.open, k.SourceMoved, time.Now().UTC(), added, why); err != nil { - return notNow(err) - } - } - } - return nil -} - -// errFirstNotApproved says a merge into a repository the mesh builds nothing from is not one the operator approved. -var errFirstNotApproved = errors.New("it is no merge made on the operator's Approve of its head, so a hand `build` " + - "registers a module from a repository the mesh does not build yet") - -// addedDirs are the module directories the merge adds to a repository no registered module is built from, none where -// the merge is not into the trunk or some module is built from the repository already. -func addedDirs(m link.SourceMoved, entries []inventory.Entry, read map[string][]inventory.ReadRepository) []string { - if m.Base != "main" { - return nil - } - for _, e := range entries { - if !e.Provided && sameRepository(e.Source.Repository, m) { - return nil - } - } - _, added, _ := touchedBy(nil, entries, m, read) - return added -} - -// firstBuildsOf is the directories whose first build a merge asks, and the warrant it was made on; none when the merge -// is no first merge into a repository the mesh builds nothing from. errFirstNotApproved when it is and the word on record -// is no Approve of its head; errNotAnnounced when mesh-delivery's word is not kept yet. -func firstBuildsOf(ctx context.Context, inv catalogueReader, m link.SourceMoved, entries []inventory.Entry, - read map[string][]inventory.ReadRepository) ([]string, string, error) { - added := addedDirs(m, entries, read) - if len(added) == 0 { - return nil, "", nil - } - why, err := approvedAnnounced(ctx, inv, m.Owner, m.Repo, keptMerge{SourceMoved: m}) - switch { - case err == nil: - return added, why, nil - case errors.Is(err, errWordNotAnApprove): - return nil, "", fmt.Errorf("%w (%v)", errFirstNotApproved, err) - } - // Not announced yet, or the bus or the store could not be asked: the caller waits or tries again, and asks nothing. - return nil, "", err -} - -// heldFirstBuild says a first build was refused because its manifest uses a word the running build-agent does not read, -// and its merge is kept waiting for a build-agent that reads it (novox/hq ADR 0482 §5, issue 1173). Read from a merge kept -// before R4e, never written since: nothing asks a first build (ADR 0501 §6), and a controller that starts answers such a -// merge and registers its modules from their keys (answerHeldMerges). -type heldFirstBuild struct { - Module string `json:"module"` - Path string `json:"path"` - Word string `json:"word"` - At time.Time `json:"at"` -} - -// unknownWord is how the strict reading of a manifest says a word is not one it knows: encoding/json's own words, which -// the build-agent's answer carries whole. The build's answer has no typed field for it (Failed is the builder's words). -var unknownWord = regexp.MustCompile(`json: unknown field "([^"]+)"`) - -// refusedWord is the word a build's failure says the reader did not know; empty when it says none. -func refusedWord(failed string) string { - if m := unknownWord.FindStringSubmatch(failed); m != nil { - return m[1] - } - return "" -} - -// probeFirstBuildsID is the probe that reads the first merges waiting for mesh-delivery's word. -const probeFirstBuildsID = "D18" - -// kindFirstBuildUnasked is what a first merge waiting too long for mesh-delivery's word raises. -const kindFirstBuildUnasked = "first-build-unasked" - -// firstBuildWaitBound is how long a first merge waits for mesh-delivery's announcement of the operator's Approve before -// it is said: a bound in the sense of ADR 0426 §3, not a delay. Nothing waits on it: the word's arrival asks the builds -// at once, and the merge is said, never dropped, once it is past this (issue 1146). -const firstBuildWaitBound = 10 * time.Minute - -// firstBuildObservations are the conditions of the first merges waiting past the bound, one for each; a merge answered is -// no longer listed as waiting, so its condition clears with the next look. -func firstBuildObservations(waiting []inventory.BatchedMerge, now time.Time) []conditions.Observation { - var out []conditions.Observation - for _, w := range waiting { - k := announcedOf(w) - if !k.FirstBuild || now.Sub(w.Heard) <= firstBuildWaitBound { - continue - } - repository := k.Owner + "/" + k.Repo - for_ := now.Sub(w.Heard).Round(time.Minute) - out = append(out, conditions.Observation{Scope: conditions.ScopeMerge, - ID: fmt.Sprintf("%s_%.8s", repository, k.Commit), Kind: kindFirstBuildUnasked, Severity: conditions.Warning, - Summary: fmt.Sprintf("the merge %.8s of %s adds modules the mesh builds nothing from yet, and has waited %s for "+ - "mesh-delivery's announcement of the operator's Approve of its head: no build is asked", k.Commit, repository, for_), - Said: fmt.Sprintf("%s at %s waits for mesh-delivery's word on pull request %d (head %.8s)", repository, k.Commit, - k.Number, k.HeadSHA), - Headline: fmt.Sprintf("The first build of %s is not asked: no Approve was announced", repository), - Explanation: fmt.Sprintf("The merge %.8s into %s adds modules the mesh builds nothing from yet. It registers them "+ - "only when mesh-delivery announces that it merged the pull request on your Approve of that head, and no such "+ - "announcement has come in %s. The merge is kept, and the modules are registered from the builder's keys the moment "+ - "the announcement is kept.", - k.Commit, repository, for_), - Resolved: "mesh-delivery announces the merge, or the operator registers the modules at the controller's terminal", - }) - } - return out -} - -// probeFirstBuilds is D18: no first merge waits past its bound for mesh-delivery's word unsaid. -func probeFirstBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, error) { - waiting, err := d.open.inventory.WaitingMerges(ctx) - if err != nil { - return nil, err - } - now := time.Now().UTC() - return append(firstBuildObservations(waiting, now), newModuleObservations(now)...), nil -} - -// answerHeldMerges answers every merge a controller of before held for a build-agent that reads its new modules' -// manifests (issue 1173): nothing asks their first builds any more, and the look at every module when this controller -// starts registers them from their keys (ADR 0501 §6, §7). -func answerHeldMerges(ctx context.Context, inv *inventory.Inventory) { - waiting, err := inv.WaitingMerges(ctx) - if err != nil { - fmt.Printf("the merges held for a newer build-agent could not be read: %v\n", err) - return - } - for _, w := range waiting { - if k := announcedOf(w); len(k.Held) > 0 { - if err := inv.AnswerMerge(ctx, w.Repository, w.Commit, inventory.WaitingRecord, false); err != nil { - fmt.Printf("the merge %.8s of %s held for a newer build-agent could not be answered: %v\n", w.Commit, - w.Repository, err) - continue - } - fmt.Printf("the merge %.8s of %s was held for a newer build-agent; its modules are registered from the "+ - "builder's keys instead\n", w.Commit, w.Repository) - } - } -} diff --git a/cmd/mesh-controller/first_builds_integration_test.go b/cmd/mesh-controller/first_builds_integration_test.go deleted file mode 100644 index 4608f763..00000000 --- a/cmd/mesh-controller/first_builds_integration_test.go +++ /dev/null @@ -1,152 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "testing" - "time" - - "github.com/nats-io/nats.go/jetstream" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// 1146/AC7 on keys (ADR 0501 §3, §6; issue 1851, A5), whole against a real bus and the store, in both orders of the two -// events: mesh-delivery #7 merged on the operator's Approve into a repository the mesh builds nothing from. The forge's -// merge, heard as the controller hears it (SourceMoved), and mesh-delivery's word, published on the subject the -// controller consumes with the router's record of the ask in its bucket, come in either order; each way the module the -// merge commit holds is registered from the builder's done key of it, and nothing asks a build. Without the Approve in -// the record it is not registered. Store-backed: skips without one. -func TestAMergeOnTheApproveIntoANewRepositoryRegistersItsModuleFromItsKey(t *testing.T) { - const head, commit = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0177ea7a00000000" - for _, c := range []struct { - what string - wordFirst bool - approved bool // the router's record holds the operator's Approve of the merged head - wantRegister bool - }{ - {what: "the forge's merge before mesh-delivery's word", approved: true, wantRegister: true}, - {what: "mesh-delivery's word before the forge's merge", wordFirst: true, approved: true, wantRegister: true}, - {what: "a word whose record holds no Approve", wordFirst: true}, - } { - t.Run(c.what, func(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - noBuildAsked(t) - b := aBuilderOnTheBus(t) - js := b.js - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "operator-channel", Version: "1", - DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.AsksSeat, Records: []string{"asks"}}}}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/operator-channel", BuiltFrom: "c0ffee"}); err != nil { - t.Fatal(err) - } - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - api, err := jetstream.New(js.Conn()) - if err != nil { - t.Fatal(err) - } - records, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: broker.BucketName("operator-channel", "asks")}) - if err != nil { - t.Fatal(err) - } - a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head}) - if c.approved { - record, _ := json.Marshal(map[string]any{"state": "closed", "ask": a.ask, "warrant": a.warrant}) - if _, err := records.Put(ctx, sdkdelivery.TellerModule+"."+a.merge.AskID, record); err != nil { - t.Fatal(err) - } - } - // The indexer's value and the builder's done key of the module the merge commit holds. - fp := fpOf('9') - manifest := b.indexed(t, "mesh-delivery-view", "1", "novox/mesh-delivery", ".", commit, fp) - b.built(t, "mesh-delivery-view", "1", fp, manifest, sdkdelivery.BuildDone) - - following := following{open: open} - server := link.ConnectNats(js, nil, noListener{}) - if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream, catalog: theDeliveryClaims, kept: following.resumeFirstBuilds}); err != nil { - t.Fatal(err) - } - if err := broker.AssertServingConsumers(js); err != nil { - t.Fatal(err) - } - serving, stop := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { defer close(done); _ = server.Serve(serving) }() - t.Cleanup(func() { stop(); <-done }) - - word := func() { - t.Helper() - body, _ := json.Marshal(a.merge) - if _, err := js.Context().Publish(broker.DeliveryMergedSubjectOf(theDelivery), body); err != nil { - t.Fatal(err) - } - becomes(t, 15*time.Second, "the word being kept", func() bool { - _, kept, err := keptDeliveryMerge(ctx, "novox", "mesh-delivery", 7, head) - return err == nil && kept - }) - time.Sleep(500 * time.Millisecond) // the keeper's hook runs as the word is kept - } - forge := func() { - t.Helper() - m := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: commit, Number: 7, - MergedAt: time.Now().Add(-time.Second).UTC().Format(time.RFC3339Nano), Paths: []string{"module.json", "main.go"}, - ModuleDirs: []string{"."}, ModuleDirsSaid: true} - if err := following.SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - } - if c.wordFirst { - word() - forge() - } else { - forge() - time.Sleep(300 * time.Millisecond) - if _, known := inventoryHolds(t, inv, "mesh-delivery-view"); known { - t.Fatal("1146/AC7: registered before mesh-delivery's word was kept") - } - word() - } - if !c.wantRegister { - time.Sleep(time.Second) - if _, known := inventoryHolds(t, inv, "mesh-delivery-view"); known { - t.Fatal("1146/AC7: a merge whose record holds no Approve registered its module") - } - return - } - becomes(t, 10*time.Second, "the module registered", func() bool { - _, known := inventoryHolds(t, inv, "mesh-delivery-view") - return known - }) - src, err := inv.SourceOf(ctx, "mesh-delivery-view") - if err != nil || src.Repository != "novox/mesh-delivery" { - t.Fatalf("1146/AC7: the module is built from %q (%v)", src.Repository, err) - } - }) - } -} - -func inventoryHolds(t *testing.T, inv *inventory.Inventory, module string) (inventory.Entry, bool) { - t.Helper() - entries, err := inv.Catalogued(t.Context()) - if err != nil { - t.Fatal(err) - } - for _, e := range entries { - if e.Manifest.Module == module { - return e, true - } - } - return inventory.Entry{}, false -} diff --git a/cmd/mesh-controller/first_builds_test.go b/cmd/mesh-controller/first_builds_test.go deleted file mode 100644 index b0c4d009..00000000 --- a/cmd/mesh-controller/first_builds_test.go +++ /dev/null @@ -1,106 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "slices" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/asks" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// 1146/AC6: the first merge into a repository no registered module is built from asks the first build of every module -// the merge commit holds when it was made on the operator's Approve of that head, and asks nothing without it. -func TestAFirstMergeIntoANewRepositoryAsksItsModulesBuildsOnlyOnTheApprove(t *testing.T) { - const head = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1" - now := time.Date(2026, 10, 19, 0, 6, 0, 0, time.UTC) - mergedAt := now.Add(-time.Hour).Format(time.RFC3339Nano) - heard := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: "0177ea7a", - Number: 1, MergedAt: mergedAt, ModuleDirsSaid: true, ModuleDirs: []string{"watchman", "view"}, - Paths: []string{"view/module.json", "watchman/module.json", "README.md"}} - built := inventory.Entry{Source: inventory.Source{Repository: "novox/mesh-catalog", Seat: "git"}} - for _, c := range []struct { - what string - change func(*anApprovedMerge, *link.SourceMoved, *[]inventory.Entry) - unkept bool - dirs []string - refused bool - waits bool - }{ - {what: "the Approve of the merged head", dirs: []string{"view", "watchman"}}, - {what: "a Decline", refused: true, change: func(a *anApprovedMerge, _ *link.SourceMoved, _ *[]inventory.Entry) { - a.warrant.Option, a.warrant.Label, a.merge.Option = "decline", "Decline", "decline" - }}, - {what: "no word of mesh-delivery yet, however long ago the merge was made", unkept: true, waits: true}, - {what: "a module already built from the repository", change: func(_ *anApprovedMerge, _ *link.SourceMoved, e *[]inventory.Entry) { - *e = append(*e, inventory.Entry{Source: inventory.Source{Repository: "novox/mesh-delivery", Seat: "git"}}) - }}, - {what: "a merge into another branch", change: func(_ *anApprovedMerge, m *link.SourceMoved, _ *[]inventory.Entry) { m.Base = "feat" }}, - {what: "a merge that holds no module.json", change: func(_ *anApprovedMerge, m *link.SourceMoved, _ *[]inventory.Entry) { - m.ModuleDirs, m.Paths = nil, []string{"README.md"} - }}, - } { - t.Run(c.what, func(t *testing.T) { - a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head}) - a.merge.Number = 1 - m, entries := heard, []inventory.Entry{built} - if c.change != nil { - c.change(&a, &m, &entries) - } - wasKept, wasAsk := keptDeliveryMerge, routerAskOf - t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk }) - keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) { - return a.merge, !c.unkept, nil - } - routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) { - return &a.ask, &a.warrant, nil - } - dirs, _, err := firstBuildsOf(t.Context(), nil, m, entries, nil) - switch { - case c.waits && !errors.Is(err, errNotAnnounced): - t.Fatalf("1146/AC6: %s: want a wait for the word, got %v", c.what, err) - case c.refused && !errors.Is(err, errFirstNotApproved): - t.Fatalf("1146/AC6: %s: want the Approve missing, got %v (dirs %v)", c.what, err, dirs) - case !c.refused && !c.waits && err != nil: - t.Fatalf("1146/AC6: %s: %v", c.what, err) - case !slices.Equal(dirs, c.dirs): - t.Fatalf("1146/AC6: %s: first builds asked of %v, want %v", c.what, dirs, c.dirs) - } - }) - } -} - -// 1146/AC8: a first merge waiting past the bound for mesh-delivery's word is said, naming the repository, the commit and -// the missing word; one inside the bound, one that is no first merge, and one answered (no longer waiting) are not. -func TestAFirstMergeWaitingPastItsBoundIsSaidAndClearsWhenAnswered(t *testing.T) { - now := time.Date(2026, 10, 19, 1, 0, 0, 0, time.UTC) - waiting := func(commit string, first bool, heard time.Time) inventory.BatchedMerge { - event, _ := json.Marshal(keptMerge{SourceMoved: link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", - Commit: commit, Number: 1, HeadSHA: "a1a1a1a1a1a1"}, FirstBuild: first}) - return inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: commit, Heard: heard, Event: event} - } - got := firstBuildObservations([]inventory.BatchedMerge{ - waiting("0177ea7a00", true, now.Add(-firstBuildWaitBound-time.Minute)), - waiting("1111111100", true, now.Add(-time.Minute)), - waiting("2222222200", false, now.Add(-time.Hour)), - }, now) - if len(got) != 1 { - t.Fatalf("1146/AC8: %d conditions raised, want the one merge past the bound: %+v", len(got), got) - } - o := got[0] - if o.Kind != kindFirstBuildUnasked || o.Scope != conditions.ScopeMerge || o.Severity != conditions.Warning || - !strings.Contains(o.Summary, "novox/mesh-delivery") || !strings.Contains(o.Summary, "0177ea7a") || - !strings.Contains(o.Summary, "announcement of the operator's Approve") { - t.Fatalf("1146/AC8: the condition is %+v", o) - } - // Answered, the merge is no longer listed as waiting: nothing is observed, and the condition clears. - if cleared := firstBuildObservations(nil, now); len(cleared) != 0 { - t.Fatalf("1146/AC8: an answered merge still raises %+v", cleared) - } -} diff --git a/cmd/mesh-controller/fixclass_test.go b/cmd/mesh-controller/fixclass_test.go deleted file mode 100644 index 9541c005..00000000 --- a/cmd/mesh-controller/fixclass_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package main - -import ( - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// **A change plan says its class** (novox/hq ADR 0313 §2): the issue tracker holds a fix to the budget of the -// slowest-budgeted module the fix moves, and a fix that moves no module has no fix budget. The plan says core, -// leaf or none, so mesh-delivery can pass it on with the fix's delivery. -func TestAChangePlanSaysTheClassOfWhatItMoves(t *testing.T) { - const repo = "http://forge.internal:20000/novox/mesh-catalog.git" - entry := func(name, path string) inventory.Entry { - e := fromRepo(name, repo, path) - e.Source.BuiltFrom = "old" - e.On = []string{"anchor"} - return e - } - gitea := entry("gitea", "modules/gitea") - resolver := entry("dnsmasq", "modules/dnsmasq") - resolver.Manifest.Claims = []catalogue.Claim{{Name: resolverSeat}} - delivery := entry(catalogue.DeliverySeat, "modules/mesh-delivery") - delivery.Manifest.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}} - webApp := entry("photos", "modules/photos") - entries := []inventory.Entry{gitea, resolver, delivery, webApp} - // photos stands on gitea: a change to gitea moves photos as its dependent. - edges := []inventory.Edge{{From: "photos", To: "gitea", Kind: inventory.EdgeStandsOn}} - policy := func(string) (inventory.Upgrade, bool) { return inventory.Upgrade{RollOut: true}, true } - plan := func(paths ...string) link.ChangePlan { - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "head", Paths: paths} - return changePlanOf("novox/mesh-catalog", "main", "head", reachOfMerge(m, entries, nil, edges), entries, policy) - } - - for _, c := range []struct { - why string - paths []string - want string - }{ - {"a leaf module and its dependent", []string{"modules/gitea/index.ts"}, inventory.ClassLeaf}, - {"the delivery's owner, on the controller's own path", []string{"modules/mesh-delivery/cmd/x.go"}, inventory.ClassCore}, - {"the mesh's resolver", []string{"modules/dnsmasq/module.json"}, inventory.ClassCore}, - {"a leaf and a core module together: the slowest-budgeted", []string{"modules/gitea/index.ts", "modules/dnsmasq/module.json"}, inventory.ClassCore}, - {"nothing any module's build reads", []string{"README.md"}, "none"}, - {"a new leaf module alone", []string{"modules/weather/module.json"}, inventory.ClassLeaf}, - {"a new module on the controller's own path, by its own class", []string{"next/mesh-host/module.json"}, inventory.ClassCore}, - {"a new leaf module beside a core one moved", []string{"modules/weather/module.json", "modules/dnsmasq/module.json"}, inventory.ClassCore}, - } { - if got := plan(c.paths...).Class; got != c.want { - t.Errorf("%s: class %q, want %q", c.why, got, c.want) - } - } -} diff --git a/cmd/mesh-controller/delivery_changes.go b/cmd/mesh-controller/forge_read.go similarity index 54% rename from cmd/mesh-controller/delivery_changes.go rename to cmd/mesh-controller/forge_read.go index 0e2a2966..f4b85673 100644 --- a/cmd/mesh-controller/delivery_changes.go +++ b/cmd/mesh-controller/forge_read.go @@ -17,9 +17,8 @@ import ( "github.com/novox/mesh-controller/internal/link" ) -// What a change says of the manifests it touches, read from the forge (novox/hq ADR 0419 clause 3, the lead's settlement -// of issue 977 question 1): the controller asks the forge for each changed module.json at the commit and at what it -// replaced, once per commit. The composed check's result is not used: the plan is cut after the merge. +// What the forge says of a change's manifests (novox/hq ADR 0502): read through the forge's module on the bus, once per +// commit. Kept by release 6 for the pull request's declared config; the delivery plan's reads of it are gone. // changeSource is one change to a repository whose manifests are read: the commit after it and, when known, what it // replaced (a pull request's base commit or branch). An empty Old is the first parent of New: the merge's base. @@ -63,24 +62,6 @@ var manifestAt = func(ctx context.Context, owner, repo, file, ref string) (catal return m, true, nil } -// parentOf is the first parent of a commit, from the forge. A variable so a test needs no forge. -var parentOf = func(ctx context.Context, owner, repo, commit string) (string, error) { - var found struct { - Result struct { - Parents []struct { - SHA string `json:"sha"` - } `json:"parents"` - } `json:"result"` - } - if _, err := askForgeApi(ctx, "/repos/"+owner+"/"+repo+"/git/commits/"+url.PathEscape(commit), &found); err != nil { - return "", err - } - if len(found.Result.Parents) == 0 { - return "", nil - } - return found.Result.Parents[0].SHA, nil -} - func escapedPath(p string) string { parts := strings.Split(p, "/") for i := range parts { @@ -114,12 +95,6 @@ func askForgeApi(ctx context.Context, apiPath string, into any) (bool, error) { return false, nil } -// manifestsRead keeps what the forge said of a manifest at a commit, which never changes: one ask per commit. -var ( - manifestsRead = map[string]manifestRead{} - manifestsReadMu sync.Mutex -) - type manifestRead struct { manifest catalogue.Manifest exists bool @@ -147,96 +122,11 @@ func manifestCached(ctx context.Context, owner, repo, file, ref string) (catalog return m, exists, err } -// verbsServed are the seat verbs a manifest serves, as ".": those of the seats it defines, those of the -// seats it claims, and its own tools as ".". -func verbsServed(m catalogue.Manifest) []string { - var out []string - for _, s := range m.DefinesSeats { - for _, v := range s.Serves { - out = appendNew(out, s.Name+"."+v.Name) - } - } - for _, c := range m.Claims { - for _, v := range c.ServesFor(m) { - out = appendNew(out, c.Name+"."+v) - } - } - for _, t := range m.Tools { - out = appendNew(out, m.Module+"."+t) - } - return out -} - -// verbsInvoked are the verbs a manifest calls, as "." or ".". -func verbsInvoked(m catalogue.Manifest) []string { - var out []string - for _, i := range m.Invokes { - if i = strings.TrimPrefix(i, "seat:"); i != "*" && i != "" { - out = appendNew(out, i) - } - } - return out -} - -func added(now, was []string) []string { - var out []string - for _, v := range now { - if !slices.Contains(was, v) { - out = append(out, v) - } - } - return out -} - -// changesOf reads, for each manifest the changes touch, what they add to the verbs its module serves and invokes, by -// module. A change whose paths are unsaid or truncated names no manifest: nothing is read for it. An error is the -// forge's, never read as "no change": the plan is not cut on a guess. -func changesOf(ctx context.Context, sources []changeSource) (map[string]moduleChange, error) { - out := map[string]moduleChange{} - for _, s := range sources { - if s.Truncated { - continue - } - for _, p := range s.Paths { - if path.Base(p) != "module.json" || slices.Contains(s.Removed, p) { - continue - } - now, exists, err := manifestCached(ctx, s.Owner, s.Repo, p, s.New) - if err != nil { - return nil, err - } - if !exists || now.Module == "" { - continue - } - old := s.Old - if old == "" { - first := s.Since - if first == "" { - first = s.New - } - if old, err = parentOf(ctx, s.Owner, s.Repo, first); err != nil { - return nil, err - } - } - var was catalogue.Manifest - if old != "" { - if was, _, err = manifestCached(ctx, s.Owner, s.Repo, p, old); err != nil { - return nil, err - } - } - c := out[now.Module] - c.Manifest = true - for _, v := range added(verbsServed(now), verbsServed(was)) { - c.Serves = appendNew(c.Serves, v) - } - for _, v := range added(verbsInvoked(now), verbsInvoked(was)) { - c.Invokes = appendNew(c.Invokes, v) - } - out[now.Module] = c - } - } - return out, nil -} +// manifestsRead keeps what the forge said of a manifest at a commit, which never changes: one ask per commit. +var ( + manifestsRead = map[string]manifestRead{} + manifestsReadMu sync.Mutex +) // truncatedPaths are the manifests a change that names only a beginning of its files is read as changing: those of the // modules it moves that are built from its repository. Said, never skipped: no edge may be lost to a long list. @@ -251,21 +141,3 @@ func truncatedPaths(entries []inventory.Entry, owner, repo string, moved []strin owner, repo, strings.Join(out, ", ")) return out } - -// changeSourcesOf are a group's members as changes to read: each head over what its pull request merges into. -func changeSourcesOf(members []orderMember, entries []inventory.Entry, reach map[string]orderReach) []changeSource { - var out []changeSource - for _, m := range members { - p := m.pull() - if p.PathsTruncated { - p.Paths, p.PathsTruncated = truncatedPaths(entries, p.Owner, p.Repo, reach[m.ID].Moved), false - } - old := p.BaseSHA - if old == "" { - old = p.Base - } - out = append(out, changeSource{Owner: p.Owner, Repo: p.Repo, New: p.Commit, Old: old, Paths: p.Paths, - Truncated: p.PathsTruncated, Removed: p.Removed}) - } - return out -} diff --git a/cmd/mesh-controller/found_wait_integration_test.go b/cmd/mesh-controller/found_wait_integration_test.go deleted file mode 100644 index 1a1f193c..00000000 --- a/cmd/mesh-controller/found_wait_integration_test.go +++ /dev/null @@ -1,127 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - for _, c := range []struct { - name string - found time.Duration // when the directory was found, against now - passes bool - }{ - {"found a day before the send", -24 * time.Hour, true}, - {"found by this send", time.Hour, false}, - } { - t.Run(c.name, func(t *testing.T) { - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { - return map[string]bool{"anchor": true, "laptop": true}, nil - } - backlogFacts := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f, err := backlogFacts(ctx, open, component) - // The used-as-found condition, raised after the send (its second statement): its own kind, never a - // fault the gate reads as the build's. - f.judged, f.openErr = true, nil - f.open = append(f.open, conditions.Condition{Key: usedAsFoundKey("app", "anchor"), Kind: kindUsedAsFound, - Subject: conditions.Subject{Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"}, - Raised: time.Now()}) - f.health = map[string]inventory.NodeHealth{} - for _, n := range []string{"anchor", "laptop"} { - f.health[n] = inventory.NodeHealth{Node: n, HeardAt: time.Now(), Resources: []inventory.ResourceHealth{ - {Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy}, - foundDirectory("app", time.Now().Add(c.found)), - {Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateHealthy}}} - } - return f, err - } - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) - gateWithin, gateEvery, gateBound = time.Hour, 0, 300*time.Millisecond - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - advancePlans(ctx, b.open) - if p := b.release(t); p.State != inventory.PlanRolling { - break - } - time.Sleep(20 * time.Millisecond) - } - p := b.release(t) - v, found, err := inv.GateOf(ctx, "build-app-c2") - if c.passes { - if p.State != inventory.PlanDone || err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("the walk is %s (%s); app's verdict %+v: want the fix through", p.State, p.Note, v) - } - if !strings.Contains(v.Why+p.Note, "hand it over") { - t.Errorf("the wait is not carried: verdict %q, walk %q", v.Why, p.Note) - } - return - } - if p.State == inventory.PlanDone { - t.Fatalf("a directory this send found let the walk through: %s", p.Note) - } - }) - } -} - -func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - k, _ := withConditionsInMemory(t) - ctx := t.Context() - rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}} - for i := 0; i < 2; i++ { - if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": i + 1}, time.Now()); err != nil { - t.Fatal(err) - } - } - open, _ := k.Open(ctx) - var got *conditions.Condition - for i, c := range open { - if c.Key == usedAsFoundKey("notes", "laptop") { - got = &open[i] - } - if c.Kind == kindModuleUnhealthy { - t.Fatalf("raised as a fault: %+v", c) - } - } - if got == nil || got.Resolver != conditions.ResolverOperator || got.Severity == conditions.Urgent || - !strings.Contains(got.Summary, "notes.data") { - t.Fatalf("the condition: %+v", got) - } - // Long open is still not urgent: only a person can hand it over, and nothing is broken by the wait. - if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": 3}, time.Now().Add(48*time.Hour)); err != nil { - t.Fatal(err) - } - open, _ = k.Open(ctx) - for _, c := range open { - if c.Key == usedAsFoundKey("notes", "laptop") && c.Severity == conditions.Urgent { - t.Fatal("a directory used as found became urgent") - } - } - if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil { - t.Fatal(err) - } - open, _ = k.Open(ctx) - for _, c := range open { - if c.Key == usedAsFoundKey("notes", "laptop") { - t.Fatal("not cleared once handed over") - } - } -} diff --git a/cmd/mesh-controller/found_wait_test.go b/cmd/mesh-controller/found_wait_test.go deleted file mode 100644 index 54621738..00000000 --- a/cmd/mesh-controller/found_wait_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A directory the node-engine uses as found (novox/hq issue 339) waits for a person to hand it over at the -// machine. Found before this send, it is no fault of the build: the gate passes with the wait carried, so an -// urgent fix of that module still goes through. Found by this send, the send brought it, and the gate holds. -func foundDirectory(module string, since time.Time) inventory.ResourceHealth { - return inventory.ResourceHealth{Module: module, Resource: module + ".data", Kind: link.KindDirectory, - Target: "/srv/" + module, State: link.StateUnhealthy, Since: since, - Reason: link.ReasonUsedAsFound + " owned by 1000:1000, mode 700, as found; root, mode 755 was declared and " + - "not given it — `nox node hand-over laptop ` on the control-node, with its path, hands it to the mesh"} -} - -func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - now := time.Now() - sent := now.Add(-time.Minute) - f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, - Resources: []inventory.ResourceHealth{foundDirectory("notes", sent.Add(-24*time.Hour))}}}} - h, why := moduleHealthWord("notes", "laptop", sent, f) - if h != healthPerson || !strings.Contains(why, "notes.data") || !strings.Contains(why, "hand-over") { - t.Fatalf("a directory found before the send reads %v %q; want a wait for a person", h, why) - } - // Found by this very send: the send brought it, and it is not passed. - f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now, - Resources: []inventory.ResourceHealth{foundDirectory("notes", sent.Add(time.Second))}} - if h, why := moduleHealthWord("notes", "laptop", sent, f); h != healthNotYet { - t.Fatalf("a directory this send found reads %v %q; want not yet", h, why) - } - // A container down beside the old wait is a fault, as before. - f.health["laptop"] = inventory.NodeHealth{Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{ - foundDirectory("notes", sent.Add(-time.Hour)), - {Module: "notes", Resource: "notes.web", Kind: "container", Target: "notes", State: link.StateUnhealthy, Reason: "down"}}} - if h, why := moduleHealthWord("notes", "laptop", sent, f); h != healthNotYet { - t.Fatalf("a container down beside the wait reads %v %q; want not yet", h, why) - } -} - -// The whole walk: a module whose directory was used as found long before still gets its fix to every machine, -// its pass kept and the wait said; a directory this very send found holds it and puts it back. - -// The condition says the wait in its own kind: the operator's, never urgent, and cleared once handed over. diff --git a/cmd/mesh-controller/gate.go b/cmd/mesh-controller/gate.go deleted file mode 100644 index 30a7499d..00000000 --- a/cmd/mesh-controller/gate.go +++ /dev/null @@ -1,1651 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "slices" - "sort" - "strings" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/micro" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" - "github.com/novox/mesh-controller/internal/link" -) - -// The gate on a release plan's first machine, and the rollback after it (novox/hq ADR 0236, to-be 45 -// §8, ADR 0227 rule 8). -// -// **"Reported applied" is not enough.** ADR 0218 sent a module to one machine first and the rest once -// that machine reported it applied. A build that applies and then does nothing, crashes, serves no -// tools, or breaks the machine's word to the mesh passed that test. Now the first machine is judged by -// the component's health — the core's health definitions, or a module's own — within ten minutes of the -// apply, each module by its rule (gateRuleOf): the node-engine and the mesh's resolver passing on their third -// healthy reading, and never later than two minutes after the send while healthy; the controller and the tool runner once, no earlier than the end of the -// node-engine's witness window for it; mesh-delivery and a leaf module once, as soon as the node-engine states -// it healthy (novox/hq ADR 0351 and ADR 0352, changing ADR 0236 decision 3, ADR 0282 decision 4 and ADR 0304 -// decision 3). -// Only then are the rest sent. -// -// **A failing gate stops the plan and puts the previous build back there.** The build is marked failed -// at its gate — registration refuses it and nothing sends it again on its own — the module's registered -// build goes back to the one the first machine ran before, and that machine is sent it: the ordinary -// path, again. Once per build: the verdict is written before the send, and a verdict once written is -// not written over. Said as a condition, urgent for the core and when it could not be put back, and as -// the event `rolled-back`. - -// The gate's bounds (to-be 45 §8). Variables so a test can judge in a second, not in minutes. -var ( - // gateWithin is the longest a module under the three-reading rule that stays healthy is held: at it, the - // gate passes on the healthy readings it has, the newest healthy and none since the send not (novox/hq ADR - // 0352). Readings gateEvery apart, the first at the apply, put three inside it. - gateWithin = 2 * time.Minute - // gateBound is how long after the apply the build may take to become healthy. - gateBound = 10 * time.Minute - // gatePasses is how many consecutive judgings must find such a core module healthy, gateEvery apart at - // the least, counted from the send: it passes on the third (novox/hq ADR 0352). gateLeafPasses is a leaf module's: one, taken once the - // node-engine states every resource of it healthy by its own check, with no settle (novox/hq ADR 0282 - // decision 4 as ADR 0304 decision 3 changed it, issue 524). - gatePasses = 3 - gateLeafPasses = 1 - gateLeafSettles = false - gateEvery = 40 * time.Second - // gateWitnessWindow is the node-engine's witness window for a new controller or tool runner, from its - // start (ADR 0236 decision 27, lease.ControllerWithin and lease.NodeToolsWithin): a reading of either - // counts only after it has ended, so the build is judged by the witness and the gate both (novox/hq ADR - // 0351 decisions 1 and 4). - gateWitnessWindow = max(lease.ControllerWithin, lease.NodeToolsWithin) -) - -// coreModule is whether a module is judged as the core (novox/hq ADR 0282 decision 1): one on the controller's -// own path, or one holding the mesh's resolver. Every other module is a leaf. Its class sets the module's -// delivery budget; how many readings its gate takes is gateRuleOf's. -func coreModule(module string, m catalogue.Manifest) bool { - if onTheOwnPath(module, m) { - return true - } - for _, c := range m.Claims { - if c.Name == resolverSeat { - return true - } - } - return coreComponent(module) != "" -} - -// gateRule is how a module's first-node gate judges it: passes healthy readings, gateEvery apart at the -// least, counted from the send or, where witnessed, from the end of the node-engine's witness window for it; -// and, where within is set, passed at within after the send on fewer, if it was healthy at every reading -// (novox/hq ADR 0352). said names the rule in the gate's text. -type gateRule struct { - passes int - within time.Duration - witnessed bool - said string -} - -// gateRuleOf is a module's gate rule (novox/hq ADR 0351): a core module keeps three readings only where its -// build, broken on every machine, could not receive its own fix — the node-engine, which applies every send, -// and the mesh's resolver, which gives every machine the bus's name (decisions 2, 3 and 6); the bus is never -// walked through a gate but kept so (decision 7). The controller, run on one machine, and the tool runner, -// whose fix the node-engine applies, are judged by one reading after the witness window (decisions 1 and 4); -// mesh-delivery, whose fix goes the controller's own path, by one (decision 5); a leaf module by one (ADR 0304 -// decision 3). -func gateRuleOf(module string, m catalogue.Manifest) gateRule { - switch { - case coreComponent(module) == lease.ComponentController: - return gateRule{passes: 1, witnessed: true, said: "the controller's rule: one reading once it is ready, " + - "after the node-engine's witness window (novox/hq ADR 0351)"} - case coreComponent(module) == lease.ComponentNodeTools: - return gateRule{passes: 1, witnessed: true, said: "the tool runner's rule: one reading once it answers, " + - "after the node-engine's witness window (novox/hq ADR 0351)"} - case m.ClaimsSeat(catalogue.DeliverySeat): - return gateRule{passes: 1, said: "mesh-delivery's rule: one reading (novox/hq ADR 0351)"} - case coreModule(module, m): - return gateRule{passes: gatePasses, within: gateWithin, said: fmt.Sprintf("the core's rule: %d readings, "+ - "or healthy throughout %s after the send (novox/hq ADR 0352)", gatePasses, gateWithin)} - } - r := gateRule{passes: gateLeafPasses, said: "a leaf module's rule: one reading (novox/hq ADR 0304)"} - if gateLeafSettles { - r.within = gateWithin - r.said = fmt.Sprintf("a leaf module's rule, judged as the core: %d readings, or healthy throughout %s after the send", - gateLeafPasses, gateWithin) - } - return r -} - -// gateFloor is the earliest a module's reading counts by its rule: the send and, for a witnessed module, the -// end of the node-engine's witness window on every machine judged (novox/hq ADR 0351 decisions 1 and 4). A -// module under the three-reading rule counts from the send (novox/hq ADR 0352). The window opens when the node-engine starts the build, which is before the machine reports it -// applied (applied, as the gate first read it) and, for the controller, before the new controller took the -// lease (holder): the later of those, the send's time at the least, plus the window, is never before its end. -// A controller that starts again takes the lease again, and so moves its floor on. Pure. -func gateFloor(module string, rule gateRule, since time.Time, machines []string, applied map[string]time.Time, - holder *lease.Holder) time.Time { - floor := since - if !rule.witnessed { - return floor - } - for _, n := range machines { - start := since - if at, read := applied[n]; read && at.After(start) { - start = at - } - if coreComponent(module) == lease.ComponentController && holder != nil && holder.Taken.After(start) { - start = holder.Taken - } - if end := start.Add(gateWitnessWindow); end.After(floor) { - floor = end - } - } - return floor -} - -// gateProbe is the registry's row for the gate's verdicts and the witnesses' rollbacks: its conditions -// are raised by a plan as it judges, and kept or cleared by the probe on every run. -const gateProbe = "DG" - -// The kinds a gate raises. -const ( - kindRolledBack = "rolled-back" - kindRollbackFailed = "rollback-failed" -) - -// coreComponent is the core component a module is, as a witness names it — controller, node-engine, -// node-tools — or empty: the core is judged by its health definitions, anything else by its own health. -func coreComponent(module string) string { - switch module { - case catalogue.ControllerSeatName: - return lease.ComponentController - case hostModule: - return lease.ComponentEngine - case broker.RuntimeModule: - return lease.ComponentNodeTools - } - return "" -} - -// health is a judging's word on one machine. -type health int - -const ( - healthGood health = iota - // healthPerson is a wait for a person (novox/hq ADR 0254): everything unhealthy of the module waits for - // one person's new login (personWait). The build did what it should; no bound a machine can meet ends - // the wait. It counts as a pass, and the gate carries the reading along in its verdict. - healthPerson - // healthWaiting is not a pass and not a fault: what the module's checks find waits on an unhealthy - // provider (ADR 0240 rule 5), so the judging waits — past the bound too — rather than putting back a - // build for something it did not do. - healthWaiting - healthNotYet - healthBroken - // healthSuperseded is a judging that cannot go on: the machine was sent another build of the module - // after the gate's send (novox/hq issue 352). No verdict on the build judged, and nothing put back. - healthSuperseded -) - -// served is what one machine's node tools answered the bus's discovery with. -type served struct { - runtime bool - tools map[string]bool -} - -// gateFacts is what one judging reads, gathered once for every machine it judges. -type gateFacts struct { - now time.Time - reports map[string]inventory.Reported - // engines is each machine's node-engine build as it last reported it. - engines map[string]string - // served is what the bus's discovery answered; servedErr why it could not be asked. - served map[string]served - servedErr error - // open are the open conditions; openErr why they could not be read (nil keeper: not judged). - open []conditions.Condition - openErr error - judged bool - // rolledBack is what each machine's witnesses say they decided. - rolledBack map[string][]lease.Rollback - // holder is who holds the controller lease, for judging the controller. - holder *lease.Holder - holderErr error - // health is each machine's newest health statement (ADR 0240); a machine absent never stated one. - // healthErr is why they could not be read. - health map[string]inventory.NodeHealth - healthErr error - // heldOn is, per "@", the provider its findings are held under (ADR 0240 rule 5). - heldOn map[string]heldReading - // groupsAdded is, per module, whether the move judged puts an account in a group its previous build did - // not (issue 318 review): the only move whose wait for a new login is excused. - groupsAdded map[string]bool - // waits is what the controller holds to check a module's wait for the operator (novox/hq ADR 0283): the - // manifest of each module's build judged, and the secrets given on each machine. - waits operatorWaitFacts - // sent is, per machine, the declaration the gate's own send carried there (novox/hq issue 352): a - // report is held against it, never against the send made last. sentBuilds is what each machine was - // last sent of every module, and judged the commit of each module this gate judges: a machine last - // sent another build of the module is not running the build judged. - sent map[string]inventory.SentDeclaration - sentBuilds map[string]map[string]string - commits map[string]string - // provisioned answers each machine's provisioner client's newest report of a module (novox/hq ADR 0349 §15): - // a module the gate's send handed to the provisioner is healthy only once it reported that build placed. Nil - // reads as nothing said. - provisioned inventory.ProvLookup -} - -// heldReading is the provider a module's findings are held under, as " on ", and, when that -// provider only waits for the operator for the part the module needs, its wait in one sentence (novox/hq issue -// 405): the module's gate then reads as a wait for a person (ADR 0254), a pass carrying the wait, as ADR 0283 -// decision 4 reads the waiting provider itself. A walk never waits on the operator's secret, there or here. -type heldReading struct { - on, waits string -} - -// heldReadings is, per "@" in every machine's newest statement, what its findings are held under. -func heldReadings(hold *holding) map[string]heldReading { - out := map[string]heldReading{} - for machine := range hold.healths { - for module, by := range hold.heldModules(machine) { - reading := heldReading{on: by.provider.Module + " on " + by.provider.Node} - if len(by.waits) > 0 { - reading.waits = operatorWaitSaid(by.provider.Module, by.provider.Node, by.waits) - } - out[module+"@"+machine] = reading - } - } - return out -} - -// reportedOn says a machine's last report is on what the gate sent it (novox/hq issue 352): on that -// declaration, or one it was sent after it — or, for a gate kept before sends were kept on it, on the -// declaration last sent. On 2026-10-09 a release's gate read the control node's report against a newer -// send another plan had just made there, and failed three builds the machine had reported healthy as -// "has not reported on what it was sent". -func (f gateFacts) reportedOn(machine string, r inventory.Reported) bool { - if sent, kept := f.sent[machine]; kept { - return sent.ReportsOn(r) - } - return r.Current -} - -// supersededOn says the machine was last sent another build of the module than the one this gate judges -// (novox/hq issue 352): the judging cannot go on, whatever the machine reports. On 2026-10-09 a controller -// put back by one gate judged another gate's newer controller build passed on the same machine, reading -// the put-back build's health as the newer one's. -func (f gateFacts) supersededOn(module, machine string) (string, bool) { - judged, known := f.commits[module] - sent, has := f.sentBuilds[machine][module] - if !known || !has || judged == "" || sent == "" || sameCommit(sent, judged) { - return "", false - } - return fmt.Sprintf("%s was sent %s %s after this gate's %s: the build judged no longer runs there, and "+ - "this judging is superseded by that send's", machine, module, short(sent), short(judged)), true -} - -// gatherGateFacts reads what a judging needs, from the store, the bus and this controller's memory. A -// variable so a test can hand a judging its facts. -var gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - inv := open.inventory - f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: witnessed.all(), provisioned: provLookup(ctx)} - reports, err := inv.LastReports(ctx) - if err != nil { - return f, err - } - for _, r := range reports { - f.reports[r.Node] = r - } - nodes, err := inv.Nodes(ctx) - if err != nil { - return f, err - } - for _, n := range nodes { - f.engines[n.Name] = n.HostVersion - } - // What each machine says of its long-running resources (ADR 0240): unreadable is said, never read as - // healthy. - f.health, f.healthErr = inv.Healths(ctx) - if d := doctorFrom; d != nil { - if d.keeper != nil { - f.judged = true - f.open, f.openErr = d.keeper.Open(ctx) - } - if d.js != nil { - f.served, f.servedErr = servedOnTheBus(ctx, d.js.Conn()) - } else { - f.servedErr = errors.New("this controller has no bus to ask") - } - } else { - f.servedErr = errors.New("this process does not serve the mesh, so it cannot ask the bus who serves what") - } - // Whose findings wait on an unhealthy provider (ADR 0240 rule 5): their gates wait, not fail. - if f.healthErr == nil && f.openErr == nil { - if hold, err := readHolding(ctx, inv, f.open); err == nil { - f.heldOn = heldReadings(hold) - } - } - if theLease != nil { - h, found, err := theLease.holder(ctx) - switch { - case err != nil: - f.holderErr = err - case found: - f.holder = &h - } - if component != lease.ComponentController && f.holderErr != nil { - f.holderErr = nil // read only for the controller's own judging - } - } - return f, nil -} - -// judgeHealth is one machine's health for a module's new build, from what one judging read: healthy, -// not yet (with what is wanting), or healthBroken — a witness put it back, or the machine refused or failed -// what it was sent. Pure. -func judgeHealth(module, component string, m catalogue.Manifest, machine string, since time.Time, f gateFacts) (health, string) { - // A witness's verdict made since the build was sent: the build failed its health there. One that - // could not judge at all (unwitnessed) is not a verdict on the build. - for _, r := range f.rolledBack[machine] { - if component == "" || r.Component != component || r.Outcome == lease.OutcomeUnwitnessed || r.At.Before(since) { - continue - } - return healthBroken, fmt.Sprintf("the witness on %s judged the %s %s and %s: %s", machine, r.Component, - short(r.From), r.Outcome, r.Why) - } - if why, superseded := f.supersededOn(module, machine); superseded { - return healthSuperseded, why - } - r, said := f.reports[machine] - switch { - case !said || r.At == nil || !f.reportedOn(machine, r): - return healthNotYet, fmt.Sprintf("%s has not reported on what it was sent", machine) - case r.Outcome == inventory.OutcomeFailed || r.Outcome == inventory.OutcomeRefused: - return healthBroken, fmt.Sprintf("%s %s what it was sent", machine, r.Outcome) - case r.Outcome != inventory.OutcomeApplied: - return healthNotYet, fmt.Sprintf("%s reported %q", machine, r.Outcome) - } - // **And its files placed** (novox/hq ADR 0349 §15): a module the send handed to the provisioner is judged only - // once that machine's client reported the build's templates placed, so a module whose files are not yet written - // is never judged; a failure there is the build's on that machine. - if h, why := judgeProvisioned(module, machine, r, f); h != healthGood { - return h, why - } - // **No new condition about it**: about the machine itself, or naming the module on that machine, - // raised since the judging began. The gate's own are not evidence about the build. A fault that was - // there at the send and reopened since is not new; one that had cleared before the send and came back - // after it is (OpenAt, novox/hq issue 348). - if f.judged { - if f.openErr != nil { - return healthNotYet, "what is wrong cannot be read, so whether the build made anything wrong is not known: " + - firstLine(f.openErr.Error()) - } - for _, c := range f.open { - // A wait for a person's new login, for a directory used as found to be handed over, or for the - // operator's secret or setting, is the module's reading, not a fault raised since the send: the gate - // reads it from the statement below (ADR 0254, novox/hq issue 339, ADR 0283). - if c.Source == gateProbe || c.OpenAt(since) || c.Kind == kindReloginNeeded || c.Kind == kindUsedAsFound || - c.Kind == kindNeedsOperator { - continue - } - onIt := c.Subject.Machine == machine || slices.Contains(c.Subject.Also, machine) || - (c.Subject.Scope == conditions.ScopeMachine && c.Subject.ID == machine) - if !onIt { - continue - } - // A module's own health condition names it whole, its name's dots and all (ADR 0240). - ownHealth := c.Subject.Scope == conditions.ScopeModule && c.Subject.ID == module+"."+machine - if c.Subject.Scope == conditions.ScopeMachine || ownHealth || slices.Contains(strings.Split(c.Subject.ID, "."), module) { - return healthNotYet, fmt.Sprintf("raised since it was sent: %s — %s", c.Key, c.Summary) - } - } - } - switch component { - case lease.ComponentEngine: - // The node-engine has reported its current declaration under its own build. - want := deliveredVersions(m) - if len(want) > 0 && !slices.Contains(want, f.engines[machine]) { - return healthNotYet, fmt.Sprintf("%s's node-engine reports build %s, not the new %s", machine, - orNotKnown(f.engines[machine]), strings.Join(want, " or ")) - } - case lease.ComponentNodeTools: - // The node tools are announced and answer. - if f.servedErr != nil { - return healthNotYet, "whether the node tools answer cannot be asked: " + firstLine(f.servedErr.Error()) - } - if !f.served[machine].runtime { - return healthNotYet, fmt.Sprintf("the node tools on %s do not answer the bus", machine) - } - case lease.ComponentController: - // The new controller holds the lease and says it is ready. - switch { - case f.holderErr != nil: - return healthNotYet, "who holds the controller lease cannot be read: " + firstLine(f.holderErr.Error()) - case f.holder == nil: - return healthNotYet, "no controller holds the lease" - case f.holder.Taken.Before(since.Add(-time.Minute)): - return healthNotYet, fmt.Sprintf("the lease is held since %s, by a controller older than the new build", - f.holder.Taken.UTC().Format(time.RFC3339)) - case f.holder.Health == nil || !f.holder.Health.Ready: - why := "the controller holding the lease does not say it is ready" - if f.holder.Health != nil && f.holder.Health.Why != "" { - why += ": " + f.holder.Health.Why - } - return healthNotYet, why - } - default: - // A module's tools answer, where it has any and the machine runs the node tools that serve them. - if len(m.Tools) > 0 { - if f.servedErr != nil { - return healthNotYet, "whether its tools are served cannot be asked: " + firstLine(f.servedErr.Error()) - } - if s := f.served[machine]; s.runtime && !s.tools[module] { - return healthNotYet, fmt.Sprintf("the node tools on %s do not serve %s's tools", machine, module) - } - } - // **And what it runs is stated healthy** (ADR 0240 §4): every long-running resource of it on that - // machine, in a statement heard since the send. A resource still starting makes the judging wait. - if h, why := moduleHealthWord(module, machine, since, f); h != healthGood { - return h, why - } - } - return healthGood, "" -} - -// judgeProvisioned is whether a module's files are placed on a machine, for a module the gate's send handed to the -// provisioner there: healthy for any other. -func judgeProvisioned(module, machine string, r inventory.Reported, f gateFacts) (health, string) { - sent, kept := f.sent[machine] - digest := sent.Provisioned[module] - if !kept || digest == "" { - return healthGood, "" - } - at := time.Time{} - if r.At != nil { - at = *r.At - } - one := inventory.SentDeclaration{Digest: sent.Digest, At: sent.At, Provisioned: map[string]string{module: digest}} - got, ok := inventory.ProvisionedSend(machine, one, inventory.AppliedReport{At: at, - Outcome: inventory.OutcomeApplied}, true, f.provisioned) - switch { - case !ok: - return healthNotYet, fmt.Sprintf("the provisioner on %s has not reported %s's files placed for the new "+ - "build", machine, module) - case got.Outcome != inventory.OutcomeApplied: - return healthBroken, fmt.Sprintf("the provisioner on %s did not place %s's files: %s", machine, module, - got.Outcome) - } - return healthGood, "" -} - -// machineWord is what one judging found wrong with a machine itself, apart from its modules: facts is -// the judging's facts without those conditions, on what each names among the modules the send moved, -// and whole what holds the machine back as a whole. -type machineWord struct { - facts gateFacts - on map[string]string - whole string - // waiting is what holds the machine on something shown to be another's (ADR 0241): the judging waits. - waiting string -} - -// kindCoreBehind is D10's kind: a machine runs core components older than the mesh holds, or has not -// yet reported applying the node tools it was last sent. -const kindCoreBehind = "core-behind" - -// aboutTheMachine sorts the conditions raised about a machine itself since a send was made there -// (novox/hq issue 281). The gate read every one of them as the module's it was kept on: a machine-level -// condition caused by anything else in the send — or by a tier sent one module at a time — failed that -// module, at the bound, with a reason that was never about it. -// -// - one naming a module the send moved is that module's; -// - the core being behind (D10) is the core's: of the node-engine or the node tools when the send -// moved them — inside their settle window, which is the gate's bound — and otherwise no evidence about -// what was sent: a send not yet applied the machine's reports already say, and a newer core build -// that no plan sends is not this send's; -// - anything else holds the machine back as a whole: everything the send moved there waits on it, and -// fails with it, together, at the bound, with that reason. -// -// Pure. -func aboutTheMachine(machine string, moved []string, since time.Time, f gateFacts) machineWord { - w := machineWord{facts: f, on: map[string]string{}} - if !f.judged || f.openErr != nil { - return w - } - var core []string - for _, m := range moved { - if c := coreComponent(m); c == lease.ComponentEngine || c == lease.ComponentNodeTools { - core = append(core, m) - } - } - kept := make([]conditions.Condition, 0, len(f.open)) - for _, c := range f.open { - aboutIt := c.Subject.Scope == conditions.ScopeMachine && (c.Subject.ID == machine || c.Subject.Machine == machine || - slices.Contains(c.Subject.Also, machine)) - // A directory used as found waits for a person, whatever the send did (novox/hq issue 339). - if !aboutIt || c.Source == gateProbe || c.OpenAt(since) || c.Kind == kindUsedAsFound { - kept = append(kept, c) - continue - } - said := fmt.Sprintf("raised since it was sent: %s — %s", c.Key, c.Summary) - parts := strings.Split(c.Subject.ID, ".") - var named []string - for _, m := range moved { - if slices.Contains(parts, m) { - named = append(named, m) - } - } - coreBehind := c.Kind == kindCoreBehind || strings.HasSuffix(c.Key, "."+kindCoreBehind) - switch { - case len(named) > 0: - for _, m := range named { - if _, already := w.on[m]; !already { - w.on[m] = said - } - } - case coreBehind && len(core) > 0: - for _, m := range core { - if _, already := w.on[m]; !already { - w.on[m] = said + " (its settle window runs to the gate's bound)" - } - } - case coreBehind: - // Not what the send moved: said nowhere against it. - case c.Kind == kindMachineUnits: - // **The machine's own failed units** (issue 315): no module places any of them, so nothing a - // send moved is among them — a module's failed unit is that module's own condition. Said - // nowhere against the send. - case c.Kind == kindNetworkRewritten || c.Kind == kindNetworkUnreachable && c.Subject.ID != machine: - // **Shown to be somebody else's** (ADR 0241): another program rewrote the resolver file the send - // did not move, or the machine cannot reach another that is down. Nothing the send did; the - // judging waits for it rather than putting back a build at the bound. - if w.waiting == "" { - w.waiting = machine + "'s network: " + said - } - default: - if w.whole == "" { - w.whole = machine + " as a whole: " + said - } - } - } - w.facts.open = kept - return w -} - -// judgeGate takes one judging of a module's first machines and records it in the plan's gate: a pass -// counted, a pass missed (and why), or the verdict. Answers the verdict once there is one. -func judgeGate(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule, - running []string, now time.Time) (string, error) { - g := state.Gate - if g == nil { - // Judged from the send: a witness's verdict, a condition, the bound — all counted from when the - // first machine was sent the build. - start := now - if state.FirstAt != nil { - start = *state.FirstAt - } - var machines []string - for _, n := range state.First { - if slices.Contains(running, n) { - machines = append(machines, n) - } - } - g = &inventory.PlanGate{Component: coreComponent(module), Machines: machines, From: state.Previous, - To: state.Commit, Since: &start} - state.Gate = g - } - pairs := []judged{} - for _, n := range g.Machines { - pairs = append(pairs, judged{module: module, node: n}) - } - return judgeMoves(ctx, open, g, pairs, now) -} - -// dueAtTheMost says a gate healthy since its send has reached gateWithin and not been judged since: the one -// judging that may come sooner than gateEvery after the last pass (novox/hq ADR 0352). -func dueAtTheMost(g *inventory.PlanGate, now time.Time) bool { - most := g.Since.Add(gateWithin) - return !g.Faulted && g.LastPass != nil && !now.Before(most) && g.LastPass.Before(most) -} - -// judged is one module on one machine, as a gate judges it. -type judged struct{ module, node string } - -// judgeMoves takes one judging of a gate over the modules it judges on their machines — its own, and -// everything the send carried (Carried) — and records it: a pass counted, a pass missed (what is -// wanting, and which modules), or the verdict. Answers the verdict once there is one. -func judgeMoves(ctx context.Context, open *stores, g *inventory.PlanGate, pairs []judged, now time.Time) (string, error) { - if g.Verdict != "" { - return g.Verdict, nil - } - // A judging comes gateEvery after the last pass, and once at gateWithin after the send whatever the spacing: - // a module healthy throughout is never held past it (novox/hq ADR 0352). - if g.LastPass != nil && now.Sub(*g.LastPass) < gateEvery && !dueAtTheMost(g, now) { - return "", nil - } - for _, c := range g.Carried { - if !slices.Contains(pairs, judged{module: c.Module, node: c.Node}) { - pairs = append(pairs, judged{module: c.Module, node: c.Node}) - } - } - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - return "", err - } - facts, err := gatherGateFacts(ctx, open, g.Component) - if err != nil { - return "", err - } - facts.groupsAdded = movesAddingGroups(ctx, open.inventory, g, pairs, shelf) - facts.waits = gateWaitFacts(ctx, open.inventory, g, pairs, shelf, facts.health) - facts.sent = g.Sent - facts.commits, facts.sentBuilds = judgedCommits(g, pairs), map[string]map[string]string{} - // A module this gate put back at once (putBackBroken) was sent its earlier build by the gate itself: - // not another send, and not a judging superseded. - for _, m := range g.Returned { - delete(facts.commits, m) - } - for _, j := range pairs { - if _, read := facts.sentBuilds[j.node]; read { - continue - } - if builds, known, err := open.inventory.SentBuilds(ctx, j.node); err == nil && known { - facts.sentBuilds[j.node] = builds - } - } - // **A busy container on a judged machine holds the judging before it starts** (novox/hq ADR 0335 §4): the - // walk says it waits there, and its gate starts at the report that says the container replaced. - if busyWait(g, pairs, facts, now) { - return "", nil - } - // **What is wrong with a machine itself is the machine's** (novox/hq issue 281): read once for each - // machine judged, apart from what is wrong with a module there, and never pinned on the module the - // gate happens to be kept on. - byMachine := map[string][]string{} - for _, j := range pairs { - byMachine[j.node] = append(byMachine[j.node], j.module) - } - words := map[string]machineWord{} - for node, moved := range byMachine { - words[node] = aboutTheMachine(node, moved, *g.Since, facts) - } - // **Each module is judged on its own** (novox/hq ADR 0254, issue 318): its reading is the worst of its - // machines', its passes are counted apart, and the send's verdict is still one. **Every module of a send - // leaves it with a verdict** (issue 318 review): one healthy for the passes the gate asks, after the - // settle time, keeps a pass when another beside it fails; one found broken holds the send's verdict until - // the modules beside it have theirs, or the bound; and at the verdict, every module that did not pass is - // put back with what failed, so none is left on the machine unjudged for other walks to wait on. - worst, why := healthGood, "" - reading := map[string]health{} - waits := map[string]string{} - var modules []string - for _, j := range pairs { - w := words[j.node] - h, said := judgeHealth(j.module, coreComponent(j.module), shelf[j.module], j.node, *g.Since, w.facts) - if why, still := busyStill(j, facts); still && (h == healthGood || h == healthPerson) { - h, said = healthNotYet, why - } - if h == healthGood || h == healthPerson { - if on, named := w.on[j.module]; named { - h, said = healthNotYet, on - } else if w.whole != "" { - h, said = healthNotYet, w.whole - } else if w.waiting != "" { - h, said = healthWaiting, w.waiting - } - } - if _, seen := reading[j.module]; !seen { - modules = append(modules, j.module) - } - if h == healthSuperseded { - // Decided at once (novox/hq issue 352): nothing of this gate's can be judged on a machine that - // was sent another build of it, and nothing is put back — the later send is what runs there. - g.Failing, g.Last = nil, "" - decide(g, inventory.GateSuperseded, said, now) - return g.Verdict, nil - } - if h == healthBroken && !slices.Contains(g.Broken, j.module) { - g.Broken = append(g.Broken, j.module) - if g.BrokenWhy == "" { - g.BrokenWhy = said - } - } - if slices.Contains(g.Broken, j.module) { - h = healthBroken // found broken once, broken for the rest of the judging - } - if h > reading[j.module] { - reading[j.module] = h - } - if h == healthPerson { - waits[j.module] = joinSaid(waits[j.module], said) - } - if h > worst { - worst, why = h, said - } else if h == worst && h > healthPerson && why == "" { - why = said - } - } - // **Each module counts its readings by its own rule** (novox/hq ADR 0351): from when the gate first read - // each machine report its send applied, and who holds the controller lease, the earliest its reading counts. - if g.Applied == nil { - g.Applied = map[string]time.Time{} - } - for node := range byMachine { - r, said := facts.reports[node] - if _, read := g.Applied[node]; !read && said && r.At != nil && r.Outcome == inventory.OutcomeApplied && - facts.reportedOn(node, r) { - g.Applied[node] = *r.At - } - } - rules, floors := map[string]gateRule{}, map[string]time.Time{} - for _, m := range modules { - var on []string - for _, j := range pairs { - if j.module == m { - on = append(on, j.node) - } - } - rules[m] = gateRuleOf(m, shelf[m]) - floors[m] = gateFloor(m, rules[m], *g.Since, on, g.Applied, facts.holder) - } - tallyGate(g, gateJudging{modules: modules, reading: reading, waits: waits, worst: worst, why: why, - rules: rules, floors: floors}, now) - return g.Verdict, nil -} - -// gateJudging is one judging's readings, as tallyGate counts them: the modules judged, each one's reading -// (the worst of its machines'), its waits for a person, the send's worst reading and why, and each module's -// rule and the earliest its reading counts. -type gateJudging struct { - modules []string - reading map[string]health - waits map[string]string - worst health - why string - rules map[string]gateRule - floors map[string]time.Time -} - -// tallyGate counts one judging into the gate: a pass counted, a pass missed (and why), or the verdict. Pure but -// for the gate it writes. -func tallyGate(g *inventory.PlanGate, j gateJudging, now time.Time) { - modules, reading, waits, worst, why := j.modules, j.reading, j.waits, j.worst, j.why - if g.Healthy == nil { - g.Healthy = map[string]int{} - } - if g.HealthyAt == nil { - g.HealthyAt = map[string]time.Time{} - } - g.Waits = nil - var failing []string - for _, m := range modules { - if reading[m] > healthPerson { - failing = append(failing, m) - g.Healthy[m] = 0 - delete(g.HealthyAt, m) - continue - } - if w := waits[m]; w != "" { - if g.Waits == nil { - g.Waits = map[string]string{} - } - g.Waits[m] = w - } - // **A reading counts only from the module's floor** (novox/hq ADR 0351 decisions 1 and 4): the - // controller's and the tool runner's after the witness window, any other module's from the send. - if now.Before(j.floors[m]) { - continue - } - // **A pass is counted only gateEvery after the one before** (issue 318 review): a send judged more - // often while another module beside it is not yet healthy counts no faster. - if at, counted := g.HealthyAt[m]; !counted || now.Sub(at) >= gateEvery { - g.Healthy[m]++ - g.HealthyAt[m] = now - } - } - // What the send needs is what its most demanding module needs — the most readings, then the latest - // floor: a send carrying the node-engine is judged by its three readings, or by being healthy throughout - // its two minutes (novox/hq ADR 0352). - wants, floor, within, said := 0, time.Time{}, time.Duration(0), "" - for _, m := range modules { - r, f := j.rules[m], j.floors[m] - if r.passes > wants || (r.passes == wants && f.After(floor)) { - wants, within, said = r.passes, r.within, r.said - } - if f.After(floor) { - floor = f - } - } - if len(modules) == 0 { - // A send judged on no machine of its modules (its first machine not running them) has no class to - // be judged by: it is judged as the core, never passed at once without a reading. - wants, floor, within = gatePasses, *g.Since, gateWithin - said = fmt.Sprintf("the core's rule: %d readings, or healthy throughout %s after the send", gatePasses, gateWithin) - } - // heldLongEnough is a rule's most: within after the send, healthy at every reading since it. - heldLongEnough := func(within time.Duration) bool { - return within > 0 && !g.Faulted && !now.Before(g.Since.Add(within)) - } - g.Wants = wants - // A module's own pass is its own rule's: a leaf beside a core module that fails keeps its build after - // its one reading, and only what did not pass is put back (issue 318's rule, by class since issue 524). - passedAlone := func(m string) bool { - r := j.rules[m] - return reading[m] <= healthPerson && (g.Healthy[m] >= r.passes || (g.Healthy[m] >= 1 && heldLongEnough(r.within))) - } - // fail decides the send failed: what passed on its own keeps its pass, everything else is put back. - fail := func(why string) { - g.Passing, g.Failing = nil, nil - for _, m := range modules { - if passedAlone(m) { - g.Passing = append(g.Passing, m) - } else { - g.Failing = append(g.Failing, m) - } - } - decide(g, inventory.GateFailed, why, now) - } - pastBound := now.Sub(*g.Since) > gateBound - switch { - case worst == healthBroken: - g.Faulted = true - g.Read(now, false, g.BrokenWhy) - var judging []string - for _, m := range modules { - if reading[m] != healthBroken && !passedAlone(m) { - judging = append(judging, m) - } - } - if len(judging) == 0 || pastBound { - fail(g.BrokenWhy) - break - } - // What broke fails the send, and is put back at once by the caller (putBackBroken); what is beside - // it is judged to its own verdict first, within the bound. - g.Passes, g.LastPass, g.Failing = 0, nil, failing - g.Last = fmt.Sprintf("%s; %s judged to its own verdict before the send's", g.BrokenWhy, strings.Join(judging, ", ")) - case worst == healthWaiting: - // Waiting on a provider that is unhealthy: not a pass, and not a failure at the bound either — - // the provider's own condition says what is wrong (ADR 0240 rule 5). - g.Faulted = g.Faulted || g.HealthySeen // not healthy after a healthy reading; before one, still starting - g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing - g.Read(now, false, why) - case worst == healthNotYet: - g.Faulted = g.Faulted || g.HealthySeen // not healthy after a healthy reading; before one, still starting - g.Passes, g.LastPass, g.Last, g.Failing = 0, nil, why, failing - g.Read(now, false, why) - if pastBound { - fail(fmt.Sprintf("not healthy within %s of its apply: %s", gateBound, why)) - } - case now.Before(floor): - // Healthy before its rule lets a reading count: not a pass yet, and not a fault (novox/hq ADR 0351). - g.HealthySeen = true - // The bound still holds: a module that cannot be read healthy after its floor fails at it. - g.Last, g.Failing = fmt.Sprintf("healthy, and a reading counts from %s by %s", - floor.UTC().Format(time.RFC3339), said), nil - if pastBound { - fail(fmt.Sprintf("not judged within %s of its apply: %s", gateBound, g.Last)) - } - default: - // Healthy, or waiting for a person (ADR 0254): a pass, the wait carried along in the verdict. - g.HealthySeen = true - g.Passes++ - g.LastPass, g.Last, g.Failing = &now, "", nil - g.Read(now, true, "") - if g.Passes >= wants || heldLongEnough(within) { - decide(g, inventory.GatePassed, fmt.Sprintf("healthy %d times over %s, by %s", g.Passes, - now.Sub(*g.Since).Round(time.Second), said)+waitsSaid(g.Waits), now) - } - } -} - -// judgedCommits is the commit of each module a gate judges: the gate's own To for its module, and each -// carried move's. Pure. -func judgedCommits(g *inventory.PlanGate, pairs []judged) map[string]string { - out := map[string]string{} - for _, c := range g.Carried { - if c.To != "" { - out[c.Module] = c.To - } - } - if g.To != "" { - for _, j := range pairs { - if _, has := out[j.module]; !has && !slices.ContainsFunc(g.Carried, func(c inventory.CarriedMove) bool { return c.Module == j.module }) { - out[j.module] = g.To - } - } - } - return out -} - -// sentNow is what each machine was just sent, read after a send for the gate to keep (novox/hq issue -// 352): a machine whose send is not on record is left out, and its report is read as before. -func sentNow(ctx context.Context, inv *inventory.Inventory, machines []string) map[string]inventory.SentDeclaration { - out := map[string]inventory.SentDeclaration{} - for _, n := range machines { - if s, found, err := inv.SentTo(ctx, n); err == nil && found { - out[n] = s - } - } - if len(out) == 0 { - return nil - } - return out -} - -// whyFor is a passing gate's why as one module's verdict says it: the send's, and that module's own wait -// for a person, never another's (issue 318 review). -func whyFor(g *inventory.PlanGate, module string) string { - why, _, _ := strings.Cut(g.Why, waitsPrefix) - if w := g.Waits[module]; w != "" { - why += waitsPrefix + w - } - return why -} - -const waitsPrefix = "; and it waits for a person: " - -// waitsSaid is the waits for a person a passing gate carries, as its verdict says them. -func waitsSaid(waits map[string]string) string { - if len(waits) == 0 { - return "" - } - modules := make([]string, 0, len(waits)) - for m := range waits { - modules = append(modules, m) - } - sort.Strings(modules) - var said []string - for _, m := range modules { - said = append(said, waits[m]) - } - return waitsPrefix + strings.Join(said, "; ") -} - -func joinSaid(a, b string) string { - switch { - case a == "": - return b - case b == "" || strings.Contains(a, b): - return a - } - return a + "; " + b -} - -// movesAddingGroups is, per module a gate judges, whether its move puts an account in a group the build it -// moved from did not: read from the builds' manifests. A module whose move is not known adds none. -func movesAddingGroups(ctx context.Context, inv *inventory.Inventory, g *inventory.PlanGate, pairs []judged, - shelf map[string]catalogue.Manifest) map[string]bool { - out := map[string]bool{} - for _, j := range pairs { - if _, done := out[j.module]; done { - continue - } - from, to := g.From, g.To - for _, c := range g.Carried { - if c.Module == j.module { - from, to = c.From, c.To - break - } - } - target, found, err := inv.ManifestAt(ctx, j.module, to) - if err != nil || !found { - target = shelf[j.module] - } - // What it moved from is not known — no build named (a plan's own module whose previous build was not - // recorded), or no manifest kept for it: no wait is excused, rather than one the move did not bring. - if from == "" { - out[j.module] = false - continue - } - before, had, err := inv.ManifestAt(ctx, j.module, from) - if err != nil || !had { - out[j.module] = false - continue - } - out[j.module] = addsAccountGroups(before, had, target) - } - return out -} - -// gateWaitFacts reads what checks the waits for the operator of the modules a gate judges (novox/hq ADR 0283): the -// manifest of the build judged — the one it moves to, else the catalogue's — and the secrets given on each machine -// that says a module of them waits. -func gateWaitFacts(ctx context.Context, inv *inventory.Inventory, g *inventory.PlanGate, pairs []judged, - shelf map[string]catalogue.Manifest, health map[string]inventory.NodeHealth) operatorWaitFacts { - var f operatorWaitFacts - judgedManifests := map[string]catalogue.Manifest{} - byMachine := map[string][]string{} - for _, j := range pairs { - waiting := false - for _, r := range health[j.node].Resources { - if r.Module == j.module && r.State == link.StateWaiting { - waiting = true - } - } - if !waiting { - continue - } - byMachine[j.node] = append(byMachine[j.node], j.module) - if _, done := judgedManifests[j.module]; done { - continue - } - to := g.To - for _, c := range g.Carried { - if c.Module == j.module { - to = c.To - break - } - } - if m, found, err := inv.ManifestAt(ctx, j.module, to); err == nil && found { - judgedManifests[j.module] = m - } else if m, known := shelf[j.module]; known { - judgedManifests[j.module] = m - } - } - for machine, modules := range byMachine { - readWaitFacts(ctx, inv, machine, modules, judgedManifests, &f) - } - return f -} - -// decide sets a gate's verdict. -func decide(g *inventory.PlanGate, verdict, why string, now time.Time) { - g.Verdict, g.Why, g.JudgedAt = verdict, why, &now - g.Took = now.Sub(*g.Since).Round(time.Second).String() -} - -// gatePassed keeps a passing build's verdict, so `plans` and the gate's probe can read it. -func gatePassed(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule) { - g := state.Gate - err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module, - Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines, - Verdict: inventory.GatePassed, Why: whyFor(g, module), Component: g.Component, JudgingFrom: g.Since}) - if err != nil && state.Build != "" { - fmt.Printf("%s: %s passed its gate, and the verdict could not be kept: %v\n", p.ID, module, err) - } - passCarried(ctx, open, p, g, module) - fmt.Printf("%s: %s passed its gate on %s (%s); the rest are sent\n", p.ID, module, - strings.Join(g.Machines, ", "), g.Why) - if module == catalogue.ControllerSeatName { - carryUserList(ctx, open, p) - } -} - -// carryUserList sends the machine holding the bus the user list a new controller composes, once that -// controller passed its gate (ADR 0236). The controller's own grants travel in that list, and the old -// controller composed the list the plan sent; on 2026-10-06 eight pushes by hand carried a new -// controller's grant into it. Not when a build its policy or a plan holds back would go with it (ADR -// 0221): then it is said, as a push would say it. -func carryUserList(ctx context.Context, open *stores, p *inventory.Plan) { - holder, behind, err := brokerBehind(ctx, open, nil) - if err != nil || holder == "" || !behind { - if err != nil { - fmt.Printf("%s: whether the bus's user list is behind the new controller cannot be read: %v\n", p.ID, err) - } - return - } - held, err := heldMachines(ctx, open, []string{holder}) - if err != nil { - fmt.Printf("%s: whether %s may be sent the new user list cannot be read: %v\n", p.ID, holder, err) - return - } - if why, isHeld := held[holder]; isHeld { - fmt.Printf("%s: the new controller's user list is not carried to %s, which holds the bus: %s — `push %s` "+ - "carries it\n", p.ID, holder, strings.Join(why, "; "), holder) - return - } - if _, err := sendRollout(ctx, open, []string{holder}); err != nil { - fmt.Printf("%s: the new controller's user list could not be carried to %s: %v\n", p.ID, holder, err) - return - } - fmt.Printf("%s: carried the new controller's user list to %s, which holds the bus\n", p.ID, holder) -} - -// gateFailed stops the plan at a build that failed its gate and puts the previous build back on the -// machines it was judged on — once per build, said as a condition and an event. The plan is saved -// before the send: a controller that is itself the build being put back does not outlive it. -func gateFailed(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule, - machines []string, why string) { - inv := open.inventory - now := time.Now().UTC() - if state.Gate == nil { - state.Gate = &inventory.PlanGate{Component: coreComponent(module), Machines: machines, - From: state.Previous, To: state.Commit, Since: state.FirstAt} - } - g := state.Gate - if g.Verdict == "" { - if g.Since == nil { - g.Since = &now - } - decide(g, inventory.GateFailed, why, now) - } - if len(g.Machines) == 0 { - g.Machines = machines - } - state.Why = "failed its gate: " + g.Why - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("%s failed its gate on %s in tier %d: %s", module, strings.Join(g.Machines, ", "), p.Tier, g.Why) - - verdict := inventory.GateVerdict{Build: state.Build, Module: module, Commit: state.Commit, Previous: state.Previous, - Plan: p.ID, Machines: g.Machines, Verdict: inventory.GateFailed, Rollback: inventory.RollingBack, Why: g.Why, - Component: g.Component, JudgingFrom: g.Since} - // **A send that changed nothing of the module there is no verdict on its build** (novox/hq issue - // 280). The machine already ran this build, or one that made the same artifacts from the same - // manifest: whatever the gate found wanting, this build did not bring it, and there is nothing to - // put back. On 2026-10-06 such a module was marked failed, and the rollback looked for an earlier - // build of the very commit it had failed — "no build kept" — while the build it had run before was - // kept all along. Said, left as it is, never marked. - if unchangedBy(ctx, inv, module, state.Previous, state.Commit) { - g.Rollback = gateUnchanged - state.Why = "stopped with its send; the send changed nothing of it: " + g.Why - p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s was left as it was — %s already ran "+ - "%s %s, or a build identical to it, before the send, so nothing of it moved and nothing is put back", - strings.Join(g.Machines, ", "), p.Tier, g.Why, module, strings.Join(g.Machines, ", "), module, short(state.Commit)) - fmt.Printf("%s: %s\n", p.ID, p.Note) - return - } - if state.Build == "" { - // A plan from before builds were asked by id: nothing to mark, so nothing is put back by the - // mesh — said, for a person. - g.Rollback = inventory.NotRolledBack - p.Note += "; not put back: the plan does not know which build it sent" - sayRollback(ctx, open, module, g, "") - return - } - if err := inv.RecordGate(ctx, verdict); err != nil { - if errors.Is(err, inventory.ErrGateKept) { - // Already judged and acted on, by this controller before a restart or by another: never twice. - if kept, found, _ := inv.GateOf(ctx, state.Build); found { - g.Rollback = kept.Rollback - } - p.Note += "; its rollback was already made once and is not made again" - return - } - g.Rollback = inventory.NotRolledBack - p.Note += "; not put back: its verdict could not be kept, and a rollback that cannot be counted is not made — " + err.Error() - sayRollback(ctx, open, module, g, "") - return - } - // The previous build: the one the first machine ran, from the build records. - notBack := func(why string) { - g.Rollback = inventory.NotRolledBack - p.Note += "; NOT put back: " + why - if err := inv.SetRollback(ctx, state.Build, inventory.NotRolledBack, g.Why+"; not put back: "+why); err != nil { - fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, module, err) - } - sayRollback(ctx, open, module, g, why) - } - if state.Previous == "" { - // A first build there: nothing ran before it, so nothing can be put back, and the machine is left - // with it — said as that, not as a build the mesh lost. - notBack(fmt.Sprintf("this is the first build of %s that %s was sent, or what it was sent before is not known: "+ - "there is no earlier build there to put back, so it is left with this one — `unassign` takes it off, "+ - "a newer merge replaces it", module, strings.Join(g.Machines, ", "))) - return - } - failed, _, err := inv.BuildByID(ctx, state.Build) - if err != nil { - notBack("the failed build's record cannot be read: " + err.Error()) - return - } - previous, found, err := inv.PreviousBuild(ctx, module, state.Previous, failed) - if err != nil { - notBack("the build records cannot be read: " + err.Error()) - return - } - if !found { - notBack(fmt.Sprintf("no build of %s from %s, asked before the failed one, is among the %d newest kept to put "+ - "back", module, short(state.Previous), inventory.KeptBuilds)) - return - } - if g.Component == lease.ComponentController { - // **The controller is never put back to a build older than the store's schema** (novox/hq issue - // 352): the build before it carries fewer migrations than the failed one applied, starts behind - // its own records, and judges the next gate with what it can read. The current build is kept and - // the condition says so; a person decides. - if why, ok := controllerSchemaAllows(ctx, inv, previous); !ok { - notBack(why) - return - } - } - if err := inv.RestoreModule(ctx, previous); err != nil { - notBack(err.Error()) - return - } - g.Rollback = inventory.RollingBack - if err := inv.SavePlan(ctx, p); err != nil { - fmt.Printf("%s: the plan could not be kept before %s is put back: %v\n", p.ID, module, err) - } - // Put back with the rest of its send, in one send per machine (issue 281). - if b, batched := ctx.Value(rollbacksKey{}).(*rollbacks); batched { - b.pending = append(b.pending, pendingRollback{module: module, state: state, g: g, previous: previous}) - b.modules[module] = true - for _, n := range g.Machines { - if !slices.Contains(b.machines, n) { - b.machines = append(b.machines, n) - } - } - return - } - sent, err := sendRollout(withScope(ctx, sendScope{modules: map[string]bool{module: true}, puttingBack: true}), open, g.Machines) - if err != nil { - notBack(fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+ - "sends it", short(previous.Commit), strings.Join(g.Machines, ", "), err, g.Machines[0])) - return - } - g.Rollback = inventory.RolledBack - p.Note += fmt.Sprintf("; put back to %s on %s", short(previous.Commit), strings.Join(sent, ", ")) - if err := inv.SetRollback(ctx, state.Build, inventory.RolledBack, g.Why); err != nil { - fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, module, err) - } - sayRollback(ctx, open, module, g, "") -} - -// controllerSchemaAllows says the store's schema lets this build of the controller be put back: the -// build recorded, when it served, a reach at or past the highest migration the store has applied. One -// that never recorded a reach is not proved safe, and is refused as such (novox/hq issue 352). Why -// says what is kept and why when it is not. -func controllerSchemaAllows(ctx context.Context, inv *inventory.Inventory, previous inventory.Build) (string, bool) { - applied, err := inv.SchemaApplied(ctx) - if err != nil { - return "what the store's schema reaches cannot be read, so whether the build before it can read it is not " + - "known; the current build is kept: " + err.Error(), false - } - build := versionOfBuild(previous) - if build == "" { - return fmt.Sprintf("the build before it (%s) names no bundle to know it by, so whether it can read the store's "+ - "schema (migration %04d) is not known; the current build is kept, and a person decides", short(previous.Commit), applied), false - } - reach, known, err := inv.SchemaReachOf(ctx, build) - if err != nil { - return "what the build before it knows of the store's schema cannot be read; the current build is kept: " + err.Error(), false - } - if !known { - return fmt.Sprintf("the build before it (%s, %s) never recorded how far it reads the store's schema — a "+ - "controller records that when it serves — so it is not proved to read migration %04d, which the store "+ - "has applied; a controller older than its store starts behind its own records and judges with what it "+ - "can read, so the current build is kept, and a person decides", short(previous.Commit), build, applied), false - } - if reach < applied { - return fmt.Sprintf("the build before it (%s, %s) reads the store's schema up to migration %04d, and the store "+ - "is at %04d: a controller older than its store starts behind its own records and judges with what it "+ - "can read, so the current build is kept, and a person decides", short(previous.Commit), build, reach, applied), false - } - return "", true -} - -// versionOfBuild is the version a build's bundle is delivered as — its archive's digest, short, as the -// catalogue names it (`${version}`) — read from the build's artifacts; empty when none is a bundle. -func versionOfBuild(b inventory.Build) string { - for _, a := range b.Made { - if a.Kind != catalogue.ArtifactBundle && a.Kind != catalogue.ArtifactArchive { - continue - } - if _, hex, found := strings.Cut(a.Reference, "sha256:"); found && len(hex) >= 12 { - return hex[:12] - } - } - return "" -} - -// rollbacks is what a failed send puts back, sent together (novox/hq issue 281): a gate that judged one -// send judges what it moved as one, and what it found wanting goes back in one send per machine — not -// in a send for each module, which is the churn that failed the gate in the first place. -type rollbacks struct { - modules map[string]bool - machines []string - pending []pendingRollback -} - -type pendingRollback struct { - module string - state *inventory.PlanModule - g *inventory.PlanGate - previous inventory.Build -} - -type rollbacksKey struct{} - -// batchingRollbacks is a context under which gateFailed registers what it puts back and leaves the send -// to sendRollbacks. -func batchingRollbacks(ctx context.Context) (context.Context, *rollbacks) { - b := &rollbacks{modules: map[string]bool{}} - return context.WithValue(ctx, rollbacksKey{}, b), b -} - -// sendRollbacks sends what a failed send put back, once to each machine, and says each module's rollback. -func sendRollbacks(ctx context.Context, open *stores, p *inventory.Plan, b *rollbacks) { - if len(b.pending) == 0 { - return - } - inv := open.inventory - sort.Strings(b.machines) - sent, err := sendRollout(withScope(ctx, sendScope{modules: b.modules, puttingBack: true}), open, b.machines) - var back []string - for _, r := range b.pending { - if err != nil { - why := fmt.Sprintf("its registered build is back at %s, and sending it to %s was refused: %v — `push %s` "+ - "sends it", short(r.previous.Commit), strings.Join(r.g.Machines, ", "), err, firstOf(r.g.Machines)) - r.g.Rollback = inventory.NotRolledBack - if err := inv.SetRollback(ctx, r.state.Build, inventory.NotRolledBack, r.g.Why+"; not put back: "+why); err != nil { - fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, r.module, err) - } - sayRollback(ctx, open, r.module, r.g, why) - continue - } - r.g.Rollback = inventory.RolledBack - back = append(back, r.module+" to "+short(r.previous.Commit)) - if err := inv.SetRollback(ctx, r.state.Build, inventory.RolledBack, r.g.Why); err != nil { - fmt.Printf("%s: how %s's rollback went could not be kept: %v\n", p.ID, r.module, err) - } - sayRollback(ctx, open, r.module, r.g, "") - } - if err != nil { - p.Note += fmt.Sprintf("; NOT put back: sending %s was refused: %v", strings.Join(b.machines, ", "), err) - return - } - p.Note += fmt.Sprintf("; put back %s on %s, in one send", strings.Join(back, ", "), strings.Join(sent, ", ")) -} - -// gateUnchanged is a failed gate's word on a module its send changed nothing of (novox/hq issue 281): -// left as it was, never marked failed, and so no verdict on its build — `plans retry` asks it again. -const gateUnchanged = "unchanged" - -// unchangedBy says whether a send moving a module from one build to another changed nothing of it: the -// same commit, builds made from the same source, or builds that made the same artifacts from the same -// manifest. Not known is changed. -func unchangedBy(ctx context.Context, inv *inventory.Inventory, module, from, to string) bool { - if from == "" || to == "" { - return false - } - if sameCommit(from, to) { - return true - } - // Made from the same source (issue 280), or the same artifacts from the same manifest. - f := moveFacts{} - var err error - if f.srcs, err = inv.SourceFingerprints(ctx); err != nil { - return false - } - if f.fps, err = inv.Fingerprints(ctx); err != nil { - return false - } - return f.identical(module, from, to) -} - -// rolledBackEvent is the body of `rolled-back` (ADR 0236): a contract, like a condition's events. -type rolledBackEvent struct { - Event string `json:"event"` - At time.Time `json:"at"` - Module string `json:"module"` - Component string `json:"component,omitempty"` - Machines []string `json:"machines"` - From string `json:"from,omitempty"` - To string `json:"to,omitempty"` - Why string `json:"why"` - // Rollback is rolled-back, or not-rolled-back with NotWhy. - Rollback string `json:"rollback"` - NotWhy string `json:"not_why,omitempty"` - Show string `json:"show"` -} - -// sayRollback raises the gate's condition at once — the probe keeps it from then — and says the event. -func sayRollback(ctx context.Context, open *stores, module string, g *inventory.PlanGate, notWhy string) { - o := gateObservation(module, g.Component, g.Machines, g.Rollback, g.Why, notWhy, g.To, g.From) - fmt.Println(o.Summary) - d := doctorFrom - if d == nil { - return - } - if d.keeper != nil { - o.Source = gateProbe - if _, err := d.keeper.Observe(ctx, o); err != nil { - fmt.Printf("the gate's condition %s could not be kept: %v\n", o.Key(), err) - } - } - if d.teller == nil { - return - } - body, err := json.Marshal(rolledBackEvent{Event: link.KeyRolledBack, At: time.Now().UTC(), Module: module, - Component: g.Component, Machines: g.Machines, From: g.To, To: g.From, Why: g.Why, Rollback: g.Rollback, - NotWhy: notWhy, Show: conditions.Condition{Key: o.Key()}.Show()}) - if err != nil { - return - } - saying, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - if err := d.teller.PublishSeatEvent(saying, conditions.Seat, link.KeyRolledBack, body); err != nil { - fmt.Printf("%s's rollback could NOT be said on the bus: %v\n", module, err) - } -} - -// gateObservation is a failed gate as a condition: `core...rolled-back` for the -// core (to-be 45 §2), `build...rolled-back` for any other module; `rollback-failed` -// when it could not be put back. Urgent for the core and for a build left in place; a warning for a -// module put back, which runs what it ran before. -func gateObservation(module, component string, machines []string, rollback, why, notWhy, failed, previous string) conditions.Observation { - machine := strings.Join(machines, ",") - o := conditions.Observation{Scope: conditions.ScopeBuild, ID: module + "." + machine, Kind: kindRolledBack, - Token: kindRolledBack, Machine: firstOf(machines), Severity: conditions.Warning, Source: gateProbe, - Summary: fmt.Sprintf("%s's build %s failed its gate on %s and was put back to %s: %s", module, short(failed), - machine, short(previous), why)} - if component != "" { - o.Scope, o.ID, o.Severity = conditions.ScopeCore, component+"."+machine, conditions.Urgent - } - if len(machines) > 1 { - o.Also = machines[1:] - } - if rollback != inventory.RolledBack && rollback != inventory.RollingBack { - o.Kind, o.Token, o.Severity = kindRollbackFailed, kindRollbackFailed, conditions.Urgent - o.Resolver = conditions.ResolverOperator - o.Summary = fmt.Sprintf("%s's build %s failed its gate on %s and was NOT put back: %s — %s", module, short(failed), - machine, why, orNone(notWhy)) - } - return o -} - -// probeGates is DG: no build that failed its gate is left without its condition, and no witness's -// rollback goes unsaid. Each module whose newest verdict is a failure keeps its condition; a newer build -// that passes clears it. Each core component a machine's witness says it put back is `core.. -// .rolled-back`, urgent, until the machine's reports stop saying it. -func probeGates(ctx context.Context, d *doctor) ([]conditions.Observation, error) { - latest, err := d.open.inventory.LatestGates(ctx) - if err != nil { - return nil, err - } - var out []conditions.Observation - for _, v := range latest { - if v.Verdict != inventory.GateFailed { - continue - } - notWhy := "" - if v.Rollback == inventory.NotRolledBack { - notWhy = v.Why - } - out = append(out, gateObservation(v.Module, v.Component, v.Machines, v.Rollback, v.Why, notWhy, v.Commit, v.Previous)) - } - for node, list := range witnessed.all() { - for _, r := range list { - out = append(out, witnessObservation(node, r)) - } - } - // A release held after a failed one, while builds still wait for a gate (ADR 0236). - out = append(out, backlogObservation()...) - // And a rest that went silent (novox/hq ADR 0353 §4). - silent, err := silentObservation(ctx, d.open.inventory) - if err != nil { - return nil, err - } - out = append(out, silent...) - return sortedFound(dedupeObservations(out)), nil -} - -// dedupeObservations keeps one observation per key, the first. -func dedupeObservations(list []conditions.Observation) []conditions.Observation { - seen := map[string]bool{} - var out []conditions.Observation - for _, o := range list { - if seen[o.Key()] { - continue - } - seen[o.Key()] = true - out = append(out, o) - } - return out -} - -// servedOnTheBus asks the bus's discovery what every machine's node tools answer and which modules' -// tools are served where. A variable so a test needs no runtime. -var servedOnTheBus = func(ctx context.Context, conn *nats.Conn) (map[string]served, error) { - inbox := conn.NewRespInbox() - sub, err := conn.SubscribeSync(inbox) - if err != nil { - return nil, err - } - defer func() { _ = sub.Unsubscribe() }() - if err := conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil { - return nil, fmt.Errorf("asking the bus who serves what: %w", err) - } - out := map[string]served{} - add := func(node string) served { - s, ok := out[node] - if !ok { - s = served{tools: map[string]bool{}} - } - return s - } - deadline := time.Now().Add(discoveryPatience) - for time.Now().Before(deadline) { - wait, cancel := context.WithTimeout(ctx, discoveryQuiet) - msg, err := sub.NextMsgWithContext(wait) - cancel() - if err != nil { - if ctx.Err() != nil { - return nil, ctx.Err() - } - break - } - var info micro.Info - if json.Unmarshal(msg.Data, &info) != nil { - continue - } - if info.Name == broker.RuntimeModule { - node := info.Metadata["node"] - if node == "" { - node = info.ID - } - s := add(node) - s.runtime = true - out[node] = s - } - for _, e := range info.Endpoints { - if e.Metadata["kind"] != "tool" || e.Metadata["module"] == "" { - continue - } - node := e.Metadata["node"] - if node == "" { - node = info.ID - } - s := add(node) - s.tools[e.Metadata["module"]] = true - out[node] = s - } - } - return out, nil -} - -// gateLines is a plan's gates as `plans ` says them: the rollout's record. -func gateLine(g *inventory.PlanGate) string { - if g == nil { - return "" - } - what := "judging" - switch g.Verdict { - case inventory.GatePassed: - what = "passed" - case inventory.GateFailed: - what = "FAILED" - } - switch { - case g.Verdict == "" && g.Pending: - what = "kept, not yet sent" // novox/hq ADR 0353 §8 - case g.Verdict == "" && g.Silent: - what = "silent on its send, left" // ADR 0353 §4 - } - if g.Verdict == "" && g.Waiting() { - what = "waits for a busy container on " + g.Busy.Machine - } - line := fmt.Sprintf("gate on %s: %s", strings.Join(g.Machines, ", "), what) - if g.Component != "" { - line += " (core: " + g.Component + ")" - } - if g.From != "" || g.To != "" { - line += fmt.Sprintf(", %s → %s", short(orNone(g.From)), short(g.To)) - } - if g.Took != "" { - line += ", after " + g.Took - } - if g.Why != "" { - line += ": " + g.Why - } else if g.Last != "" { - line += fmt.Sprintf(" (%d of %d passes; wanting: %s)", g.Passes, gateWanted(g), g.Last) - } else if g.Verdict == "" { - line += fmt.Sprintf(" (%d of %d passes)", g.Passes, gateWanted(g)) - } - if g.Verdict == "" && len(g.Waits) > 0 { - line += waitsSaid(g.Waits) - } - if len(g.Passing) > 0 { - line += "; passed on their own and kept: " + strings.Join(g.Passing, ", ") - } - if g.Rollback != "" { - line += "; " + g.Rollback - } - // Before the send it judges (novox/hq issue 490). - if said := grantsSaid(g.Grants); said != "" { - line += "; " + said - } - return line -} - -// witnessObservation is a witness's verdict as a condition (ADR 0236, the host's contract): -// `core...`, urgent for rolled-back, not-reversible, restore-failed and -// halted, a warning for nothing-to-restore and unwitnessed. -func witnessObservation(node string, r lease.Rollback) conditions.Observation { - severity := conditions.Warning - if lease.Urgent(r.Outcome) { - severity = conditions.Urgent - } - outcome := r.Outcome - if outcome == "" { - outcome = lease.OutcomeRolledBack - } - what := map[string]string{ - lease.OutcomeRolledBack: "and put back " + short(orNone(r.To)), - lease.OutcomeNotReversible: "and left it: it is not reversible", - lease.OutcomeNothingToRestore: "and had nothing to put back", - lease.OutcomeRestoreFailed: "and could not put the previous build back", - lease.OutcomeUnwitnessed: "and could not judge it at all", - lease.OutcomeHalted: "and gave up: the build before it fails too", - }[outcome] - return conditions.Observation{Scope: conditions.ScopeCore, ID: r.Component + "." + node, Kind: kindRolledBack, - Token: outcome, Machine: node, Severity: severity, - Summary: fmt.Sprintf("the witness on %s judged the %s %s not healthy %s at %s: %s", node, r.Component, - short(r.From), what, r.At.UTC().Format(time.RFC3339), r.Why)} -} - -// gateWanted is the passes a gate's send needs, three where the gate was judged before it was said. -func gateWanted(g *inventory.PlanGate) int { - if g.Wants > 0 { - return g.Wants - } - return gatePasses -} diff --git a/cmd/mesh-controller/gate_followup_test.go b/cmd/mesh-controller/gate_followup_test.go deleted file mode 100644 index ca5dbb9f..00000000 --- a/cmd/mesh-controller/gate_followup_test.go +++ /dev/null @@ -1,468 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "reflect" - "slices" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" - "github.com/novox/mesh-controller/internal/link" -) - -// What the review of issue 318's fix found (novox/hq ADR 0254): every module of a send leaves it with a -// verdict, a pass is counted no faster than the gate's spacing, a carried build's verdict carries only its own -// wait, and a provider waiting for a login says who waits on it. - -// withGateBounds sets the gate's bounds for one test. -func withGateBounds(t *testing.T, within, every, bound time.Duration) { - t.Helper() - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - gateWithin, gateEvery, gateBound = within, every, bound - t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) -} - -// factsOn is a judging's facts for one machine that applied its send: the node tools answer, and what it says -// of its modules' resources. -func factsOn(t *testing.T, machine string, since time.Time, rs ...inventory.ResourceHealth) func() gateFacts { - return func() gateFacts { - now := time.Now() - at := now - return gateFacts{now: now, - reports: map[string]inventory.Reported{machine: {Node: machine, Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, - engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, - served: map[string]served{machine: {runtime: true, tools: map[string]bool{}}}, - health: map[string]inventory.NodeHealth{machine: {Node: machine, HeardAt: now, Resources: rs}}, - } - } -} - -func healthyContainer(module string) inventory.ResourceHealth { - return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module, - State: link.StateHealthy} -} - -// **A fault decided before the settle time does not strand the module beside it** (review (a)): the node -// tools' witness put its build back at once, and the send waits for the healthy module's own verdict — a pass, -// counted over the gate's spacing — before it says its own. The broken one alone is put back. -func TestAFaultBeforeTheSettleTimeWaitsForTheModuleBesideIt(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - open := aMesh(t) - ctx := t.Context() - withGateBounds(t, time.Hour, 20*time.Millisecond, 10*time.Second) - since := time.Now().Add(-time.Second) - base := factsOn(t, "laptop", since, healthyContainer("app")) - wasGather := gatherGateFacts - t.Cleanup(func() { gatherGateFacts = wasGather }) - gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { - f := base() - f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack, - At: since.Add(100 * time.Millisecond), Why: "the node tools did not answer"}} - return f, nil - } - g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since} - pairs := []judged{{module: broker_runtime, node: "laptop"}, {module: "app", node: "laptop"}} - judgings := 0 - for deadline := time.Now().Add(5 * time.Second); g.Verdict == "" && time.Now().Before(deadline); { - if _, err := judgeMoves(ctx, open, g, pairs, time.Now()); err != nil { - t.Fatal(err) - } - judgings++ - time.Sleep(30 * time.Millisecond) - } - if g.Verdict != inventory.GateFailed || judgings < gatePasses { - t.Fatalf("verdict %q after %d judging(s): %+v; want failed, after the module beside it was judged", g.Verdict, - judgings, g) - } - if !reflect.DeepEqual(g.Passing, []string{"app"}) || !reflect.DeepEqual(g.Failing, []string{broker_runtime}) { - t.Fatalf("passing %v, failing %v; want app kept and the node tools put back", g.Passing, g.Failing) - } - if !strings.Contains(g.Why, "witness") { - t.Errorf("the verdict does not say what broke: %q", g.Why) - } -} - -// broker_runtime is the node tools' module name, a core component a witness judges. -const broker_runtime = "node-tools" - -// **A pass is counted only the gate's spacing after the one before** (review (c)): a send judged every tick -// while a module beside it is not yet healthy counts the healthy one once. -func TestAPassIsCountedNoFasterThanTheGatesSpacing(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - withGateBounds(t, time.Hour, time.Hour, time.Hour) - since := time.Now().Add(-time.Minute) - base := factsOn(t, "laptop", since, healthyContainer("app"), inventory.ResourceHealth{Module: "late", - Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, Reason: "down"}) - wasGather := gatherGateFacts - t.Cleanup(func() { gatherGateFacts = wasGather }) - gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return base(), nil } - g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since} - pairs := []judged{{module: "app", node: "laptop"}, {module: "late", node: "laptop"}} - now := time.Now() - for i := 0; i < 3; i++ { - if _, err := judgeMoves(ctx, open, g, pairs, now.Add(time.Duration(i)*time.Second)); err != nil { - t.Fatal(err) - } - } - if g.Healthy["app"] != 1 || g.Healthy["late"] != 0 { - t.Fatalf("counted %v in three judgings within a second; want app once and late never", g.Healthy) - } -} - -// **A carried build's pass carries its own wait, never another's** (review (e)). -func TestACarriedPassCarriesOnlyItsOwnWait(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - since := time.Now().Add(-time.Minute) - g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &since, Verdict: inventory.GatePassed, - Why: "healthy 3 times over 2m0s" + waitsSaid(map[string]string{"late": "relogin needed on laptop: late waits"}), - Waits: map[string]string{"late": "relogin needed on laptop: late waits"}, - Carried: []inventory.CarriedMove{{Module: "app", Node: "laptop", From: "c1", To: "c2", Build: "build-app-c2"}, - {Module: "late", Node: "laptop", From: "c1", To: "c2", Build: "build-late-c2"}}} - passCarried(ctx, b.open, &inventory.Plan{ID: "release-test"}, g, "") - app, _, _ := inv.GateOf(ctx, "build-app-c2") - late, _, _ := inv.GateOf(ctx, "build-late-c2") - if strings.Contains(app.Why, "waits for a person") || app.Verdict != inventory.GatePassed { - t.Errorf("app's pass: %+v; want it without late's wait", app) - } - if !strings.Contains(late.Why, "relogin needed on laptop") { - t.Errorf("late's pass: %+v; want its own wait", late) - } -} - -// **The tier path keeps the pass of the module the gate is kept on** (review (b)): a plan's first send -// carried its own module, healthy, and a build waiting on that machine that never became healthy. At the -// bound the waiting one is put back and marked; the plan's own module keeps its pass, so no walk waits on it. -func TestThePlansOwnModuleKeepsItsPassWhenItsSendFails(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - g := aGateMesh(t) - ctx := t.Context() - inv := g.open.inventory - // `late` waits on anchor for a gate: c1 sent, c2 registered and built. - for _, b := range []inventory.Build{ - {ID: "build-late-1", Module: "late", Commit: "l1", Repository: "novox/mesh-catalog", Path: "modules/late", - Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)}, - {ID: "build-late-2", Module: "late", Commit: "l2", Repository: "novox/mesh-catalog", Path: "modules/late", - Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)}, - } { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "late", Version: b.Commit}) - b.Manifest = manifest - b.Made = []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + b.Commit}} - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "late", Version: b.Commit}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/late", BuiltFrom: b.Commit, Head: b.Commit, - Asked: b.Asked}); err != nil { - t.Fatal(err) - } - if b.Commit == "l1" { - if _, err := inv.Assign(ctx, "anchor", "late"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-late", map[string]string{"app": "c1", - "late": "l1"}); err != nil { - t.Fatal(err) - } - } - } - wasMoves := machineMoves - t.Cleanup(func() { machineMoves = wasMoves }) - machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { - modules, err := open.inventory.Assigned(ctx, node) - if err != nil { - return nil, err - } - sent, known, err := open.inventory.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - return f.moves(node, modules, sent, known, all), nil - } - gather := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f, err := gather(ctx, open, component) - f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", HeardAt: time.Now(), Resources: []inventory.ResourceHealth{ - healthyContainer("app"), {Module: "late", Resource: "late.web", Kind: "container", Target: "late", - State: link.StateUnhealthy, Reason: "down"}}}} - return f, err - } - gateEvery, gateBound = 0, 400*time.Millisecond - for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); { - advancePlans(ctx, g.open) - if p := g.plan(t); p.State == inventory.PlanFailed || p.State == inventory.PlanDone { - break - } - time.Sleep(30 * time.Millisecond) - } - p := g.plan(t) - if p.State != inventory.PlanFailed { - t.Fatalf("the plan is %s: %s; want it failed on late", p.State, p.Note) - } - if v, found, err := inv.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed || - !strings.Contains(v.Why, "on its own") { - t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err) - } - if failed, _ := inv.GateFailed(ctx, "build-late-2"); !failed { - t.Fatal("late's build is not marked failed at its gate") - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" || current["late"].Commit != "l1" { - t.Fatalf("registered app %s, late %s; want app kept at c2 and late put back to l1", current["app"].Commit, - current["late"].Commit) - } -} - -// **A provider waiting for a login says who waits on it** (review (g)): its consumer, failing a check that -// needs it, is held under it, and the provider's relogin-needed condition lists it and is urgent. -func TestAProviderWaitingForALoginSaysWhoWaitsOnIt(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - k := conditionsFrom - register(t, open, catalogue.Manifest{Module: "db", Version: "1", - Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}) - register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}}) - for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} { - if _, err := inv.Assign(ctx, a[0], a[1]); err != nil { - t.Fatal(err) - } - } - if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop", - Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil { - t.Fatal(err) - } - at := h0 - say := func(machine string, rs ...link.ResourceHealth) { - t.Helper() - at = at.Add(time.Second) - for i := range rs { - rs[i].Since = at - } - if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil { - t.Fatal(err) - } - } - account := link.ResourceHealth{Module: "db", Resource: "db.operator", Kind: link.KindAccount, Target: "operator", - Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group db"} - unit := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: link.KindUnit, Target: "db.service", - Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"} - shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop", - State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"} - for look := 0; look < 2; look++ { - say("anchor", account, unit) - say("laptop", shop) - } - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - var keys []string - var c conditions.Condition - for _, x := range list { - keys = append(keys, x.Key) - if x.Key == "module.db.anchor.relogin-needed" { - c = x - } - } - if !reflect.DeepEqual(keys, []string{"module.db.anchor.relogin-needed"}) { - t.Fatalf("open %v; want the provider's wait alone, its consumer held under it", keys) - } - if c.Severity != conditions.Urgent || !strings.Contains(c.Evidence[0].Said, "shop on laptop") { - t.Fatalf("the provider's wait: %s, %q; want it urgent and naming its consumer", c.Severity, c.Evidence[0].Said) - } -} - -// **A broken module is put back at once** (issue 318 review): the laptop's witness put the node tools back -// within a minute of a send that also moved `app`, and `app` reads not yet healthy because of it. The node -// tools are marked and put back in the very judging that found them broken — their registered build is the -// one before, and the laptop is sent it — while `app` goes on being judged, recovers, and keeps its own pass. -func TestABrokenModuleIsPutBackAtOnceAndTheOneBesideItGetsItsOwnVerdict(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - withConditionsInMemory(t) - was := doctorFrom - doctorFrom = nil - t.Cleanup(func() { doctorFrom = was }) - old, recent := time.Now().Add(-3*time.Hour), time.Now().Add(-time.Minute) - build := func(module, commit string, asked time.Time) { - manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit, - Repository: "novox/mesh-catalog", Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + commit}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit, - Asked: asked}); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"app", broker_runtime} { - build(m, "c1", old) - if _, err := inv.Assign(ctx, "laptop", m); err != nil { - t.Fatal(err) - } - } - if err := inv.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop", map[string]string{"app": "c1", broker_runtime: "c1"}); err != nil { - t.Fatal(err) - } - build("app", "c2", recent) - build(broker_runtime, "c2", recent) - - wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts - t.Cleanup(func() { - machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather - }) - machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { - modules, _ := open.inventory.Assigned(ctx, node) - sent, known, err := open.inventory.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - return f.moves(node, modules, sent, known, all), nil - } - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return map[string]bool{"laptop": true}, nil } - var sends []string - n := 0 - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - n++ - carried := map[string]string{"app": current["app"].Commit, broker_runtime: current[broker_runtime].Commit} - sends = append(sends, node+":"+carried[broker_runtime]) - digest := "d-" + node + "-" + time.Now().Format("150405.000000") + string(rune('a'+n)) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { - return nil, err - } - if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { - return nil, err - } - } - return names, nil - } - var seen []string // the node tools' registered build at each judging - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - now := time.Now() - f := gateFacts{now: now, reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} - reports, _ := open.inventory.LastReports(ctx) - for _, r := range reports { - f.reports[r.Node] = r - } - current, _ := open.inventory.CurrentBuilds(ctx) - seen = append(seen, current[broker_runtime].Commit) - app := healthyContainer("app") - if current[broker_runtime].Commit == "c2" { - // The witness reverted the node tools; app cannot reach them yet. - f.rolledBack["laptop"] = []lease.Rollback{{Component: lease.ComponentNodeTools, Outcome: lease.OutcomeRolledBack, - From: "c2", To: "c1", At: now, Why: "the node tools did not answer"}} - app.State, app.Reason = link.StateUnhealthy, "down" - } - f.served["laptop"] = served{runtime: current[broker_runtime].Commit == "c1", tools: map[string]bool{}} - f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, Resources: []inventory.ResourceHealth{app}}} - return f, nil - } - withGateBounds(t, time.Hour, 0, 10*time.Second) - - release := func() inventory.Plan { - t.Helper() - plans, _ := inv.RecentPlans(ctx, 10) - for _, p := range plans { - if p.Release != nil { - return p - } - } - t.Fatal("no walk") - return inventory.Plan{} - } - // Judged until the first judging has found the node tools broken, and one more. - for i := 0; i < 20 && len(seen) < 2; i++ { - advancePlans(ctx, open) - } - if len(seen) < 2 || seen[0] != "c2" || seen[1] != "c1" { - t.Fatalf("the node tools' registered build at each judging: %v; want c2, then c1 at the very next judging", seen) - } - if failed, _ := inv.GateFailed(ctx, "build-"+broker_runtime+"-c2"); !failed { - t.Fatal("the node tools' build is not marked failed at once") - } - if p := release(); p.State != inventory.PlanRolling || p.Release.Gate == nil || p.Release.Gate.Verdict != "" { - t.Fatalf("the walk is %s with gate %+v; want it still judging app", p.State, p.Release.Gate) - } - if !slices.Contains(sends, "laptop:c1") { - t.Fatalf("sends %v; want the laptop sent the node tools' earlier build at once", sends) - } - for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); { - advancePlans(ctx, open) - if release().State != inventory.PlanRolling { - break - } - time.Sleep(20 * time.Millisecond) - } - p := release() - if p.State != inventory.PlanFailed { - t.Fatalf("the walk is %s: %s; want failed, for the node tools", p.State, p.Note) - } - if v, found, _ := inv.GateOf(ctx, "build-app-c2"); !found || v.Verdict != inventory.GatePassed || !strings.Contains(v.Why, "on its own") { - t.Fatalf("app's verdict: %+v; want its own pass", v) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { - t.Fatalf("app is registered at %s; want it kept at c2", current["app"].Commit) - } -} - -// **A move from a build not known excuses no wait** (issue 318 review): a plan's own module whose previous -// build was not recorded, or whose previous manifest is not kept, cannot be shown to have added the group. -func TestAMoveFromAnUnknownBuildExcusesNoWait(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - record := func(commit string, m catalogue.Manifest) { - raw, _ := json.Marshal(m) - at := time.Now().Add(-time.Hour) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-lights-" + commit, Module: "lights", Commit: commit, - Repository: "novox/mesh-catalog", Path: "modules/lights", Manifest: raw, Asked: at, At: at}); err != nil { - t.Fatal(err) - } - } - record("c1", catalogue.Manifest{Module: "lights"}) - record("c2", catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user", - "name": "operator", "groups": []any{"lights"}}}}) - pairs := []judged{{module: "lights", node: "laptop"}} - shelf := map[string]catalogue.Manifest{} - for _, c := range []struct { - from string - want bool - }{{"c1", true}, {"", false}, {"c0-never-built", false}} { - g := &inventory.PlanGate{From: c.from, To: "c2"} - if got := movesAddingGroups(ctx, inv, g, pairs, shelf)["lights"]; got != c.want { - t.Errorf("a move from %q adds a group: %v; want %v", c.from, got, c.want) - } - } -} diff --git a/cmd/mesh-controller/gate_integration_test.go b/cmd/mesh-controller/gate_integration_test.go deleted file mode 100644 index f0357293..00000000 --- a/cmd/mesh-controller/gate_integration_test.go +++ /dev/null @@ -1,486 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "reflect" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" - "github.com/novox/mesh-controller/internal/link" -) - -func aGateMesh(t *testing.T) *gateMesh { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - g := &gateMesh{open: open, health: map[string]health{}} - // The gate's own tests judge a first machine, as walks a controller before novox/hq ADR 0360 started still are. - wasGated := gateFirstMachine - gateFirstMachine = true - t.Cleanup(func() { gateFirstMachine = wasGated }) - g.keeper, _ = withConditionsInMemory(t) - g.told = &conditions.Told{} - was := doctorFrom - doctorFrom = &doctor{open: open, keeper: g.keeper, teller: g.told} - t.Cleanup(func() { doctorFrom = was }) - - for _, b := range []inventory.Build{ - {ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app", - Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)}, - {ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app", - Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)}, - } { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit}) - b.Manifest = manifest - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - register := func(commit string, asked time.Time) { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit, - Head: commit, Asked: asked}); err != nil { - t.Fatal(err) - } - } - register("c1", time.Now().Add(-2*time.Hour)) - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, "app"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - } - register("c2", time.Now().Add(-time.Minute)) - - // Every send: recorded with the build the module is at, applied and reported by the machine. - n := 0 - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - g.sent = append(g.sent, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil { - return nil, err - } - if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { - return nil, err - } - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - - // What a judging reads: the store's reports, and a health the test says per machine. - wasGather := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return f, err - } - for _, r := range reports { - if g.health[r.Node] == healthBroken { - r.Outcome = inventory.OutcomeFailed - } - f.reports[r.Node] = r - } - for node, h := range g.health { - if h == healthNotYet { - // Not reported on the send: neither the send made last, nor the gate's own (issue 352). - f.rolledBack[node] = nil - r := f.reports[node] - r.Current, r.Declared, r.ReportedSequence = false, "", 0 - f.reports[node] = r - } - } - return f, nil - } - t.Cleanup(func() { gatherGateFacts = wasGather }) - - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. - gateWithin, gateEvery = time.Hour, time.Hour - t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) - - built := time.Now().UTC() - plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", - Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", - Build: "build-2"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - return g -} - -func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) { - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - inv := g.open.inventory - - advancePlans(ctx, g.open) // the first machine is sent the new build - if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v, not the first machine alone", g.sent) - } - if p := g.plan(t); p.Modules["app"].Previous != "c1" { - t.Fatalf("the build the first machine ran before was not kept: %+v", p.Modules["app"]) - } - - g.health["anchor"] = healthBroken // the new build breaks its first machine, after one good judging - gateEvery = 0 - advancePlans(ctx, g.open) - - p := g.plan(t) - gate := p.Modules["app"].Gate - if p.State != inventory.PlanFailed || gate == nil || gate.Verdict != inventory.GateFailed || - gate.Rollback != inventory.RolledBack { - t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate) - } - if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"anchor"}}) { - t.Fatalf("sent %v: the rollback goes to the first machine, and nothing reaches laptop", g.sent) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { - t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit) - } - if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c1" { - t.Fatalf("anchor was last sent %v, not the previous build", sent) - } - if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["app"] != "c1" { - t.Fatalf("laptop was sent the failed build: %v", sent) - } - if failed, err := inv.GateFailed(ctx, "build-2"); err != nil || !failed { - t.Fatalf("the build is not marked failed at its gate: %v %v", failed, err) - } - - // Said: a condition for the operator, and the event. - open, err := g.keeper.Open(ctx) - if err != nil { - t.Fatal(err) - } - var key string - for _, c := range open { - if c.Kind == kindRolledBack { - key = c.Key - } - } - if key != "build.app.anchor.rolled-back" { - t.Fatalf("no rolled-back condition for app on anchor: %+v", open) - } - saidIt := false - for _, e := range g.told.Said() { - saidIt = saidIt || e.Event == link.KeyRolledBack - } - if !saidIt { - t.Fatalf("the rollback was not said as an event: %+v", g.told.Said()) - } - - // Never again: more passes send nothing, a second judging rolls nothing back, and the failed build - // heard again is not registered. - advancePlans(ctx, g.open) - state := p.Modules["app"] - gateFailed(ctx, g.open, &p, "app", state, []string{"anchor"}, "again") - if len(g.sent) != 2 { - t.Fatalf("sent again after the rollback: %v", g.sent) - } - _, _, err = takeIn(ctx, inv, link.BuildResult{ID: "build-2", Repository: "novox/mesh-catalog", Path: "modules/app", - Commit: "c2", Manifest: mustJSON(t, catalogue.Manifest{Module: "app", Version: "c2"}), - Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}) - if err == nil || !strings.Contains(err.Error(), "failed its gate") { - t.Fatalf("the failed build was registered again: %v", err) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { - t.Fatalf("the module moved to %s", current["app"].Commit) - } - if _, err := retryPlan(ctx, g.open, "plan-gate"); err == nil || !strings.Contains(err.Error(), "failed its gate") { - t.Fatalf("a plan stopped at a failed gate was retried: %v", err) - } - - // The probe keeps the condition while the newest verdict is the failure. - obs, err := probeGates(ctx, doctorFrom) - if err != nil || len(obs) != 1 || obs[0].Key() != key { - t.Fatalf("the gate's probe found %+v %v", obs, err) - } -} - -func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) { - g := aGateMesh(t) - ctx := t.Context() - gateEvery = 0 - for i := 0; i < 6; i++ { - advancePlans(ctx, g.open) - } - p := g.plan(t) - if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("sent %v", g.sent) - } - gate := p.Modules["app"].Gate - if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gateWanted(gate) { - t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate) - } - if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("the verdict was not kept: %+v %v %v", v, found, err) - } - if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 { - t.Fatalf("a passing build left a condition: %+v %v", obs, err) - } -} - -func TestABuildRollsToEveryMachineAtOnceWithNoGate(t *testing.T) { - g := aGateMesh(t) - gateFirstMachine = false - ctx := t.Context() - judged := false - wasGather := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - judged = true - return wasGather(ctx, open, component) - } - t.Cleanup(func() { gatherGateFacts = wasGather }) - for i := 0; i < 6; i++ { - advancePlans(ctx, g.open) - } - p := g.plan(t) - if !reflect.DeepEqual(g.sent, [][]string{{"anchor", "laptop"}}) { - t.Fatalf("sent %v, not both machines in one send", g.sent) - } - if s := p.Modules["app"]; p.State != inventory.PlanDone || s.Gate != nil || s.FirstAt != nil || judged { - t.Fatalf("the plan is %s (%s), app %+v, judged %v", p.State, p.Note, s, judged) - } - if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 { - t.Fatalf("a rolled build left a condition: %+v %v", obs, err) - } -} - -func TestALeafModulePassesOnOneHealthyReading(t *testing.T) { - g := aGateMesh(t) - ctx := t.Context() - gateWithin = time.Hour // a core module could not pass on fewer readings for an hour - advancePlans(ctx, g.open) // the first machine is sent the new build - advancePlans(ctx, g.open) // one healthy judging - p := g.plan(t) - gate := p.Modules["app"].Gate - if gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes != 1 || gate.Wants != 1 { - t.Fatalf("a leaf module's gate after one healthy judging: %+v (plan %s: %s)", gate, p.State, p.Note) - } - if !strings.Contains(gate.Why, "healthy 1 times") { - t.Errorf("the verdict says %q", gate.Why) - } -} - -func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) { - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - advancePlans(ctx, g.open) - g.health["anchor"] = healthNotYet - gateEvery = 0 - advancePlans(ctx, g.open) - if p := g.plan(t); !p.Open() || len(g.sent) != 1 { - t.Fatalf("judged before the bound: %s %v", p.State, g.sent) - } - gateBound = -time.Second - advancePlans(ctx, g.open) - p := g.plan(t) - if gate := p.Modules["app"].Gate; p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed || - !strings.Contains(gate.Why, "not healthy within") || gate.Rollback != inventory.RolledBack { - t.Fatalf("the plan is %s, its gate %+v", p.State, gate) - } -} - -func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - bus := catalogue.Manifest{Module: "nats", Version: "2", Provides: []catalogue.Offer{{Name: "mesh-bus"}}, - Upgrade: &catalogue.UpgradePolicy{Policy: catalogue.PolicyRoll}} - if err := inv.RegisterModule(ctx, bus, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", - Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1"}); err != nil { - t.Fatal(err) - } - u, err := inv.UpgradeOf(ctx, "nats") - if err != nil || u.RollOut || u.From != catalogue.FromBus { - t.Fatalf("the bus's policy is %+v %v", u, err) - } - if err := inv.SetUpgradeOf(ctx, "nats", inventory.Upgrade{RollOut: true}); !errors.Is(err, inventory.ErrBusIsPlanned) { - t.Fatalf("a person rolled the bus out: %v", err) - } - var sent [][]string - was := sendRollout - sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) { - sent = append(sent, names) - return names, nil - } - t.Cleanup(func() { sendRollout = was }) - now := time.Now().UTC() - plan := inventory.Plan{ID: "plan-bus", Repository: "novox/mesh-catalog", Commit: "n2", Created: now, - State: inventory.PlanBuilding, Tiers: [][]string{{"nats"}}, - Modules: map[string]*inventory.PlanModule{"nats": {State: "built", BuiltAt: &now, Commit: "n2", Build: "b"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - advancePlans(ctx, open) - if p, _ := inv.PlanByID(ctx, "plan-bus"); p.State != inventory.PlanDone || len(sent) != 0 { - t.Fatalf("a plan sent the bus: %s %v", p.State, sent) - } - held, err := busHeld(ctx, inv, []string{"anchor", "laptop"}) - if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" { - t.Fatalf("a push may send the bus's machine: %v %v", held, err) - } - // Nor may any other send — a plan's for another module on that machine carried the bus with it. - if _, err := sendToEach(ctx, open, []string{"laptop", "anchor"}); !errors.Is(err, errBusWaits) { - t.Fatalf("a send to the bus's machine was not refused: %v", err) - } - // A rebuild that made the same artifacts is no move. - for _, b := range []inventory.Build{{ID: "nb1", Module: "nats", Commit: "n1"}, {ID: "nb2", Module: "nats", Commit: "n2"}} { - b.Made = []inventory.Artifact{{Name: "server", Kind: "image", Reference: "registry/nats@sha256:same"}} - b.Asked, b.At = time.Now(), time.Now() - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 { - t.Fatalf("a rebuild that changes nothing held the bus's machine: %v %v", held, err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "nb3", Module: "nats", Commit: "n2", Asked: time.Now().Add(time.Second), - At: time.Now().Add(time.Second), Made: []inventory.Artifact{{Name: "server", Kind: "image", - Reference: "registry/nats@sha256:new"}}}); err != nil { - t.Fatal(err) - } - // The planned step refuses to start without its word on reversibility, and without a snapshot taken - // first by the bus machine's backup holder. - if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") { - t.Fatalf("a bus upgrade started without saying whether it can be reverted: %v", err) - } - wasSnapshot := takeBusSnapshot - t.Cleanup(func() { takeBusSnapshot = wasSnapshot }) - takeBusSnapshot = func(context.Context, string, string) (string, error) { return "", errors.New("no holder answers") } - if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err == nil || - !strings.Contains(err.Error(), "snapshotted") || len(sent) != 0 { - t.Fatalf("a bus upgrade started without its snapshot: %v, sent %v", err, sent) - } - takeBusSnapshot = func(_ context.Context, module, node string) (string, error) { - return node + "'s restore point of " + module, nil - } - if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(sent, [][]string{{"anchor"}}) { - t.Fatalf("the step sent %v, not the bus's machine", sent) - } - s, found, err := inv.LatestBusStep(ctx) - if err != nil || !found || s.Snapshot != "anchor's restore point of nats" || s.Ended != nil || - !reflect.DeepEqual(s.Machines, []string{"anchor"}) { - t.Fatalf("the step is %+v %v %v", s, found, err) - } -} - -func TestAMergeThatDeletesAModulePlansNothingToBuildForIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := asksRecorded(t) - for _, name := range []string{"gone", "kept"} { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: name, Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + name, BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1deadbeef", - Paths: []string{"modules/gone/module.json", "modules/gone/index.ts"}, - Removed: []string{"modules/gone/module.json", "modules/gone/index.ts"}} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - if len(*asked) != 0 { - t.Fatalf("a deleted module was asked to build: %v", *asked) - } - if plans, _ := inv.OpenPlans(ctx); len(plans) != 0 { - t.Fatalf("a merge that only deleted a module made a plan: %+v", plans) - } - shelf, err := inv.Catalogue(ctx) - if err != nil { - t.Fatal(err) - } - if _, still := shelf["gone"]; still { - t.Fatal("a deleted module nothing holds was not forgotten") - } - - // Without the announcer saying what went: the build finds no manifest, and the plan goes on. - asked0 := time.Now().UTC().Add(-time.Minute) - plan := inventory.Plan{ID: "plan-deleted", Repository: "novox/mesh-catalog", Commit: "c2", Created: asked0, - State: inventory.PlanBuilding, Tiers: [][]string{{"kept"}}, - Modules: map[string]*inventory.PlanModule{"kept": {State: "asked", AskedAt: &asked0, Build: "build-k"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-k", Module: "kept", Repository: "novox/mesh-catalog", - At: time.Now().UTC(), Asked: asked0, Failed: "http://forge/novox/mesh-catalog.git has no module.json at modules/kept, so " + - "there is nothing saying what it is: open …/module.json: no such file or directory"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, open) - p, _ := inv.PlanByID(ctx, "plan-deleted") - if p.State == inventory.PlanFailed || p.Modules["kept"].State != planDeleted { - t.Fatalf("a module deleted at its source failed the plan: %s %+v", p.State, p.Modules["kept"]) - } -} - -func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) { - open := aMesh(t) - d := &doctor{open: open} - at := time.Now().UTC() - witnessed.heard("control", []lease.Rollback{ - {Component: lease.ComponentController, From: "sha256:new", To: "sha256:old", Outcome: lease.OutcomeRolledBack, - Why: "the new controller did not take the lease within 60s", At: at}, - {Component: lease.ComponentNodeTools, From: "sha256:t2", Outcome: lease.OutcomeUnwitnessed, Why: "no grant", At: at}, - }, at) - t.Cleanup(func() { witnessed.heard("control", nil, time.Now()) }) - obs, err := probeGates(t.Context(), d) - if err != nil { - t.Fatal(err) - } - got := map[string]conditions.Severity{} - for _, o := range obs { - got[o.Key()] = o.Severity - } - want := map[string]conditions.Severity{"core.controller.control.rolled-back": conditions.Urgent, - "core.node-tools.control.unwitnessed": conditions.Warning} - if !reflect.DeepEqual(got, want) { - t.Fatalf("the witness's verdicts are %v", got) - } - witnessed.heard("control", nil, time.Now()) - if obs, err := probeGates(t.Context(), d); err != nil || len(obs) != 0 { - t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err) - } -} diff --git a/cmd/mesh-controller/gate_rule_test.go b/cmd/mesh-controller/gate_rule_test.go deleted file mode 100644 index b1b1033e..00000000 --- a/cmd/mesh-controller/gate_rule_test.go +++ /dev/null @@ -1,347 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" -) - -// The first-node gate's readings by module (novox/hq ADR 0351, ADR 0352): the controller and the tool runner pass -// on one reading after the node-engine's witness window, mesh-delivery on one, the node-engine and the mesh's -// resolver on their third healthy reading and never later than two minutes while healthy; the bus is kept so. - -// withTheGatesTimes sets the gate's production bounds for a test, whatever another test left. -func withTheGatesTimes(t *testing.T) { - t.Helper() - wasWithin, wasEvery, wasBound, wasWindow := gateWithin, gateEvery, gateBound, gateWitnessWindow - wasPasses, wasLeafPasses, wasLeafSettles := gatePasses, gateLeafPasses, gateLeafSettles - gateWithin, gateEvery, gateBound, gateWitnessWindow = 2*time.Minute, 40*time.Second, 10*time.Minute, 60*time.Second - gatePasses, gateLeafPasses, gateLeafSettles = 3, 1, false - t.Cleanup(func() { - gateWithin, gateEvery, gateBound, gateWitnessWindow = wasWithin, wasEvery, wasBound, wasWindow - gatePasses, gateLeafPasses, gateLeafSettles = wasPasses, wasLeafPasses, wasLeafSettles - }) -} - -func TestEachModuleHasItsGateRule(t *testing.T) { - withTheGatesTimes(t) - for _, c := range []struct { - module string - claims []catalogue.Claim - passes int - within time.Duration - witnessed bool - said string - }{ - {"mesh-controller", nil, 1, 0, true, "the controller's rule"}, - {"node-tools", nil, 1, 0, true, "the tool runner's rule"}, - {catalogue.DeliverySeat, []catalogue.Claim{{Name: catalogue.DeliverySeat}}, 1, 0, false, "mesh-delivery's rule"}, - {"mesh-host", nil, 3, 2 * time.Minute, false, "the core's rule"}, - {"dnsmasq", []catalogue.Claim{{Name: resolverSeat}}, 3, 2 * time.Minute, false, "the core's rule"}, - {"nats", nil, 3, 2 * time.Minute, false, "the core's rule"}, - {"app", nil, 1, 0, false, "a leaf module's rule"}, - } { - r := gateRuleOf(c.module, catalogue.Manifest{Module: c.module, Claims: c.claims}) - if r.passes != c.passes || r.within != c.within || r.witnessed != c.witnessed || !strings.HasPrefix(r.said, c.said) { - t.Errorf("%s: %+v; want %d readings, within %s, witnessed %v, said %q", c.module, r, c.passes, c.within, - c.witnessed, c.said) - } - } -} - -// Every module that was core stays core for its delivery budget and its walk: only its readings change. -func TestTheCoreStaysTheCore(t *testing.T) { - for _, m := range []string{"mesh-controller", "mesh-host", "node-tools", "nats", catalogue.DeliverySeat} { - // The delivery's owner is core by the claim of the seat, not by its name (novox/hq ADR 0437 §3, issue 1210). - claims := []catalogue.Claim(nil) - if m == catalogue.DeliverySeat { - claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}} - } - if !coreModule(m, catalogue.Manifest{Module: m, Claims: claims}) { - t.Errorf("%s is no longer judged as the core for its class", m) - } - } - if !coreModule("dnsmasq", catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}) { - t.Error("the resolver's holder is no longer judged as the core for its class") - } -} - -// gateOn judges one module on one machine, healthy at every reading. -type gateOn struct { - module, node string - g *inventory.PlanGate - holder *lease.Holder -} - -func aGateOn(module, node string, since time.Time) *gateOn { - return &gateOn{module: module, node: node, g: &inventory.PlanGate{Machines: []string{node}, Since: &since, - Applied: map[string]time.Time{}}} -} - -func (o *gateOn) read(at time.Time, h health, why string) { - m := catalogue.Manifest{Module: o.module} - if o.module == catalogue.DeliverySeat { // today's module, which claims the seat - m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat}} - } - r := gateRuleOf(o.module, m) - floor := gateFloor(o.module, r, *o.g.Since, []string{o.node}, o.g.Applied, o.holder) - tallyGate(o.g, gateJudging{modules: []string{o.module}, reading: map[string]health{o.module: h}, - worst: h, why: why, rules: map[string]gateRule{o.module: r}, floors: map[string]time.Time{o.module: floor}}, at) -} - -func TestTheControllerPassesOnOneReadingAfterTheWitnessWindow(t *testing.T) { - withTheGatesTimes(t) - sent := time.Date(2026, 10, 13, 20, 0, 0, 0, time.UTC) - o := aGateOn("mesh-controller", "novox", sent) - o.g.Applied["novox"] = sent.Add(15 * time.Second) - o.holder = &lease.Holder{Instance: "new", Taken: sent.Add(20 * time.Second)} - - // Ready, but inside the witness window, which opened before it took the lease: not counted. - for _, after := range []time.Duration{30 * time.Second, 60 * time.Second, 79 * time.Second} { - o.read(sent.Add(after), healthGood, "") - if o.g.Verdict != "" || o.g.Passes != 0 || !strings.Contains(o.g.Last, "counts from") { - t.Fatalf("at %s: %+v; want no reading counted inside the witness window", after, o.g) - } - } - o.read(sent.Add(80*time.Second), healthGood, "") - if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || o.g.Wants != 1 { - t.Fatalf("after the witness window: %+v; want passed on one reading", o.g) - } - if !strings.Contains(o.g.Why, "healthy 1 times") || !strings.Contains(o.g.Why, "the controller's rule") || - !strings.Contains(o.g.Why, "ADR 0351") { - t.Errorf("the verdict says %q; want the rule it applied", o.g.Why) - } -} - -// A controller that starts again takes the lease again: its witness window starts again, and so does the -// gate's wait. -func TestAControllerThatStartsAgainWaitsOutItsNewWindow(t *testing.T) { - withTheGatesTimes(t) - sent := time.Now() - o := aGateOn("mesh-controller", "novox", sent) - o.g.Applied["novox"] = sent.Add(10 * time.Second) - o.holder = &lease.Holder{Instance: "again", Taken: sent.Add(70 * time.Second)} - o.read(sent.Add(90*time.Second), healthGood, "") - if o.g.Verdict != "" { - t.Fatalf("passed %s after a start the witness still judges: %+v", 20*time.Second, o.g) - } - o.read(sent.Add(130*time.Second), healthGood, "") - if o.g.Verdict != inventory.GatePassed { - t.Fatalf("not passed after the new window: %+v", o.g) - } -} - -// Without the lease's word (a controller carried by another gate), the window counts from the applied report, -// and from the send at the least. -func TestTheWitnessWindowCountsFromTheSendAtTheLeast(t *testing.T) { - withTheGatesTimes(t) - sent := time.Now() - r := gateRuleOf("mesh-controller", catalogue.Manifest{}) - if f := gateFloor("mesh-controller", r, sent, []string{"novox"}, nil, nil); !f.Equal(sent.Add(time.Minute)) { - t.Errorf("floor %s after the send; want 1m", f.Sub(sent)) - } - if f := gateFloor("node-tools", gateRuleOf("node-tools", catalogue.Manifest{}), sent, []string{"ace"}, - map[string]time.Time{"ace": sent.Add(12 * time.Second)}, nil); !f.Equal(sent.Add(72 * time.Second)) { - t.Errorf("the tool runner's floor %s after the send; want 1m12s", f.Sub(sent)) - } -} - -// A controller that a witness put back, or that never becomes ready, fails its gate as before. -func TestAControllerThatFailsIsStillFailed(t *testing.T) { - withTheGatesTimes(t) - sent := time.Now() - o := aGateOn("mesh-controller", "novox", sent) - o.read(sent.Add(30*time.Second), healthBroken, "the witness on novox judged the controller and rolled-back") - if o.g.Verdict != inventory.GateFailed { - t.Fatalf("a controller put back by its witness: %+v; want failed", o.g) - } - o = aGateOn("mesh-controller", "novox", sent) - o.read(sent.Add(5*time.Minute), healthNotYet, "the controller holding the lease does not say it is ready") - if o.g.Verdict != "" { - t.Fatalf("failed before the bound: %+v", o.g) - } - o.read(sent.Add(10*time.Minute+time.Second), healthNotYet, "the controller holding the lease does not say it is ready") - if o.g.Verdict != inventory.GateFailed || !strings.Contains(o.g.Why, "not healthy within") { - t.Fatalf("never ready: %+v; want failed at the bound", o.g) - } -} - -// threeReadings judges a module under the three-reading rule on one machine, the send at sent. -func threeReadings(t *testing.T, module string, sent time.Time) (*inventory.PlanGate, func(time.Duration, health)) { - t.Helper() - g := &inventory.PlanGate{Machines: []string{"ace"}, Since: &sent} - rule := gateRuleOf(module, catalogue.Manifest{Module: module, Claims: []catalogue.Claim{{Name: resolverSeat}}}) - floor := gateFloor(module, rule, sent, []string{"ace"}, nil, nil) - return g, func(after time.Duration, h health) { - why := "" - if h != healthGood { - why = "the node-engine reports another build" - } - tallyGate(g, gateJudging{modules: []string{module}, reading: map[string]health{module: h}, worst: h, why: why, - rules: map[string]gateRule{module: rule}, floors: map[string]time.Time{module: floor}}, sent.Add(after)) - } -} - -// A healthy node-engine or resolver passes at its third healthy reading, 40 s apart from the apply: before two -// minutes (novox/hq ADR 0352). -func TestTheNodeEngineAndTheResolverPassAtTheirThirdReading(t *testing.T) { - withTheGatesTimes(t) - for _, module := range []string{"mesh-host", "dnsmasq"} { - g, judge := threeReadings(t, module, time.Now()) - judge(15*time.Second, healthGood) - judge(55*time.Second, healthGood) - if g.Verdict != "" || g.Passes != 2 { - t.Fatalf("%s: after two readings %+v; want still judging", module, g) - } - judge(95*time.Second, healthGood) - if g.Verdict != inventory.GatePassed || g.Passes != 3 || g.Wants != 3 || - !strings.Contains(g.Why, "over 1m35s") || !strings.Contains(g.Why, "the core's rule: 3 readings") { - t.Fatalf("%s: after three readings %+v; want passed at 1m35s by the core's rule", module, g) - } - } -} - -// One healthy throughout whose readings came late passes at two minutes on the readings it has; the judging at -// two minutes comes sooner than the spacing after the last pass. -func TestAHealthyNodeEngineIsNeverHeldPastTwoMinutes(t *testing.T) { - withTheGatesTimes(t) - sent := time.Now() - last := sent.Add(100 * time.Second) - held := &inventory.PlanGate{Since: &sent, LastPass: &last} - if dueAtTheMost(held, sent.Add(119*time.Second)) || !dueAtTheMost(held, sent.Add(2*time.Minute)) { - t.Fatal("the judging at two minutes is not due, or is due before") - } - held.Faulted = true - if dueAtTheMost(held, sent.Add(2*time.Minute)) { - t.Fatal("a send that was not healthy at a reading is judged at two minutes") - } - - g, judge := threeReadings(t, "mesh-host", time.Now()) - judge(100*time.Second, healthGood) - judge(2*time.Minute, healthGood) - if g.Verdict != inventory.GatePassed || g.Passes != 2 { - t.Fatalf("late readings, healthy throughout, at two minutes: %+v; want passed on two", g) - } - g, judge = threeReadings(t, "mesh-host", time.Now()) - judge(2*time.Minute, healthGood) - if g.Verdict != inventory.GatePassed || g.Passes != 1 { - t.Fatalf("one healthy reading, at two minutes: %+v; want passed on it", g) - } -} - -// A reading not yet healthy before the first healthy one is the build still starting: the two minutes still hold. -func TestANodeEngineStillStartingKeepsItsTwoMinutes(t *testing.T) { - withTheGatesTimes(t) - g, judge := threeReadings(t, "mesh-host", time.Now()) - judge(10*time.Second, healthNotYet) - judge(100*time.Second, healthGood) - judge(2*time.Minute, healthGood) - if g.Verdict != inventory.GatePassed || g.Faulted { - t.Fatalf("%+v; want passed at two minutes", g) - } -} - -// A reading that is not healthy after a healthy one takes the two minutes away: the module needs its three readings, and one -// never healthy fails at the bound and is put back; one found broken fails. -func TestANodeEngineThatFailsAReadingIsNotPassedAtTwoMinutes(t *testing.T) { - withTheGatesTimes(t) - g, judge := threeReadings(t, "mesh-host", time.Now()) - judge(20*time.Second, healthNotYet) - judge(60*time.Second, healthGood) - judge(100*time.Second, healthGood) - judge(2*time.Minute+10*time.Second, healthGood) - if g.Verdict != inventory.GatePassed || g.Passes != 3 { - t.Fatalf("%+v; want passed only on its third healthy reading", g) - } - g, judge = threeReadings(t, "mesh-host", time.Now()) - judge(60*time.Second, healthGood) - judge(100*time.Second, healthNotYet) - judge(2*time.Minute, healthGood) - if g.Verdict != "" || !g.Faulted { - t.Fatalf("passed at two minutes after a reading that was not healthy: %+v", g) - } - judge(10*time.Minute+time.Second, healthNotYet) - if g.Verdict != inventory.GateFailed || !strings.Contains(g.Why, "not healthy within") { - t.Fatalf("never healthy again: %+v; want failed at the bound", g) - } - g, judge = threeReadings(t, "mesh-host", time.Now()) - judge(60*time.Second, healthGood) - judge(100*time.Second, healthBroken) - if g.Verdict != inventory.GateFailed { - t.Fatalf("a broken reading: %+v; want failed, to be put back", g) - } -} - -// The tool runner passes on one reading after its witness window; mesh-delivery on one, at once. -func TestTheToolRunnerAndMeshDeliveryPassOnOneReading(t *testing.T) { - withTheGatesTimes(t) - sent := time.Now() - o := aGateOn("node-tools", "ace", sent) - o.g.Applied["ace"] = sent.Add(10 * time.Second) - o.read(sent.Add(30*time.Second), healthGood, "") - if o.g.Verdict != "" { - t.Fatalf("the tool runner passed inside its witness window: %+v", o.g) - } - o.read(sent.Add(70*time.Second), healthGood, "") - if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "the tool runner's rule") { - t.Fatalf("the tool runner: %+v; want passed on one reading", o.g) - } - - o = aGateOn(catalogue.DeliverySeat, "novox", sent) - o.read(sent.Add(5*time.Second), healthGood, "") - if o.g.Verdict != inventory.GatePassed || o.g.Passes != 1 || !strings.Contains(o.g.Why, "mesh-delivery's rule") { - t.Fatalf("mesh-delivery: %+v; want passed on one reading", o.g) - } -} - -// A dip after a healthy reading taken before the controller's witness floor is a fault: the send is not passed at -// two minutes (novox/hq ADR 0352 decision 3). -func TestADipAfterAnUncountedHealthyReadingIsAFault(t *testing.T) { - withTheGatesTimes(t) - sent := time.Now() - g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent} - modules := []string{"mesh-controller", "mesh-host"} - rules, floors := map[string]gateRule{}, map[string]time.Time{} - for _, m := range modules { - rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m}) - } - floors["mesh-host"] = sent - floors["mesh-controller"] = sent.Add(90 * time.Second) - judge := func(after time.Duration, h health) { - tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": h, "mesh-host": h}, - worst: h, why: "not yet", rules: rules, floors: floors}, sent.Add(after)) - } - judge(30*time.Second, healthGood) - if g.Passes != 0 { - t.Fatalf("a reading before the controller's floor was counted: %+v", g) - } - judge(70*time.Second, healthNotYet) - judge(2*time.Minute, healthGood) - if !g.Faulted || g.Verdict != "" { - t.Fatalf("%+v; want the dip a fault, and no pass at two minutes", g) - } -} - -// A send carrying the controller and the node-engine is judged by the node-engine's three readings. -func TestASendIsJudgedByItsMostDemandingRule(t *testing.T) { - withTheGatesTimes(t) - sent := time.Now() - g := &inventory.PlanGate{Machines: []string{"novox"}, Since: &sent} - modules := []string{"mesh-controller", "mesh-host"} - rules, floors := map[string]gateRule{}, map[string]time.Time{} - for _, m := range modules { - rules[m] = gateRuleOf(m, catalogue.Manifest{Module: m}) - floors[m] = gateFloor(m, rules[m], sent, []string{"novox"}, nil, nil) - } - for _, after := range []time.Duration{70 * time.Second, 110 * time.Second} { - tallyGate(g, gateJudging{modules: modules, reading: map[string]health{"mesh-controller": healthGood, - "mesh-host": healthGood}, worst: healthGood, rules: rules, floors: floors}, sent.Add(after)) - } - if g.Verdict != "" || g.Wants != 3 || g.Passes != 2 || g.Healthy["mesh-controller"] != 2 { - t.Fatalf("%+v; want the send judged by the node-engine's three readings, the controller's own counted", g) - } -} diff --git a/cmd/mesh-controller/gate_test.go b/cmd/mesh-controller/gate_test.go deleted file mode 100644 index 8109ede3..00000000 --- a/cmd/mesh-controller/gate_test.go +++ /dev/null @@ -1,205 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" - "github.com/novox/mesh-controller/internal/link" -) - -// The gate on a plan's first machine and the rollback after it (novox/hq ADR 0236, to-be 45 §8). - -// gateMesh is a mesh with `app` running on anchor and laptop at build c1, a newer build c2 registered, -// a plan whose tier built c2, and every send recorded and answered — the machine applies what it is -// sent and reports it — with the gate's bounds shortened to judge in three steps. -type gateMesh struct { - open *stores - sent [][]string - health map[string]health // per machine, what a judging finds; healthy when unsaid - keeper *conditions.Keeper - told *conditions.Told -} - -func (g *gateMesh) plan(t *testing.T) inventory.Plan { - t.Helper() - p, err := g.open.inventory.PlanByID(t.Context(), "plan-gate") - if err != nil { - t.Fatal(err) - } - return p -} - -// A module's build that fails its gate on the first machine is put back there — the previous build -// registered and sent to it again — and never reaches the second machine; it is marked, said as a -// condition and an event, never registered or rolled back again. - -// A passing build rolls everywhere with no hand: judged healthy on its first machine three times, then -// the rest are sent, the verdict kept, and the plan done. - -// A build rolls everywhere with no hand and no gate (novox/hq ADR 0360 §1): both machines in one send, no health -// judged, and the walk done once both reported on the send. Before, anchor was judged first, then laptop sent. - -// A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the -// core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524). - -// A first machine that never becomes healthy fails its gate at the bound, and is put back. - -// The health a judging finds, per core component and for a module. -func TestTheHealthDefinitions(t *testing.T) { - now := time.Now() - since := now.Add(-time.Minute) - applied := func(node string) gateFacts { - at := now - return gateFacts{now: now, reports: map[string]inventory.Reported{node: {Node: node, - Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{}, - served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}} - } - m := catalogue.Manifest{Module: "app", Tools: []string{"app_list"}} - - f := applied("anchor") - f.served["anchor"] = served{runtime: true, tools: map[string]bool{}} - if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "tools") { - t.Errorf("a module whose tools are not served is %v (%s)", h, why) - } - f.served["anchor"].tools["app"] = true - if h, why := judgeHealth("app", "", m, "anchor", since, f); h != healthGood { - t.Errorf("a module applied with its tools served is %v (%s)", h, why) - } - // A condition raised about it on that machine since it was sent: not yet healthy. - f.judged = true - f.open = []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{ - Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Raised: now, Summary: "failing"}} - if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthNotYet { - t.Errorf("a module with a new condition about it is %v", h) - } - f.open[0].Raised = since.Add(-time.Hour) - if h, _ := judgeHealth("app", "", m, "anchor", since, f); h != healthGood { - t.Errorf("a condition older than the send counted against it: %v", h) - } - // A witness that put it back: broken. - f.rolledBack["anchor"] = []lease.Rollback{{Component: lease.ComponentNodeTools, From: "sha256:aa", To: "sha256:bb", - Outcome: lease.OutcomeRolledBack, Why: "x", At: now}} - if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthBroken { - t.Errorf("a component a witness put back is %v", h) - } - // The node tools answer, or not. - f = applied("anchor") - if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthNotYet { - t.Errorf("node tools not answering are %v", h) - } - f.served["anchor"] = served{runtime: true} - if h, _ := judgeHealth("node-tools", lease.ComponentNodeTools, catalogue.Manifest{}, "anchor", since, f); h != healthGood { - t.Errorf("node tools answering are %v", h) - } - // The controller: the lease held since the send, by a controller that says it is ready. - f = applied("control") - f.holder = &lease.Holder{Taken: since.Add(-time.Hour), Health: &lease.Health{Ready: true}} - if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet { - t.Errorf("a lease held by a controller older than the build is %v", h) - } - f.holder.Taken = now - if h, _ := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthGood { - t.Errorf("a new controller holding the lease and ready is %v", h) - } - f.holder.Health = &lease.Health{Why: "the self-check has not finished its first run"} - if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "control", since, f); h != healthNotYet || - !strings.Contains(why, "first run") { - t.Errorf("a controller not ready is %v (%s)", h, why) - } - // What the module runs (novox/hq ADR 0240 §4): a judging passes only when every long-running - // resource of it on that machine is stated healthy since the send; starting and unhealthy wait. - f = applied("anchor") - stated := func(state, reason string, heard time.Time) { - f.health = map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Contract: 1, SaidAt: heard, HeardAt: heard, - Resources: []inventory.ResourceHealth{ - {Module: "app", Resource: "app.server", Kind: "container", Target: "app-server", State: state, Reason: reason}, - {Module: "other", Resource: "other.server", Kind: "container", State: link.StateUnhealthy, Reason: "down"}}}} - } - plain := catalogue.Manifest{Module: "app"} - if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood { - t.Errorf("a machine whose engine states no health is judged as before, not %v (%s)", h, why) - } - stated(link.StateStarting, "", now) - if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "starting") { - t.Errorf("a resource still starting is not yet a pass: %v (%s)", h, why) - } - stated(link.StateUnhealthy, "restarting", now) - if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "restarting") { - t.Errorf("a resource unhealthy fails the judging: %v (%s)", h, why) - } - stated(link.StateHealthy, "", since.Add(-time.Minute)) - if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet { - t.Errorf("a statement from before the send says nothing of the new build: %v (%s)", h, why) - } - stated(link.StateHealthy, "", now) - if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthGood { - t.Errorf("every resource of it healthy since the send — another module's state is not its — is %v (%s)", h, why) - } - f.healthErr = errors.New("the store is away") - if h, _ := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet { - t.Errorf("health that cannot be read is never read as healthy: %v", h) - } - // And the condition it raises after the send holds it, as every condition about it does. - f.healthErr = nil - f.judged = true - f.open = []conditions.Condition{{Key: "module.app.anchor.unhealthy", Kind: kindModuleUnhealthy, Subject: conditions.Subject{ - Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"}, Raised: now, Summary: "app on anchor is not healthy"}} - if h, why := judgeHealth("app", "", plain, "anchor", since, f); h != healthNotYet || !strings.Contains(why, "module.app.anchor.unhealthy") { - t.Errorf("a module held on its own unhealthy condition is %v (%s)", h, why) - } - - // A machine that refused what it was sent: broken. - f = applied("anchor") - r := f.reports["anchor"] - r.Outcome = inventory.OutcomeRefused - f.reports["anchor"] = r - if h, _ := judgeHealth("app", "", catalogue.Manifest{}, "anchor", since, f); h != healthBroken { - t.Errorf("a refusal is %v", h) - } -} - -// The bus is never rolled out: its policy records whatever is said, a person's roll-out is refused, -// a plan builds it and sends nothing, and a push naming its machine is refused while a new build waits. - -// A merge that deletes a module's directory builds nothing for it: the module is forgotten where -// nothing holds it, and a plan whose build finds no manifest goes on instead of failing. - -func mustJSON(t *testing.T, v any) []byte { - t.Helper() - b, err := json.Marshal(v) - if err != nil { - t.Fatal(err) - } - return b -} - -func nodeID(t *testing.T, open *stores, name string) string { - t.Helper() - n, err := open.inventory.NodeByName(context.Background(), name) - if err != nil { - t.Fatal(err) - } - return n.ID -} - -// What a machine's witness says in its reports is a condition while it says it, by the host's words: -// `core...`, urgent for a rollback, a warning when it could not judge — and -// gone with the first report that no longer carries it. - -// judgeLeavesAsTheCore makes a test's leaf modules judged as a core module is, three readings after the settle -// time, for a test of the gate's other rules (a fault, the bound, a supersession) that needs a judging still -// open after a healthy reading. -func judgeLeavesAsTheCore(t *testing.T) { - t.Helper() - wasPasses, wasSettles := gateLeafPasses, gateLeafSettles - gateLeafPasses, gateLeafSettles = gatePasses, true - t.Cleanup(func() { gateLeafPasses, gateLeafSettles = wasPasses, wasSettles }) -} diff --git a/cmd/mesh-controller/generation.go b/cmd/mesh-controller/generation.go index ce649ce0..66bca026 100644 --- a/cmd/mesh-controller/generation.go +++ b/cmd/mesh-controller/generation.go @@ -7,10 +7,8 @@ import ( "io" "os" "strings" - "time" "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -162,44 +160,6 @@ func counterBehind(r link.GenerationRefusal, now int64) bool { return r.Generation >= now && r.Applied > r.Generation } -// watchGenerationRefusals is S20: every send a machine refused within the hour for the generation it was -// composed from, naming who sent it (novox/hq issue 234). One per machine, the newest refusal. -func watchGenerationRefusals(f *signalFacts) []conditions.Observation { - seen := map[string]bool{} - var out []conditions.Observation - for _, s := range f.refusedSends { - if s.RefusedAt == nil || f.now.Sub(*s.RefusedAt) > generationRefusalsSaid || seen[s.NodeName] { - continue - } - seen[s.NodeName] = true - o := conditions.Observation{Scope: conditions.ScopeMachine, ID: s.NodeName, Kind: kindOlderGeneration, - Machine: s.NodeName, Severity: conditions.Warning, - Summary: fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, and "+ - "%s applied generation %d — a sender composing from a view of the assignments the mesh has moved "+ - "past; it named %s", s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied, - modulesWords(s.Modules)), - Said: fmt.Sprintf("refused at %s", s.RefusedAt.UTC().Format(time.RFC3339)), - Headline: fmt.Sprintf("%s refused an out-of-date update", s.NodeName), - Explanation: fmt.Sprintf("Something sent %s an update made from an older list of what runs there. %s "+ - "refused it, so nothing was removed. Nothing for you to do unless it repeats.", s.NodeName, s.NodeName), - Resolved: fmt.Sprintf("%s has had no out-of-date update for an hour", s.NodeName)} - if s.CounterRaisedTo > 0 { - // Not a stale sender: the mesh's counter was behind the machine (a store put back from a backup) and - // was raised; nothing has sent the machine its declaration since, so a person is asked to. - o.Summary = fmt.Sprintf("%s refused sequence %d from %s: it was composed from assignment generation %d, "+ - "the mesh's own, and %s applied generation %d — the mesh's counter was behind the machine (a store "+ - "put back from a backup?) and is raised to %d; `push %s` sends it what the mesh holds now", - s.NodeName, s.Sequence, s.Sender, s.Generation, s.NodeName, s.RefusedApplied, s.CounterRaisedTo, - s.NodeName) - o.Explanation = fmt.Sprintf("Needs you: push %s. The mesh's record was older than %s, so %s refused its "+ - "update and kept what it had. The record is repaired; a push sends the update again.", - s.NodeName, s.NodeName, s.NodeName) - } - out = append(out, o) - } - return out -} - // modulesWords is a send's modules as a sentence says them. func modulesWords(modules []string) string { if len(modules) == 0 { diff --git a/cmd/mesh-controller/generation_test.go b/cmd/mesh-controller/generation_test.go deleted file mode 100644 index a2372bcc..00000000 --- a/cmd/mesh-controller/generation_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package main - -import ( - "encoding/json" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// The assignment generation a declaration was composed from (novox/hq issue 234). - -func bodyKeys(t *testing.T, s sendable) map[string]any { - t.Helper() - raw, err := s.Body() - if err != nil { - t.Fatal(err) - } - var keys map[string]any - if err := json.Unmarshal(raw, &keys); err != nil { - t.Fatal(err) - } - return keys -} - -// **The generation goes on the wire only to a machine whose node-engine said it reads one**: an older -// node-engine decodes strictly and refuses an unknown key, whole. No put-back mark is ever sent: a gate -// composes its put-back afresh, with the generation as it stands. -func TestTheGenerationIsSentOnlyToAMachineThatReadsOne(t *testing.T) { - resources := []map[string]any{{"id": "a", "type": "file", "path": "/etc/a", "content": "x\n"}} - reads := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: true, acting: 57} - var told sendable - told.Resources = resources - reads.stamp(&told) - keys := bodyKeys(t, told) - if keys["generation"] != float64(40) || keys["sequence"] != float64(12) { - t.Fatalf("a machine that reads a generation was sent %v", keys) - } - if _, there := keys["put_back"]; there { - t.Fatalf("a put-back mark was sent: %v", keys) - } - if told.composedFrom != 40 || told.actingEpoch != 57 { - t.Errorf("the send does not keep what it was composed from for its record: %+v", told) - } - - older := order{sequence: 12, epoch: 57, generation: 40, readsGeneration: false, acting: 57} - var untold sendable - untold.Resources = resources - older.stamp(&untold) - if _, there := bodyKeys(t, untold)["generation"]; there { - t.Fatal("a machine whose node-engine never said it reads a generation was sent one") - } - if untold.composedFrom != 40 { - t.Errorf("the generation it was composed from is recorded whether or not it was sent: %+v", untold) - } -} - -// **The sender is named**: the caller of the seat call when there is one, else the shell's account, and the -// process and build that composed it. -func TestASendNamesItsSender(t *testing.T) { - said := senderOf("g14.node-tools, through the mesh-controller seat") - if !strings.Contains(said, "g14.node-tools") || !strings.Contains(said, "pid ") || !strings.Contains(said, "build ") { - t.Fatalf("the sender reads %q", said) - } -} - -// refusedSend is a send a machine refused for its generation at a moment. -func refusedSend(at time.Time) inventory.Send { - return inventory.Send{NodeName: "anchor", Sequence: 12, Epoch: 57, Generation: 38, RefusedApplied: 40, - Sender: "a one-shot push by jochen at a shell on anchor (pid 4242 on anchor, build 2026.10.11)", - Modules: []string{"docker"}, SentAt: at.Add(-time.Second), RefusedAt: &at, Recorded: true} -} - -// **A refused send is raised naming its sender** (novox/hq issue 234): who sent it, from which generation, -// against which the machine applied, and its sequence — the facts that took a morning to look for. -func TestARefusedSendIsRaisedNamingItsSender(t *testing.T) { - now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC) - f := calm(now) - f.refusedSends = []inventory.Send{refusedSend(now.Add(-time.Minute))} - got := watchGenerationRefusals(f) - if len(got) != 1 { - t.Fatalf("%+v", got) - } - o := got[0] - if o.Key() != "machine.anchor.older-generation" || o.Machine != "anchor" { - t.Errorf("raised as %s about %q", o.Key(), o.Machine) - } - for _, want := range []string{"a one-shot push by jochen", "generation 38", "generation 40", "sequence 12"} { - if !strings.Contains(o.Summary, want) { - t.Errorf("the summary does not say %q: %s", want, o.Summary) - } - } - if o.Headline == "" || o.Explanation == "" || o.Resolved == "" { - t.Errorf("the condition is not worded for the operator: %+v", o) - } -} - -// **A refusal of a send the mesh has no record of is raised too, its sender said to be unknown** — never -// only a line on stderr. -func TestARefusalOfAnUnrecordedSendIsRaisedSayingTheSenderIsUnknown(t *testing.T) { - now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC) - f := calm(now) - s := refusedSend(now.Add(-time.Minute)) - s.Recorded, s.Sender, s.Modules = false, "a sender the mesh has no record of (no send of this sequence was recorded)", nil - f.refusedSends = []inventory.Send{s} - got := watchGenerationRefusals(f) - if len(got) != 1 || !strings.Contains(got[0].Summary, "no record of") { - t.Fatalf("an unattributed refusal was not raised as one: %+v", got) - } -} - -// **When the refusal showed the mesh's counter behind the machine, the condition asks for a push** — it was -// raised, and nothing has sent the machine its declaration since — and does not say there is nothing to do. -func TestACounterBehindTheMachineAsksForAPush(t *testing.T) { - now := time.Date(2026, 10, 11, 12, 0, 0, 0, time.UTC) - f := calm(now) - s := refusedSend(now.Add(-time.Minute)) - s.Generation, s.RefusedApplied, s.CounterRaisedTo = 12, 40, 41 - f.refusedSends = []inventory.Send{s} - got := watchGenerationRefusals(f) - if len(got) != 1 { - t.Fatalf("%+v", got) - } - if !strings.Contains(got[0].Summary, "`push anchor`") || !strings.Contains(got[0].Explanation, "push anchor") || - strings.Contains(got[0].Explanation, "Nothing for you to do") { - t.Errorf("a counter behind the machine does not ask for a push: %s / %s", got[0].Summary, got[0].Explanation) - } -} - -// **Only a refusal of the counter as it stands is the counter behind**: a stale sender's generation is below -// it, and the counter is left alone for that. -func TestOnlyARefusalOfTheCounterAsItStandsRaisesIt(t *testing.T) { - for _, c := range []struct { - refused, applied, now int64 - behind bool - }{ - {refused: 12, applied: 40, now: 12, behind: true}, // the store was put back: the mesh says 12, the machine had 40 - {refused: 38, applied: 40, now: 41, behind: false}, // a stale sender: the counter is already past - {refused: 38, applied: 40, now: 40, behind: false}, // a stale sender: the counter is where the machine is - } { - r := link.GenerationRefusal{Generation: c.refused, Applied: c.applied} - if got := counterBehind(r, c.now); got != c.behind { - t.Errorf("refused %d, applied %d, counter %d: behind %v, want %v", c.refused, c.applied, c.now, got, c.behind) - } - } -} diff --git a/cmd/mesh-controller/grants_step_integration_test.go b/cmd/mesh-controller/grants_step_integration_test.go deleted file mode 100644 index 7e54c6d8..00000000 --- a/cmd/mesh-controller/grants_step_integration_test.go +++ /dev/null @@ -1,27 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// The store test's manifests are manifests the controller takes in: checked here, where no store is needed, -// so a fixture the module rules refuse fails before the store test is run (repo-check of #230 at 325575b2). -func TestTheGrantsStepFixturesAreManifests(t *testing.T) { - for name, m := range map[string]map[string]any{"nats": natsFixture(), "restic-c1": resticFixture(false), - "restic-c2": resticFixture(true), "wallpaper": wallpaperFixture()} { - m["module"], m["version"] = strings.SplitN(name, "-", 2)[0], "1" - raw, err := json.Marshal(m) - if err != nil { - t.Fatal(err) - } - if _, err := catalogue.ParseManifest(raw); err != nil { - t.Errorf("%s: %v", name, err) - } - } -} diff --git a/cmd/mesh-controller/grants_step_store_test.go b/cmd/mesh-controller/grants_step_store_test.go deleted file mode 100644 index 8432483d..00000000 --- a/cmd/mesh-controller/grants_step_store_test.go +++ /dev/null @@ -1,196 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "errors" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/identity" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// The grants step on a store (novox/hq issue 490): the machine holding the bus is sent its new user list, -// and nothing else — not the new build of a module it runs, not a module newly assigned there. -// -// The restic shape: anchor holds the bus and runs restic at c1, as does laptop; restic's c2 gives it a -// health tool. The grants step's send to anchor composes restic at c1, carries the user list that grants -// laptop's node-engine c2's tool, and records that anchor still runs c1. A module newly assigned to anchor -// makes the step fail, said, rather than install it unjudged. -func TestTheGrantsStepSendsTheUserListAndNothingElse(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - start := time.Now().Add(-time.Hour) - build := func(module, commit string, at time.Time, manifest map[string]any) link.BuildResult { - manifest["module"], manifest["version"] = module, "1" - raw, _ := json.Marshal(manifest) - return link.BuildResult{ID: link.NewBuildID(at), Repository: "novox/mesh-catalog", Path: "modules/" + module, - On: "anchor", Module: module, Commit: commit, Manifest: raw, - Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}} - } - for _, b := range []link.BuildResult{ - build("nats", "n1", start, natsFixture()), - build("restic", "c1", start.Add(time.Second), resticFixture(false)), - build("wallpaper", "w1", start.Add(2*time.Second), wallpaperFixture()), - } { - if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil { - t.Fatal(err) - } - } - for _, a := range [][2]string{{"anchor", "nats"}, {"anchor", "restic"}, {"laptop", "restic"}} { - if _, err := inv.Assign(ctx, a[0], a[1]); err != nil { - t.Fatal(err) - } - } - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: "node." + n, Kind: inventory.BusNode, Node: n}); err != nil { - t.Fatal(err) - } - } - - wasEpoch := epochForActs - epochForActs = func(context.Context) (uint64, error) { return 7, nil } - t.Cleanup(func() { epochForActs = wasEpoch }) - - // A send to anchor composed as sendToEach composes it: numbered, planned, declared, stamped. - compose := func(under context.Context) (catalogue.Resolution, sendable) { - t.Helper() - numbered, err := allot(under, inv, "anchor") - if err != nil { - t.Fatal(err) - } - plan, _, err := resolveNode(under, open, "anchor") - if err != nil { - t.Fatal(err) - } - declared, err := composeSend(under, open, "anchor") - if err != nil { - t.Fatal(err) - } - numbered.stamp(&declared) - return plan, declared - } - record := func(declared sendable) { - t.Helper() - body, err := declared.Body() - if err != nil { - t.Fatal(err) - } - if _, err := recordSent(ctx, inv, "anchor", body, declared.Builds, declared.Epoch, - sentRecordOf(ctx, declared)); err != nil { - t.Fatal(err) - } - } - // Kept for anchor alone. laptop is never recorded as sent, on purpose: composing anchor resolves laptop - // too (who is on the private network), and a step that kept every machine's builds refused anchor's - // composition for want of laptop's last send (repo-check of #230 at 0f853e93). - kept := keepingEveryBuild(keepingRecorded(ctx), "anchor") - - // What anchor was last sent: everything at the builds of before the merge, as an ordinary send sends it. - _, before := compose(keepingRecorded(ctx)) - record(before) - plan, declared := compose(kept) - if only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared); err != nil || only != "" { - t.Fatalf("with nothing changed, the step reads %q, %v", only, err) - } - - // The merge: restic's c2 gives it a health tool. - if _, _, err := takeIn(ctx, inv, build("restic", "c2", start.Add(time.Minute), - resticFixture(true))); err != nil { - t.Fatal(err) - } - - // The step's send to anchor: restic as anchor runs it, c1, and the user list granting c2's tool. - generation, err := inv.AssignmentGeneration(ctx) - if err != nil { - t.Fatal(err) - } - plan, declared = compose(kept) - for _, m := range plan.Modules { - if m.Module == "restic" && len(m.Tools) > 0 { - t.Fatalf("the grants step composed restic's new build on anchor: tools %v", m.Tools) - } - } - if declared.Builds["restic"] != "c1" { - t.Fatalf("the step's send records it carries restic %q; want c1, which anchor runs", declared.Builds["restic"]) - } - if !strings.Contains(declared.BusUsers, "mesh.mod.restic.tool.backup_health.laptop") { - t.Errorf("the step's declaration does not grant laptop's node-engine restic's new tool:\n%s", declared.BusUsers) - } - // The generation it carries is the current one (novox/hq issue 234), and the step does not raise it. - if declared.composedFrom != generation { - t.Errorf("the step's send was composed from generation %d; the mesh is at %d", declared.composedFrom, generation) - } - // The guard, on the very declaration the send sends: only the user list differs from the last send. - if only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared); err != nil || only != "" { - t.Fatalf("the step reads as changing more than the user list: %q, %v", only, err) - } - // And the gate's refusal still reads the step's send: it moves nothing there. - if names, err := ungatedIn(kept, open, []string{"anchor"}, ""); err != nil || strings.Join(names, ",") != "anchor" { - t.Fatalf("the step's send is refused as a move: %v, %v", names, err) - } - // Recorded as the send records it: anchor still runs restic c1, its gate still to come there. - record(declared) - if sent, _, err := inv.SentBuilds(ctx, "anchor"); err != nil || sent["restic"] != "c1" { - t.Fatalf("after the step anchor reads as sent restic %q (%v); want c1", sent["restic"], err) - } - if after, err := inv.AssignmentGeneration(ctx); err != nil || after != generation { - t.Fatalf("the step moved the assignment generation from %d to %d (%v)", generation, after, err) - } - - // A module newly assigned to anchor: the step would install it, unjudged, so it is refused unsent. - if _, err := inv.Assign(ctx, "anchor", "wallpaper"); err != nil { - t.Fatal(err) - } - plan, declared = compose(kept) - only, err := grantsOnlyIn(ctx, open, "anchor", plan, declared) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(only, "wallpaper newly assigned") || !strings.Contains(only, "+wallpaper") { - t.Fatalf("a new assignment on the bus's machine reads %q; want the step refused, naming it", only) - } - // And the send itself refuses it, unsent: judged where it is composed, before the bus is dialled. - identity.ForTest(t) - if _, err := sendToEach(kept, open, []string{"anchor"}); !errors.Is(err, errNotGrantsOnly) || - !strings.Contains(err.Error(), "wallpaper") { - t.Fatalf("the grants step's send of a new assignment was not refused by the send: %v", err) - } -} - -// fixtureImage is the image every fixture container of the grants step's tests runs. -var fixtureImage = "registry.invalid:5000/restic/backup@sha256:" + strings.Repeat("b", 64) - -// resticFixture is restic's manifest in the shape of mesh-catalog #210: at c2 (withTool) its backup judged by -// its new tool `backup_health`, beside a check of another kind it does not run itself — a tool check alone -// is refused (ADR 0227 rule 8, ADR 0240 rule 2) — and at c1 neither. -func resticFixture(withTool bool) map[string]any { - backup := map[string]any{"id": "backup", "type": "container", "name": "restic-backup", "image": fixtureImage} - measure := map[string]any{"id": "measure", "type": "container", "name": "restic-measure", "image": fixtureImage} - m := map[string]any{"resources": []any{backup, measure}} - if withTool { - backup["health"] = map[string]any{"kind": catalogue.HealthTool, "tool": "backup_health"} - measure["health"] = map[string]any{"kind": "runtime"} - m["tools"] = []string{"backup_health"} - } - return m -} - -// natsFixture is the bus's module: it holds mesh-broker and is sent the user list. -func natsFixture() map[string]any { - return map[string]any{"bus-users": "/var/lib/nats-module/conf/accounts.conf", - "claims": []any{map[string]any{"name": catalogue.BrokerSeat, "scope": catalogue.ScopeMesh}}} -} - -// wallpaperFixture is a module the bus's machine is newly assigned. -func wallpaperFixture() map[string]any { - return map[string]any{"resources": []any{ - map[string]any{"id": "paper", "type": "container", "name": "wallpaper", "image": fixtureImage}}} -} diff --git a/cmd/mesh-controller/grants_step_test.go b/cmd/mesh-controller/grants_step_test.go deleted file mode 100644 index c8502758..00000000 --- a/cmd/mesh-controller/grants_step_test.go +++ /dev/null @@ -1,176 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "slices" - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// The grants reach the bus before the gate (novox/hq issue 490). -// -// On 2026-10-11 a merge gave restic, assigned to every machine, a new health tool. Its walk sent restic to -// its first machine and judged it there; that machine's node-engine asked the new tool and the bus refused -// it, because the grant was in the bus's user list, which only the machine holding the bus carries — and -// that machine runs restic too, so it was left out of the send while restic's new build waited there for -// the very gate that could not pass. A person pushed it by hand, carrying restic's new build there unjudged. - -// aSend is one send the walk made: to which machines, and whether every build there was kept. -type aSend struct { - names []string - keepsAll bool - judged []string -} - -// sendsRecorded replaces the walk's send and the reading of the bus's user list for one test: the machine -// holding the bus is novox, and its list is behind as behind says. -func sendsRecorded(t *testing.T, holder string, behind bool, refuse error) *[]aSend { - t.Helper() - var sends []aSend - wasSend, wasBehind := sendRollout, brokerBehindOf - t.Cleanup(func() { sendRollout, brokerBehindOf = wasSend, wasBehind }) - brokerBehindOf = func(_ context.Context, _ *stores, names []string) (string, bool, error) { - if slices.Contains(names, holder) { - return holder, false, nil - } - return holder, behind, nil - } - sendRollout = func(ctx context.Context, _ *stores, names []string) ([]string, error) { - s := aSend{names: append([]string(nil), names...), keepsAll: len(names) == 1 && everyBuildKept(ctx, names[0])} - for n := range scopeOf(ctx).judged { - s.judged = append(s.judged, n) - } - sends = append(sends, s) - if refuse != nil && s.keepsAll { - return nil, refuse - } - return names, nil - } - return &sends -} - -// The restic shape: a new tool on a module on every machine, its first machine ace, the bus on novox. -func TestAWalkSendsTheGrantsToTheBusBeforeTheFirstMachineIsJudged(t *testing.T) { - sends := sendsRecorded(t, "novox", true, nil) - sent, grants, err := sendJudged(t.Context(), nil, "ace") - if err != nil { - t.Fatal(err) - } - if len(*sends) != 2 { - t.Fatalf("the walk made %d send(s), want the grants step and then the first machine: %+v", len(*sends), *sends) - } - first, then := (*sends)[0], (*sends)[1] - if strings.Join(first.names, ",") != "novox" || !first.keepsAll || len(first.judged) != 0 { - t.Errorf("the first send is %+v, want novox alone, every build there kept, judging nothing", first) - } - if strings.Join(then.names, ",") != "ace" || then.keepsAll || strings.Join(then.judged, ",") != "ace" { - t.Errorf("the second send is %+v, want ace, judged", then) - } - if strings.Join(sent, ",") != "ace" { - t.Errorf("the gate's machines are %v, want ace alone: the bus's machine is not judged", sent) - } - if grants == nil || grants.Node != "novox" || grants.At == nil || grants.Failed != "" { - t.Fatalf("the gate keeps %+v as its grants step", grants) - } - line := gateLine(&inventory.PlanGate{Machines: sent, Grants: grants}) - if !strings.Contains(line, "grants step: novox sent the bus's user list first, its builds kept") { - t.Errorf("plans says the gate as %q", line) - } -} - -// Nothing more when there is nothing to carry: the list unchanged, or the first machine holds the bus. -func TestAWalkTakesNoGrantsStepWhenTheListIsAlreadyThere(t *testing.T) { - for _, c := range []struct { - name string - node string - behind bool - }{{"the list unchanged", "ace", false}, {"the first machine holds the bus", "novox", true}} { - t.Run(c.name, func(t *testing.T) { - sends := sendsRecorded(t, "novox", c.behind, nil) - _, grants, err := sendJudged(t.Context(), nil, c.node) - if err != nil { - t.Fatal(err) - } - if len(*sends) != 1 || (*sends)[0].keepsAll || grants != nil { - t.Errorf("the walk made %+v with grants %+v, want the judged send alone", *sends, grants) - } - }) - } -} - -// A grants step that cannot be sent is said on the gate, and the walk goes on as it did before. -func TestAGrantsStepThatFailsIsSaidAndTheSendGoesOn(t *testing.T) { - sends := sendsRecorded(t, "novox", true, errors.New("a bus upgrade waits for a person")) - sent, grants, err := sendJudged(t.Context(), nil, "ace") - if err != nil { - t.Fatal(err) - } - if strings.Join(sent, ",") != "ace" || len(*sends) != 2 { - t.Errorf("the walk made %+v", *sends) - } - if grants == nil || grants.Failed == "" || grants.At != nil { - t.Fatalf("the gate keeps %+v", grants) - } - if line := gateLine(&inventory.PlanGate{Machines: sent, Grants: grants}); !strings.Contains(line, - "grants step to novox FAILED, sent without it: a bus upgrade waits for a person") { - t.Errorf("plans says the gate as %q", line) - } -} - -// A step whose send would change more than the user list is refused unsent by the send (sendToEach): the -// gate says it FAILED with what differed, and the walk goes on. -func TestAGrantsStepThatWouldCarryMoreIsNotSent(t *testing.T) { - refusal := fmt.Errorf("%w: its send would change more than the bus's user list: -postgres.login-n8n", errNotGrantsOnly) - sends := sendsRecorded(t, "novox", true, refusal) - sent, grants, err := sendJudged(t.Context(), nil, "ace") - if err != nil { - t.Fatal(err) - } - if strings.Join(sent, ",") != "ace" || len(*sends) != 2 { - t.Errorf("the walk made %+v, want the refused step and then ace's judged send", *sends) - } - if grants == nil || grants.At != nil || !strings.Contains(grants.Failed, "-postgres.login-n8n") { - t.Fatalf("the gate keeps %+v", grants) - } -} - -// The step's exemption is the list's content alone: its path or mode moving is a change like any other. -func TestOnlyTheUserListsContentIsExempt(t *testing.T) { - list := "nats.bus-users" - file := func(fields map[string]any) sentResource { - r := map[string]any{"id": list, "type": "file", "path": "/conf/accounts.conf", "mode": "0600", "content": "a"} - for k, v := range fields { - r[k] = v - } - body, _ := json.Marshal(map[string]any{"resources": []any{r}}) - s, err := summarize(body) - if err != nil { - t.Fatal(err) - } - return s[0] - } - was := file(nil) - if other := otherThanTheList(diffSent([]sentResource{was}, []sentResource{file(map[string]any{"content": "b"})}), list); len(other) != 0 { - t.Errorf("the list's new content reads as more: %v", other) - } - for _, f := range []map[string]any{{"mode": "0644"}, {"path": "/elsewhere"}, {"content": "b", "owner": "nats"}} { - other := otherThanTheList(diffSent([]sentResource{was}, []sentResource{file(f)}), list) - if len(other) != 1 || !strings.HasPrefix(other[0], "~"+list) { - t.Errorf("the list's resource changed by %v reads %v; want it refused", f, other) - } - } -} - -// The step keeps the builds of the machine it sends alone: the others its composition resolves are composed -// as any send composes them. -func TestTheGrantsStepKeepsOneMachinesBuilds(t *testing.T) { - ctx := keepingEveryBuild(t.Context(), "novox") - if !everyBuildKept(ctx, "novox") || everyBuildKept(ctx, "ace") || everyBuildKept(t.Context(), "novox") { - t.Error("the grants step keeps the builds of a machine it does not send") - } -} diff --git a/cmd/mesh-controller/group_one_batch_test.go b/cmd/mesh-controller/group_one_batch_test.go deleted file mode 100644 index b7c69e57..00000000 --- a/cmd/mesh-controller/group_one_batch_test.go +++ /dev/null @@ -1,139 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// A group is one batch and one plan (novox/hq ADR 0418 clauses 3 and 6): the batch holding a merge of a group with a -// member still to merge does not close by the merge window, however long that member takes, and a group with a -// member of each kind is one batch whatever order its members are heard in. - -func TestAGroupsBatchWaitsForItsLastMemberHoweverLongItTakes(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - states := map[string]string{"novox/one": "ready", "novox/two": "ready"} - groupStates(t, "feat/g", states) - first := fromHead(repoMerge("one", "c1", t0), "feat/g") - hear(t, open, first, t0) - // Past the window and past its maximum, with the second member still to merge: one batch, no walk. - for _, after := range []time.Duration{90 * time.Second, 11 * time.Minute, 30 * time.Minute} { - cutAt(t, open, t0.Add(after)) - ws, bs := walks(t, open) - if len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Batch.Holds) != 1 { - t.Fatalf("%s after the first member: %d walk(s), %d batch(es) %+v", after, len(ws), len(bs), bs) - } - } - // The last member merges: the window runs again from that merge. - at := t0.Add(30 * time.Minute) - states["novox/two"] = "merged" - hear(t, open, fromHead(repoMerge("two", "c2", at), "feat/g"), at.Add(time.Second)) - cutAt(t, open, at.Add(time.Second+89*time.Second)) - if ws, _ := walks(t, open); len(ws) != 0 { - t.Fatalf("cut before the window ran again: %+v", ws) - } - cutAt(t, open, at.Add(time.Second+90*time.Second)) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 0 || ws[0].CommitOf("novox/one") != "c1" || ws[0].CommitOf("novox/two") != "c2" { - t.Fatalf("after the window: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs)) - } -} - -// A group with a member of each kind is one batch and one walk, in both orders of hearing, started without -// mesh-delivery's word. -func TestAGroupWithAMemberOfEachKindIsOneBatchAndOneWalk(t *testing.T) { - for name, ownFirst := range map[string]bool{"controller member first": true, "catalogue member first": false} { - t.Run(strings.ReplaceAll(name, " ", "_"), func(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - states := map[string]string{"novox/mesh-controller": "ready", "novox/mesh-catalog": "ready"} - groupStates(t, "feat/both-kinds", states) - own := fromHead(repoMerge("mesh-controller", "k1", t0), "feat/both-kinds") - cat := fromHead(catalogueMerge("cat1aaaaaaaa", "app", t0), "feat/both-kinds") - first, second := cat, own - if ownFirst { - first, second = own, cat - } - second.MergedAt = t0.Add(10 * time.Second).Format(time.RFC3339Nano) - hear(t, open, first, t0) - states["novox/"+second.Repo] = "merged" - hear(t, open, second, t0.Add(10*time.Second)) - ws, bs := walks(t, open) - if len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 { - t.Fatalf("both members heard: %d walk(s), %d batch(es) %+v; want one batch of both", len(ws), len(bs), bs) - } - cutAt(t, open, t0.Add(10*time.Second+90*time.Second)) - ws, bs = walks(t, open) - if len(ws) != 1 || len(bs) != 0 || ws[0].CommitOf("novox/mesh-controller") != "k1" || - ws[0].CommitOf("novox/mesh-catalog") != cat.Commit { - t.Fatalf("after the window: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs)) - } - if ws[0].Delivery != nil && ws[0].Delivery.Awaits != "" { - t.Fatalf("a plan carrying the controller waits for %s's word", ws[0].Delivery.Awaits) - } - }) - } -} - -// A group whose last member has not merged ten minutes after its first raises a condition naming it, and the held -// batch is not the batch S18 reads as a cut the controller failed to make. -func TestAHeldBatchRaisesTheGroupConditionAndNotTheBatchNotCutOne(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - groupStates(t, "feat/g", map[string]string{"novox/one": "ready", "novox/two": "ready"}) - hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/g"), t0) - now := t0.Add(12 * time.Minute) - batches, err := gatherBatches(t.Context(), open.inventory, now) - if err != nil || len(batches) != 1 { - t.Fatalf("batches %+v: %v", batches, err) - } - f := &signalFacts{now: now, batches: batches} - if got := watchBatchesNotCut(f); len(got) != 0 { - t.Fatalf("a held batch is read as not cut: %+v", got) - } - got := watchGroupsNotMerged(f) - if len(got) != 1 || got[0].Kind != kindGroupNotMerged || !strings.Contains(got[0].Summary, "novox/two") || - !strings.Contains(got[0].Summary, "feat/g") { - t.Fatalf("the group's condition: %+v", got) - } - if early := watchGroupsNotMerged(&signalFacts{now: t0.Add(5 * time.Minute), batches: batches}); len(early) != 0 { - t.Fatalf("raised before the maximum: %+v", early) - } -} - -// The batch held for a group survives a restart: the hold is in the record. -func TestAHeldBatchSurvivesARestart(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - groupStates(t, "feat/g", map[string]string{"novox/one": "ready", "novox/two": "ready"}) - hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/g"), t0) - reopened, err := open.inventory.Batches(t.Context()) - if err != nil || len(reopened) != 1 || len(reopened[0].Delivery.Batch.Holds) != 1 { - t.Fatalf("the record of the held batch: %+v %v", reopened, err) - } - cutAt(t, open, t0.Add(time.Hour)) - if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 || bs[0].State != inventory.PlanAssembling { - t.Fatalf("after an hour: %d walk(s), batches %+v", len(ws), bs) - } -} - -// A mesh-delivery that does not yet say the group's gate is composed merges member two only after member one is -// delivered, so a batch held for it would wait for ever: the merge waits for the window, as before. -func TestAnOlderMeshDeliverysGroupIsNotHeldOpen(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - groupStatesFrom(t, "feat/g", map[string]string{"novox/one": "ready", "novox/two": "ready"}, false) - hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/g"), t0) - if _, bs := walks(t, open); len(bs) != 1 || len(bs[0].Delivery.Batch.Holds) != 0 { - t.Fatalf("held for an older mesh-delivery's group: %+v", bs) - } - cutAt(t, open, t0.Add(90*time.Second)) - if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 { - t.Fatalf("at its window: %d walk(s), %d batch(es)", len(ws), len(bs)) - } -} diff --git a/cmd/mesh-controller/group_wait.go b/cmd/mesh-controller/group_wait.go deleted file mode 100644 index 8713318e..00000000 --- a/cmd/mesh-controller/group_wait.go +++ /dev/null @@ -1,228 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "sort" - "strings" - "time" - - "github.com/nats-io/nats.go" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// **A delivery group still assembles** (novox/hq ADR 0306 decision 2): a merge whose delivery group has a member -// not yet merged waits for the merge window, whether or not a batch is assembling, so the group's members walk in -// one batch, in their order (ADR 0276 decision 5). Which groups exist and where each member stands is -// mesh-delivery's record (ADR 0239): a group is the pull requests of two repositories or more sharing a head -// branch name, which is its id. The controller asks it, through its `groups` verb, when a merge from a head is -// heard; a member whose pull request is open (proposed, checked, ready or rejected) is still to merge. When -// mesh-delivery does not answer within groupAskWithin, or answers something unreadable, the merge waits: the safe -// error is to wait, and its batch says why. - -// groupAskWithin is how long mesh-delivery is given to say a merge's group, while the merge is being heard. -var groupAskWithin = 3 * time.Second - -// openPullStates are the states of a delivery whose pull request is open and not merged (mesh-delivery's table): -// a member in one of them is still to merge. Closed unmerged is stopped, final, and waits for nothing. -var openPullStates = map[string]bool{"proposed": true, "checked": true, "ready": true, "rejected": true} - -// groupMember is one member of a delivery group as mesh-delivery's `groups` says it: `@ (state)`. -type groupMember struct { - Repository, State string - // Composed says the group's line says its gate is the composed check's (a mesh-delivery that merges a group in - // one pass): only such a group is held open for its last member, since an older mesh-delivery merges member two - // only after member one is delivered, and a held batch would wait for ever. - Composed bool -} - -// groupsLine is one group as mesh-delivery's `groups` says it: its id (the head branch name), its members as text and, -// from a mesh-delivery that knows novox/hq ADR 0382, each member again as an object (`member_details`) naming its pull -// request's number, whole head, base branch and whether it is open. -type groupsLine struct { - ID string `json:"id"` - Members []string `json:"members"` - Details []memberLine `json:"member_details,omitempty"` - // ComposedGate says the group's gate is the composed check's (novox/hq ADR 0403): a member's own check runs the - // repository's check alone. Said by a mesh-delivery that merges a group in one pass; absent from an older one, whose - // members keep their own gate. - ComposedGate bool `json:"composed_gate,omitempty"` -} - -// memberLine is one member of a group, as `groups` says it beside its text. -type memberLine struct { - ID string `json:"id"` - Repository string `json:"repository"` - State string `json:"state"` - Number int `json:"number"` - Head string `json:"head"` - Base string `json:"base"` - Open bool `json:"open"` -} - -// groupsAsked is mesh-delivery's `groups`: every group it keeps, or an error when it cannot be asked or did not -// answer within groupAskWithin. A seam a test replaces. -var groupsAsked = func(ctx context.Context, held bool) ([]groupsLine, error) { - if !held { - return nil, fmt.Errorf("no holder of %s is on record to say it", catalogue.DeliverySeat) - } - // On the bus as this process is: the serving controller's own connection, lent, or — in a verb that runs as a command - // of its own, such as a recheck's `delivery check` — one dialled for the moment (onTheBus). - var raw json.RawMessage - err := onTheBus(func(conn *nats.Conn) error { - asking, cancel := context.WithTimeout(ctx, groupAskWithin) - defer cancel() - var err error - raw, err = askDeliveryOwner(asking, conn, "groups", map[string]any{}) - return err - }) - if err != nil { - if errors.Is(err, link.ErrNothingServes) { - return nil, fmt.Errorf("nothing serves %s.groups", catalogue.DeliverySeat) - } - return nil, err - } - var lines []groupsLine - if err := json.Unmarshal(raw, &lines); err != nil { - return nil, fmt.Errorf("%s.groups answered something unreadable: %w", catalogue.DeliverySeat, err) - } - return lines, nil -} - -// deliveryGroupOf is what mesh-delivery says of the group a head names: its members, and whether the head names a -// group at all; an error when it cannot be asked or did not answer. A seam a test replaces. -var deliveryGroupOf = func(ctx context.Context, held bool, head string) ([]groupMember, bool, error) { - lines, err := groupsAsked(ctx, held) - if err != nil { - return nil, false, err - } - for _, l := range lines { - if l.ID != head { - continue - } - var members []groupMember - for _, s := range l.Members { - m := memberOf(s) - m.Composed = l.ComposedGate - members = append(members, m) - } - return members, true, nil - } - return nil, false, nil -} - -// memberOf reads one member as `groups` says it: `novox/mesh-sdk@d41d255f28ad (held)`. -func memberOf(s string) groupMember { - id, state := strings.TrimSpace(s), "" - if i := strings.LastIndex(id, " ("); i >= 0 && strings.HasSuffix(id, ")") { - id, state = id[:i], id[i+2:len(id)-1] - } - repository, _, _ := strings.Cut(id, "@") - return groupMember{Repository: repository, State: state} -} - -// groupWait says whether a merge heard now waits for the window for its delivery group, and why in words; nothing -// when it names no head, its head names no group, or every other member is merged or closed. -// -// A member of another repository whose merge this controller already keeps in a batch not yet cut, of either -// kind, is merged, whatever mesh-delivery says yet (it marks a member merged once its walk opened): the merge joins -// that batch by their head. **Kinds do not part a group** (novox/hq ADR 0418 clause 6): a controller member and a -// catalogue member share one batch. -// -// Asked before the plans are held: a slow answer holds up no walk. -func groupWait(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, held, own bool) (string, error) { - why, _, err := groupWaitHolding(ctx, inv, m, held) - return why, err -} - -// groupWaitHolding is groupWait, and whether the wait is for a member still to merge: such a batch is held open -// however long it takes (ADR 0418 clause 3). A wait only because mesh-delivery did not say is no hold. -func groupWaitHolding(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, held bool) (string, bool, error) { - if m.Head == "" { - return "", false, nil - } - open, isGroup, composed, err := openMembersOf(ctx, inv, held, m.Head, m.Owner+"/"+m.Repo) - if err != nil { - return fmt.Sprintf("%s did not say whether group %s has a member still to merge (%v): it waits for the "+ - "window, the safe side", catalogue.DeliverySeat, m.Head, err), false, nil - } - if !isGroup || len(open) == 0 { - return "", false, nil - } - return "group " + m.Head + " has a member still to merge: " + strings.Join(open, ", "), composed, nil -} - -// openMembersOf are the members of the group a head names that are still to merge, in words: those whose pull -// request is open and whose merge no batch not yet cut keeps. self is a repository to leave out, the merge being heard. -func openMembersOf(ctx context.Context, inv *inventory.Inventory, held bool, head, self string) ([]string, bool, bool, error) { - kept, err := keptFromHead(ctx, inv, head) - if err != nil { - return nil, false, false, err - } - members, isGroup, err := deliveryGroupOf(ctx, held, head) - if err != nil || !isGroup { - return nil, isGroup, false, err - } - var open []string - composed := false - for _, g := range members { - composed = composed || g.Composed - if strings.EqualFold(g.Repository, self) || kept[strings.ToLower(g.Repository)] || !openPullStates[g.State] { - continue - } - open = append(open, g.Repository+" ("+g.State+")") - } - sort.Strings(open) - return open, true, composed, nil -} - -// keptFromHead are the repositories (lower case) of the merges from a head that batches not yet cut hold, of -// either kind. -func keptFromHead(ctx context.Context, inv *inventory.Inventory, head string) (map[string]bool, error) { - batches, err := inv.Batches(ctx) - if err != nil { - return nil, err - } - kept := map[string]bool{} - for _, b := range batches { - merges, err := inv.MergesOf(ctx, b.ID) - if err != nil { - return nil, err - } - for _, k := range merges { - if announcedOf(k).Head == head { - kept[strings.ToLower(k.Repository)] = true - } - } - } - return kept, nil -} - -// batchOfGroup is the batch not yet cut that holds a merge of another repository from the same head: the group's -// batch, whatever kind it is. -func batchOfGroup(ctx context.Context, inv *inventory.Inventory, head, repository string) (inventory.Plan, bool, error) { - if head == "" { - return inventory.Plan{}, false, nil - } - batches, err := inv.Batches(ctx) - if err != nil { - return inventory.Plan{}, false, err - } - for _, b := range batches { - merges, err := inv.MergesOf(ctx, b.ID) - if err != nil { - return inventory.Plan{}, false, err - } - for _, k := range merges { - if announcedOf(k).Head == head && !strings.EqualFold(k.Repository, repository) { - return b, true, nil - } - } - } - return inventory.Plan{}, false, nil -} diff --git a/cmd/mesh-controller/handacts.go b/cmd/mesh-controller/handacts.go index 97e34a25..6da4383c 100644 --- a/cmd/mesh-controller/handacts.go +++ b/cmd/mesh-controller/handacts.go @@ -55,23 +55,9 @@ const causeDrill = "drill" // repair, and does not count toward `healer-wanted` — whatever cause it gives. A verb not listed, or // listed without a decision, counts, so a new verb is a repair until its entry says otherwise. var handActVerbs = []handActVerb{ - // Repairs: each repeated is a healer the mesh lacks. A push by hand is exactly what roll-out by - // default (ADR 0236) exists to end — except a push that only moved builds a `record` policy held for - // a person's word (ADR 0242), which the push itself reads from what it carried (recorded_push.go). - {Verb: "push", Decision: "a recorded build moves only by a person's push: that push is the word its " + - "upgrade policy asks for (ADR 0242)", DecidedWhen: pushedRecorded}, - // Stopping or starting a walk the operator chose on a warrant (novox/hq ADR 0259) is their decision. - {Verb: "plans stop", Decision: "the operator's answer to an ask is their decision, not a repair (ADR 0259)", - DecidedFor: []string{conditions.CauseOperatorAnswer}}, - {Verb: "plans close"}, // A module registered at references by hand, because the builder had no key for it (ADR 0526 clause 3): a repair // the builder should have made, so it counts. {Verb: "register"}, - // A walk started by a person instead of its delivery's owner (novox/hq ADR 0239): the owner down, or - // not trusted with it — either is a repair the owner should have made. Unless the operator chose it on - // a warrant (ADR 0259). - {Verb: "plans go", Decision: "the operator's answer to an ask is their decision, not a repair (ADR 0259)", - DecidedFor: []string{conditions.CauseOperatorAnswer}}, // An act the operator chose on a warrant (novox/hq ADR 0259): asked by the controller, answered on a // channel that proved who answered, performed by the controller as itself. {Verb: handActWarrant, Decision: "the operator chose it, answering what the controller asked (ADR 0259)"}, @@ -200,19 +186,6 @@ func (f handActFlags) record(ctx context.Context, verb string, args []string) { } act := link.HandAct{Verb: verb, Args: args, Why: strings.TrimSpace(*f.why), Cause: strings.TrimSpace(*f.cause), Condition: strings.TrimSpace(*f.condition)} - // A push naming one machine says whether it only moves recorded builds (novox/hq issue 301): - // read from what it carries, before it is sent. - if verb == "push" && len(args) == 1 && !strings.HasPrefix(args[0], "-") { - switch carried, why, err := recordedPushOf(ctx, args[0]); { - case err != nil: - fmt.Fprintf(os.Stderr, "whether this push only moves recorded builds could not be read, so it is "+ - "recorded as a push by hand: %v\n", err) - case len(carried) > 0: - act.Kind, act.Carried = link.KindRecordedBuilds, carried - default: - fmt.Printf("a push by hand, not of recorded builds only: %s\n", why) - } - } err := onTheBus(func(conn *nats.Conn) error { written, err := link.RecordHandAct(ctx, conn, act) act = written @@ -319,13 +292,6 @@ func listHandActs(ctx context.Context, args []string, w io.Writer) error { fmt.Fprintf(w, ", condition %s", a.Condition) } fmt.Fprintln(w, ")") - if pushedRecorded(a) { - carried := strings.Join(a.Carried, "; ") - if carried == "" { - carried = recordedBefore[a.ID] - } - fmt.Fprintf(w, " a push of recorded builds, no repair: %s\n", carried) - } } if len(repeated) > 0 { causes := make([]string, 0, len(repeated)) diff --git a/cmd/mesh-controller/handacts_test.go b/cmd/mesh-controller/handacts_test.go index 19d5a4f5..27e53b3d 100644 --- a/cmd/mesh-controller/handacts_test.go +++ b/cmd/mesh-controller/handacts_test.go @@ -9,44 +9,6 @@ import ( "github.com/novox/mesh-controller/internal/inventory" ) -// **Every verb that repairs by hand takes a required why** (novox/hq to-be 45 §7): refused before -// anything is done, through the seat, through `command`, and at a shell where nothing automated runs -// the verb. -func TestARepairByHandWithoutAReasonIsRefused(t *testing.T) { - for _, c := range []struct { - verb string - args map[string]any - }{ - {"push", map[string]any{"node": "anchor"}}, - {"push", map[string]any{}}, - {"plans", map[string]any{"close": "plan-1"}}, - {"plans", map[string]any{"stop": "plan-1"}}, - {"hand-act", map[string]any{"what": "restarted the proxy", "cause": "proxy-stuck"}}, - } { - argv, err := argvFor(c.verb, c.args) - if c.verb == "plans" && err == nil { - // The seat composes the command line; the command refuses it, before opening anything. - err = plansCommand(context.Background(), argv[1:]) - } - if err == nil || !strings.Contains(err.Error(), "why") { - t.Errorf("%s %v was not refused for want of why: %v %v", c.verb, c.args, argv, err) - } - } - for _, args := range [][]string{{"EVENTS", "controller"}} { - if err := consumerReset(context.Background(), args); err == nil || !strings.Contains(err.Error(), "--why") { - t.Errorf("consumer-reset without why: %v", err) - } - } - if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--cause", "x"}); err == nil || - !strings.Contains(err.Error(), "--why") { - t.Errorf("hand-act record without why: %v", err) - } - if err := handActCommand(context.Background(), []string{"record", "restarted the proxy", "--why", "it hung"}); err == nil || - !strings.Contains(err.Error(), "--cause") { - t.Errorf("hand-act record without a cause: %v", err) - } -} - // With a reason, the seat passes it to the command, and a verb that only reads is not held to one. func TestARepairByHandCarriesItsReason(t *testing.T) { for _, c := range []struct { @@ -54,11 +16,6 @@ func TestARepairByHandCarriesItsReason(t *testing.T) { args map[string]any want string }{ - {"push", map[string]any{"node": "anchor", "why": "stuck", "cause": "sent-not-reported"}, - "push anchor --wait 0s --why stuck --cause sent-not-reported"}, - {"plans", map[string]any{"close": "plan-1", "why": "the report will not come"}, - "plans close plan-1 --why the report will not come"}, - {"plans", map[string]any{"retry": "plan-1"}, "plans retry plan-1"}, {"hand-act", map[string]any{"what": "restarted", "why": "hung", "cause": "proxy", "condition": "machine.a.silent"}, "hand-act record restarted --why hung --cause proxy --condition machine.a.silent"}, {"command", map[string]any{"command": "plans plan-1"}, "plans plan-1"}, diff --git a/cmd/mesh-controller/handover_test.go b/cmd/mesh-controller/handover_test.go index 449f2f9e..053041ba 100644 --- a/cmd/mesh-controller/handover_test.go +++ b/cmd/mesh-controller/handover_test.go @@ -5,10 +5,7 @@ import ( "errors" "strings" "testing" - "time" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -69,28 +66,6 @@ func TestAHandOverIsRefusedThroughEveryVerb(t *testing.T) { } } -// The condition's words are plain and name no machine's binary; the operator's line, with the node and the path -// (novox/hq ADR 0272), is in the summary the module's health gives (moduleHealthWord) and in the evidence. -func TestTheUsedAsFoundConditionNamesTheOperatorsLine(t *testing.T) { - rs := []inventory.ResourceHealth{foundDirectory("notes", time.Now().Add(-24*time.Hour))} - o := usedAsFoundObservation("notes", "laptop", "notes on laptop uses notes.data as found", rs) - if strings.Contains(o.Needs, "mesh-host") || strings.Contains(o.Explanation, "mesh-host") || - !strings.Contains(o.Needs, "control-node") { - t.Fatalf("the condition's words: %q %q", o.Needs, o.Explanation) - } - if why, ok := conditions.PlainWords(conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Needs: o.Needs, - Resolved: o.Resolved}, "laptop"); !ok { - t.Fatalf("not plain: %s", why) - } - f := gateFacts{now: time.Now(), health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(), - Resources: rs}}} - h, why := moduleHealthWord("notes", "laptop", time.Now().Add(-time.Hour), f) - if h != healthPerson || !strings.Contains(why, "`nox node hand-over laptop ` on the control-node") || - strings.Contains(why, "mesh-host") || strings.Contains(why, "/srv/") { - t.Fatalf("the module's health reads %v %q; want the operator's line", h, why) - } -} - // **Nothing is asked of a node the mesh does not know, and the engine's refusal is the command's failure** // (review of issue 356): a refused hand-over never exits as a success. func TestAHandOverAsksOnlyAKnownNodeAndFailsOnARefusal(t *testing.T) { diff --git a/cmd/mesh-controller/healers.go b/cmd/mesh-controller/healers.go index 7bc1c999..f9879cfc 100644 --- a/cmd/mesh-controller/healers.go +++ b/cmd/mesh-controller/healers.go @@ -12,8 +12,6 @@ import ( "sync" "time" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" @@ -100,25 +98,6 @@ const actsInController = "controller" // test generated from it holds every row to a condition kind the mesh raises, a budget, a brake and its // event (healers_test.go). var healerRegistry = []healerRow{ - {ID: "H1", Kinds: []string{"sent-not-reported"}, - Condition: "a machine was sent a declaration and has not reported it (S2)", - Repair: "ask the machine's node-engine to say again what it last applied (the `report` verb); if what " + - "comes back is not the declaration it was sent, or nothing comes, send it its current declaration " + - "again — what a push of that one machine does, never moving a build a policy or a plan holds back", - Then: "resolver operator, urgent", From: "a push by hand to unstick a plan waiting on a report (230, 257, 264, 267)", - Budget: 2, Window: 6 * time.Hour, Settle: 3 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed, - applies: appliesToAMachine, repair: repairReport}, - {ID: "H2", Kinds: []string{"stalled"}, - Condition: "a plan stalled on a wait that is superseded — a newer plan of its repository and branch " + - "exists — or already finished — every module of every tier built or failed, and every one that " + - "rolls out sent (S3); or a delivery held past its state's bound where mesh-delivery's table lets H2 " + - "take a transition (D14, novox/hq ADR 0239)", - Repair: "close the plan with its note, as `plans close` does: superseded, naming the newer plan, or " + - "done; what it asked still builds and registers — or, for a delivery, mesh-delivery's `close`, which " + - "reads its walk again and takes only the transition its table names", - Then: "resolver operator, urgent", From: "a stuck plan closed by hand (214, 254)", - Budget: 1, Window: 24 * time.Hour, Settle: 2 * time.Minute, ActsIn: actsInController, Event: link.KeyHealerActed, - applies: appliesToAStalePlan, repair: repairPlan}, {ID: "H3", Kinds: []string{"holder-silent", "consumer-lost"}, Condition: "a seat's holder that does not answer (D3), or a durable consumer the mesh expects and the " + "bus does not hold (D6, S9)", @@ -181,12 +160,8 @@ type healing struct { // The acts, as seams a test replaces: asking a machine to report, sending it again, asserting the // bus's objects, resetting a consumer. - askReport func(ctx context.Context, node string) error - sendAgain func(ctx context.Context, node string) error assertObjects func(ctx context.Context) error resetConsumer func(stream, name string) (string, error) - // reportWait is how long H1 waits for the machine's report after asking. - reportWait time.Duration mu sync.Mutex // declined is why each condition was last passed over, so it is said once and `healers` can show it. @@ -199,26 +174,7 @@ func newHealing(open *stores, keeper *conditions.Keeper, teller conditions.Telle h := &healing{open: open, keeper: keeper, teller: teller, js: js, acting: link.Holding, epoch: func(ctx context.Context) (uint64, error) { return theLease.epoch(ctx) }, now: time.Now, say: func(format string, args ...any) { fmt.Printf(format+"\n", args...) }, - reportWait: 45 * time.Second, declined: map[string]string{}} - h.askReport = func(ctx context.Context, node string) error { - if h.js == nil { - return errors.New("this controller is not on the bus") - } - return askToReport(ctx, h.js.Conn(), node) - } - h.sendAgain = func(ctx context.Context, node string) error { - // **Never what a policy or a plan holds back** (ADR 0221): a send by the mesh itself is a push - // that did not name the machine — a person's word sends a held build, a healer's does not. - held, err := heldMachines(ctx, open, []string{node}) - if err != nil { - return err - } - if why := held[node]; len(why) > 0 { - return fmt.Errorf("not sent again: it would move what a policy or a plan holds back (ADR 0221) — %s; "+ - "`push %s` sends it, on a person's word", strings.Join(why, "; "), node) - } - return sendTo(ctx, open, []string{node}) - } + declined: map[string]string{}} h.assertObjects = func(ctx context.Context) error { if h.js == nil { return errors.New("this controller is not on the bus") @@ -239,21 +195,6 @@ func newHealing(open *stores, keeper *conditions.Keeper, teller conditions.Telle return h } -// askToReport asks one machine's node-engine to say again what it last applied (to-be 45 §6). On core -// NATS, fired and flushed: the answer is the machine's ordinary report, read from the store. -func askToReport(ctx context.Context, conn *nats.Conn, node string) error { - body, err := json.Marshal(map[string]any{"asked": time.Now().UTC(), "by": "the controller's healer H1"}) - if err != nil { - return err - } - if err := conn.Publish(broker.AskReportSubject(node), body); err != nil { - return err - } - flushing, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - return conn.FlushWithContext(flushing) -} - // keep runs the healers until ctx ends. func (h *healing) keep(ctx context.Context) { tick := time.NewTicker(healEvery) @@ -537,184 +478,6 @@ func brakeObservation(acts []inventory.Heal, said string) conditions.Observation Said: strings.Join(most, ", ")} } -// --- H1 ---------------------------------------------------------------------------------------------- - -// appliesToAMachine is H1's: a machine named, its budget counted against the condition. -func appliesToAMachine(_ context.Context, _ *healing, c conditions.Condition) (string, bool, string, error) { - if c.Subject.Machine == "" { - return "", false, "it names no machine", nil - } - return c.Key, true, "", nil -} - -// repairReport is H1: ask the machine to report; if what comes back is not what it was sent, or nothing -// comes, send it again. -func repairReport(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) { - node := c.Subject.Machine - inv := h.open.inventory - // The store's clock, as the report's time is: not the runner's, which a test moves by hand. - asked := time.Now() - askErr := h.askReport(ctx, node) - current, heard := false, false - if askErr == nil { - deadline := time.Now().Add(h.reportWait) - for { - r, found, err := lastReportOf(ctx, inv, node) - if err != nil { - return "", "", err - } - if found && r.At != nil && r.At.After(asked) { - heard, current = true, r.Current - if current { - break - } - } - if time.Now().After(deadline) { - break - } - select { - case <-ctx.Done(): - return "", "", ctx.Err() - case <-time.After(min(2*time.Second, h.reportWait/4+time.Millisecond)): - } - } - } - if current { - return "asked " + node + "'s node-engine to say again what it last applied", - "it reported the declaration it was sent: its report had not reached the mesh", nil - } - why := "it said nothing within " + h.reportWait.String() + " — its node-engine may be older than the report verb" - switch { - case askErr != nil: - why = "it could not be asked: " + askErr.Error() - case heard: - why = "it reported a declaration other than the one it was sent" - } - if err := h.sendAgain(ctx, node); err != nil { - return "asked " + node + " to report, then sent it its current declaration again", why + "; the send failed", - err - } - return "asked " + node + " to report, then sent it its current declaration again", why + "; sent again", nil -} - -// lastReportOf is one machine's last report beside its last send. -func lastReportOf(ctx context.Context, inv *inventory.Inventory, node string) (inventory.Reported, bool, error) { - reports, err := inv.LastReports(ctx) - if err != nil { - return inventory.Reported{}, false, err - } - for _, r := range reports { - if r.Node == node { - return r, true, nil - } - } - return inventory.Reported{}, false, nil -} - -// --- H2 ---------------------------------------------------------------------------------------------- - -// planStale is why a plan's wait is superseded or finished, and the state closing it leaves it in; empty -// when it is neither, which is not H2's to repair. -func planStale(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (state, why string, err error) { - if p.Release != nil { - return "", "", nil // a release plan walks machines, and its own gate says when it is done (ADR 0236) - } - recent, err := inv.RecentPlans(ctx, 50) - if err != nil { - return "", "", err - } - for _, newer := range recent { - if newer.ID == p.ID || !newer.Created.After(p.Created) || !repositoryMatches(newer.Repository, p.Repository) || - newer.Branch != p.Branch || newer.State == inventory.PlanSuperseded || newer.Batch() { - continue - } - return inventory.PlanSuperseded, fmt.Sprintf("superseded by %s (%s %s), a newer merge of the same repository "+ - "and branch", newer.ID, newer.Repository, short(newer.Commit)), nil - } - for _, tier := range p.Tiers { - for _, m := range tier { - s := p.Modules[m] - if s == nil || (s.State != "built" && s.State != "failed") { - return "", "", nil - } - if s.State == "built" && s.SentAt == nil { - u, err := inv.UpgradeOf(ctx, m) - if err != nil { - return "", "", err - } - // A recorded module whose move goes out on a machine is still the walk's to send (novox/hq ADR 0377). - if u.RollOut || len(wentOut(s.Recorded)) > 0 { - return "", "", nil - } - } - } - } - return inventory.PlanDone, "finished: every module of every tier is built or failed, and every one that rolls " + - "out was sent — nothing is left to wait on", nil -} - -// appliesToAStalePlan is H2's: the plan is open, and its wait is superseded or finished. -func appliesToAStalePlan(ctx context.Context, h *healing, c conditions.Condition) (string, bool, string, error) { - if c.Subject.Scope == conditions.ScopeDelivery { - return appliesToAStalledDelivery(ctx, h, c) - } - p, err := h.open.inventory.PlanByID(ctx, c.Subject.ID) - if err != nil { - return "", false, "", err - } - if !p.Open() { - return "", false, "the plan is " + p.State + " already: its condition clears on the next look", nil - } - state, _, err := planStale(ctx, h.open.inventory, p) - if err != nil { - return "", false, "", err - } - if state == "" { - return "", false, "its wait is neither superseded nor finished: it waits on something still to come, " + - "which its own signal says", nil - } - return c.Key, true, "", nil -} - -// repairPlan is H2: the plan closed with its note, under the plans' hold, by compare-and-set. -func repairPlan(ctx context.Context, h *healing, c conditions.Condition) (string, string, error) { - if c.Subject.Scope == conditions.ScopeDelivery { - return repairDelivery(ctx, h, c) - } - inv := h.open.inventory - release, err := inv.HoldPlans(ctx, true) - if err != nil { - return "", "", err - } - defer release() - p, err := inv.PlanByID(ctx, c.Subject.ID) - if err != nil { - return "", "", err - } - if !p.Open() { - return "closed nothing", "the plan is " + p.State + " already", nil - } - state, why, err := planStale(ctx, inv, p) - if err != nil { - return "", "", err - } - if state == "" { - return "closed nothing", "its wait is no longer superseded or finished", nil - } - p.State = state - p.Note = fmt.Sprintf("closed by healer H2 at tier %d: %s", p.Tier, why) - if state != inventory.PlanDone { - sayUnsent(&p, func(m string) bool { - u, err := inv.UpgradeOf(ctx, m) - return err == nil && u.RollOut - }) - } - if err := inv.SavePlan(ctx, &p); err != nil { - return "", "", err - } - return "closed the plan " + p.ID + " (" + state + ")", why, nil -} - // --- H3 ---------------------------------------------------------------------------------------------- // appliesToAnObject is H3's: every holder or consumer the probe or the bus names, its budget per object. diff --git a/cmd/mesh-controller/healers_integration_test.go b/cmd/mesh-controller/healers_integration_test.go index 3a55294d..1d0f476b 100644 --- a/cmd/mesh-controller/healers_integration_test.go +++ b/cmd/mesh-controller/healers_integration_test.go @@ -5,7 +5,6 @@ package main import ( "context" "errors" - "slices" "strconv" "strings" "testing" @@ -18,94 +17,6 @@ import ( "github.com/novox/mesh-controller/internal/testbus" ) -func TestH1AsksAMachineToReportAndSendsItAgain(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - h, told, clock := healingOn(t, open) - record, err := open.inventory.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordSent(ctx, record.ID, "d2", nil); err != nil { - t.Fatal(err) - } - key := sentNotReported(t, "laptop") - - // First: the report had not reached the mesh, and asking for it brings it. - asked := 0 - h.askReport = func(ctx context.Context, node string) error { - asked++ - if node != "laptop" { - t.Errorf("asked %s", node) - } - _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeApplied, - At: time.Now().Add(time.Second), Declared: "d2"}) - return err - } - h.tick(ctx) - c, found, err := conditionsFrom.Get(ctx, key) - if err != nil || !found { - t.Fatalf("the condition is gone after the act — a healer cleared it, not an observation: %v", err) - } - if asked != 1 || len(c.Tried) != 1 || c.Tried[0].By != "healer H1" || !strings.Contains(c.Tried[0].Outcome, "acted") || - c.Resolver != "healer:H1" { - t.Fatalf("after the first act: asked %d, %+v", asked, c) - } - if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H1" || acts[0].Outcome != inventory.HealActed || - acts[0].Condition != key || acts[0].By != "healer H1" || acts[0].Epoch != 57 { - t.Fatalf("the act was not said as healer-acted: %+v", acts) - } - - // Within its settle, nothing more: the observation has its turn. - clock.pass(time.Minute) - h.tick(ctx) - if asked != 1 { - t.Fatalf("acted again inside the settle: asked %d", asked) - } - - // Second: the machine says nothing, so it is sent again. - clock.pass(3 * time.Minute) - sent := 0 - h.askReport = func(context.Context, string) error { asked++; return nil } - h.sendAgain = func(_ context.Context, node string) error { sent++; return nil } - h.tick(ctx) - if asked != 2 || sent != 1 { - t.Fatalf("the second act: asked %d, sent %d", asked, sent) - } - c, _, _ = conditionsFrom.Get(ctx, key) - if len(c.Tried) != 2 || !strings.Contains(c.Tried[1].Outcome, "sent again") || !strings.Contains(c.Tried[1].Outcome, "act 2 of 2") { - t.Fatalf("tried %+v", c.Tried) - } - - // The budget is spent: the operator's, urgent, said; and no healer touches it again. - clock.pass(4 * time.Minute) - h.tick(ctx) - c, _, _ = conditionsFrom.Get(ctx, key) - if !c.Escalated() || c.Severity != conditions.Urgent || len(c.Tried) != 3 || - !strings.Contains(c.Tried[2].Outcome, "budget of 2") { - t.Fatalf("not handed to the operator: %+v", c) - } - if acts := told.said(); len(acts) != 3 || acts[2].Outcome != inventory.HealEscalated { - t.Fatalf("the escalation was not said: %+v", acts) - } - clock.pass(time.Hour) - h.tick(ctx) - if asked != 2 || sent != 1 || len(told.said()) != 3 { - t.Fatalf("a healer acted on a condition the operator holds: asked %d sent %d said %d", asked, sent, len(told.said())) - } - heals, err := open.inventory.HealsSince(ctx, time.Now().Add(-time.Hour)) - if err != nil || len(heals) != 3 || heals[0].Outcome != inventory.HealActed || heals[1].Outcome != inventory.HealActed || - heals[2].Outcome != inventory.HealEscalated || heals[0].Epoch != 57 { - t.Fatalf("the heals kept: %+v %v", heals, err) - } - // And a heal is not a hand act: what S15 counts never sees it. - for _, x := range heals { - if x.Healer == "" || strings.HasPrefix(x.Act, "hand-act") { - t.Errorf("a heal reads as a hand act: %+v", x) - } - } -} - func TestNoHealerActsWithoutTheLease(t *testing.T) { open := aMesh(t) ctx := t.Context() @@ -123,93 +34,6 @@ func TestNoHealerActsWithoutTheLease(t *testing.T) { } } -func TestTheBrakeStopsEveryHealerAndSaysSo(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - h, told, clock := healingOn(t, open) - for i := 0; i < healBrakeLimit; i++ { - if _, err := open.inventory.BeginHeal(ctx, inventory.Heal{Healer: "H3", ConditionKey: "seat.x.anchor.silent", - Kind: "holder-silent", Act: "asserted", Outcome: inventory.HealActed, - At: clock.now().Add(-time.Duration(healBrakeLimit-i) * time.Minute)}); err != nil { - t.Fatal(err) - } - } - sentNotReported(t, "laptop") - h.tick(ctx) // the fakes fail the test if anything acts - braked, found, err := conditionsFrom.Get(ctx, "mesh.healers.braked") - if err != nil || !found || braked.Severity != conditions.Urgent || !strings.Contains(braked.Evidence[0].Said, "H3 on seat.x.anchor.silent ×12") { - t.Fatalf("the brake was not said: %+v %v", braked, err) - } - if len(told.said()) != 0 { - t.Fatalf("a healer acted under the brake: %+v", told.said()) - } - // Past the hour of the first act, still held: it lets go an hour after the last. - clock.pass(30 * time.Minute) - h.tick(ctx) - if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); !held { - t.Fatal("the brake let go before an hour had passed since the last act") - } - clock.pass(31 * time.Minute) - asked := 0 - h.askReport = func(context.Context, string) error { asked++; return nil } - h.sendAgain = func(context.Context, string) error { return nil } - h.tick(ctx) - if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); held { - t.Fatal("the brake held an hour after the last act") - } - if asked != 1 { - t.Fatalf("the healers did not act again after the brake let go: asked %d", asked) - } -} - -func TestH2ClosesAPlanAnotherHasTakenOver(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - h, told, _ := healingOn(t, open) - created := time.Now().UTC().Add(-time.Hour) - older := inventory.Plan{ID: "plan-old", Repository: "novox/app", Branch: "main", Commit: "0ld0ld0", Created: created, - State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "asked"}}} - waiting := inventory.Plan{ID: "plan-other", Repository: "novox/other", Branch: "main", Commit: "07he707", Created: created, - State: inventory.PlanBuilding, Tiers: [][]string{{"b"}}, Modules: map[string]*inventory.PlanModule{"b": {State: "asked"}}} - newer := inventory.Plan{ID: "plan-new", Repository: "novox/app", Branch: "main", Commit: "new0new", Created: created.Add(time.Minute), - State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}} - for _, p := range []*inventory.Plan{&older, &waiting, &newer} { - if err := open.inventory.SavePlan(ctx, p); err != nil { - t.Fatal(err) - } - } - for _, id := range []string{"plan-old", "plan-other"} { - if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopePlan, ID: id, Kind: "stalled", - Severity: conditions.Warning, Summary: id + " is stalled", Source: "S3"}); err != nil { - t.Fatal(err) - } - } - h.tick(ctx) - closed, err := open.inventory.PlanByID(ctx, "plan-old") - if err != nil || closed.State != inventory.PlanSuperseded || !strings.Contains(closed.Note, "closed by healer H2") || - !strings.Contains(closed.Note, "plan-new") { - t.Fatalf("the superseded plan: %+v %v", closed, err) - } - if other, _ := open.inventory.PlanByID(ctx, "plan-other"); other.State != inventory.PlanBuilding { - t.Fatalf("a plan still waiting was closed: %+v", other) - } - if c, _, _ := conditionsFrom.Get(ctx, "plan.plan-other.stalled"); len(c.Tried) != 0 || c.Resolver != conditions.ResolverSelf { - t.Fatalf("a plan H2 does not repair was touched: %+v", c) - } - if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "plan.plan-old.stalled" { - t.Fatalf("said %+v", acts) - } - // Finished: every module built, none rolled out unsent — closed as done. - done := inventory.Plan{ID: "plan-done", Repository: "novox/third", Branch: "main", Commit: "d0ned0n", Created: created, - State: inventory.PlanRolling, Tier: 0, Tiers: [][]string{{"c"}}, Modules: map[string]*inventory.PlanModule{"c": {State: "built"}}} - if err := open.inventory.SavePlan(ctx, &done); err != nil { - t.Fatal(err) - } - if state, why, err := planStale(ctx, open.inventory, done); err != nil || state != inventory.PlanDone || !strings.Contains(why, "finished") { - t.Fatalf("a finished plan reads %q %q %v", state, why, err) - } -} - func TestH4ResetsOnlyWhatTheTableMarksResettable(t *testing.T) { open := aMesh(t) ctx := t.Context() @@ -422,28 +246,3 @@ func TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers(t *testing.T } } } - -func TestNatsAskToReportReachesTheMachine(t *testing.T) { - url := testbus.URL(t) - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - asked, err := conn.SubscribeSync(broker.AskReportSubject("laptop")) - if err != nil { - t.Fatal(err) - } - if err := askToReport(t.Context(), conn, "laptop"); err != nil { - t.Fatal(err) - } - msg, err := asked.NextMsg(5 * time.Second) - if err != nil || !strings.Contains(string(msg.Data), "healer H1") { - t.Fatalf("the machine heard %v, %v", msg, err) - } - perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindNode, Node: "laptop", PasswordHash: "x"}) - if err != nil || !slices.Contains(perms.Subscribe, "mesh.node.laptop.ask.report") || - slices.Contains(perms.Subscribe, "mesh.node.anchor.ask.report") { - t.Fatalf("a machine's grant for the question: %v %v", perms.Subscribe, err) - } -} diff --git a/cmd/mesh-controller/healers_test.go b/cmd/mesh-controller/healers_test.go index 11829431..bd35774e 100644 --- a/cmd/mesh-controller/healers_test.go +++ b/cmd/mesh-controller/healers_test.go @@ -42,8 +42,6 @@ func raisedKinds() map[string]string { // inducedFailures are the healers seen acting in this file, by id: a row without one fails. var inducedFailures = map[string]string{ - "H1": "TestH1AsksAMachineToReportAndSendsItAgain", - "H2": "TestH2ClosesAPlanAnotherHasTakenOver", "H3": "TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget", "H4": "TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers", } @@ -170,10 +168,8 @@ func healingOn(t *testing.T, open *stores) (*healing, *heard, *testClock) { open.inventory.ActsUnder(func(context.Context) (uint64, error) { return 57, nil }) h := &healing{open: open, keeper: conditionsFrom, teller: told, epoch: func(context.Context) (uint64, error) { return 57, nil }, now: clock.now, - say: func(f string, a ...any) { t.Logf(f, a...) }, reportWait: 300 * time.Millisecond, + say: func(f string, a ...any) { t.Logf(f, a...) }, declined: map[string]string{}} - h.askReport = func(context.Context, string) error { t.Error("asked a machine to report"); return nil } - h.sendAgain = func(context.Context, string) error { t.Error("sent a machine again"); return nil } h.assertObjects = func(context.Context) error { t.Error("asserted the bus's objects"); return nil } h.resetConsumer = func(string, string) (string, error) { t.Error("reset a consumer"); return "", nil } return h, told, clock diff --git a/cmd/mesh-controller/health_probes.go b/cmd/mesh-controller/health_probes.go index d9cc994c..8d2a2887 100644 --- a/cmd/mesh-controller/health_probes.go +++ b/cmd/mesh-controller/health_probes.go @@ -10,10 +10,8 @@ import ( "github.com/nats-io/nats.go" - "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/lease" ) @@ -60,95 +58,6 @@ func probeControllerHealth(ctx context.Context, d *doctor) ([]conditions.Observa return nil, nil } -// probeEngineHealth is H-engine: every machine heard from has reported its current declaration — the -// last one it was sent — under a node-engine build the mesh delivered, or is inside the bound of a send. -func probeEngineHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) { - return machinesHealth(ctx, d, hostModule) -} - -// probeToolsHealth is H-tools: every machine heard from that is assigned the node tools has them -// announced, answering the bus's discovery. -func probeToolsHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) { - return machinesHealth(ctx, d, broker.RuntimeModule) -} - -// machinesHealth judges a component on every machine running it and heard from, by its health -// definition, as the gate does — with no condition taken as the build's doing. -func machinesHealth(ctx context.Context, d *doctor, component string) ([]conditions.Observation, error) { - inv := d.open.inventory - running, err := inv.Running(ctx, component) - if err != nil { - return nil, err - } - shelf, err := inv.Catalogue(ctx) - if err != nil { - return nil, err - } - f, err := gatherGateFacts(ctx, d.open, coreComponent(component)) - if err != nil { - return nil, err - } - f.judged = false - heard := heardMachines(d) - var out []conditions.Observation - for _, node := range running { - if !heard[node] { - continue // a machine not heard from is S1's - } - if r, said := f.reports[node]; said && !r.Current && r.Sent != nil && time.Since(*r.Sent) < gateBound { - continue // inside the bound of a send: S2's, and the gate's - } - // What the machine did with what it was sent is not the component's health: the node-engine's is - // that it reported its current declaration at all, the node tools' that they answer. - if r := f.reports[node]; r.Current || component == broker.RuntimeModule { - now := time.Now() - r.Current, r.Outcome = true, inventory.OutcomeApplied - if r.At == nil { - r.At = &now - } - f.reports[node] = r - } - if component == hostModule { - // A node-engine reporting a build the mesh delivered, current or previous, is the version - // split's (D10), not ill health; a build the mesh did not deliver is. - f.engines[node] = deliveredOr(shelf[component], f.engines[node]) - } - h, why := judgeHealth(component, coreComponent(component), shelf[component], node, time.Time{}, f) - if h == healthGood { - continue - } - out = append(out, conditions.Observation{Scope: conditions.ScopeCore, ID: coreComponent(component) + "." + node, - Token: "unhealthy", Kind: kindCoreUnhealthy, Machine: node, Severity: conditions.Warning, - Summary: fmt.Sprintf("the %s on %s is not healthy: %s", componentWords(component), node, why)}) - } - return sortedFound(out), nil -} - -// deliveredOr is the reported engine build, or the current delivered one when the report names any -// build at all: what H-engine judges is that it reports, not which. -func deliveredOr(m catalogue.Manifest, reported string) string { - if reported == "" { - return reported - } - if v := deliveredVersions(m); len(v) > 0 { - return v[0] - } - return reported -} - -// componentWords is a core component as a sentence names it. -func componentWords(component string) string { - switch component { - case hostModule: - return "node-engine" - case broker.RuntimeModule: - return "node tools" - case catalogue.ControllerSeatName: - return "controller" - } - return component -} - // probeBusHealth is H-bus: every stream and durable consumer the mesh defines is on the bus, and a // request crosses it to every machine's node tools and back. func probeBusHealth(ctx context.Context, d *doctor) ([]conditions.Observation, error) { diff --git a/cmd/mesh-controller/held_back.go b/cmd/mesh-controller/held_back.go deleted file mode 100644 index 16ffc4d5..00000000 --- a/cmd/mesh-controller/held_back.go +++ /dev/null @@ -1,283 +0,0 @@ -package main - -import ( - "context" - "fmt" - "io" - "sort" - "strings" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// A push sends no build a policy or a plan holds back, except to the machine it names (novox/hq -// issue 259, ADR 0221). -// -// A named push ends by sending every other machine whose declaration differs from what it was last -// sent (ADR 0083), so that a grant the push's work minted reaches the provider in the same act. A -// digest cannot say why a machine differs. A module whose upgrade policy records rather than rolls -// out makes every machine running it differ from the merge on, and so did a module whose plan was -// still waiting on its first machine (ADR 0218): `push ` sent all four machines the build -// that was meant to be walked through the mesh one machine at a time, and a fault in it was met -// everywhere at once. -// -// What tells the two apart is which build of each module the machine was last sent, kept with every -// send. A machine any of whose modules would move to a build its policy or an open plan holds back -// is not sent by a push that did not name it; the push says which, and why, and how to send it. - -// heldBack is why a push that did not name a machine must not send it, empty when it may. -// -// `modules` is what the machine would be sent now; `sent` and `known` what it was last sent, as -// Inventory.SentBuilds answers. A module moves when the build it would carry is not the one the -// machine was last sent — including a module the machine was never sent at all. A move is held when -// the module's policy records rather than rolls out, or when an open plan has not yet sent this -// machine (planStillToSend). A machine whose last send was not recorded is held whole: what it carried -// is not known, so a held upgrade cannot be told from anything else. -func heldBack(node string, modules []string, sent map[string]string, known bool, - current map[string]inventory.CurrentBuild, plans []inventory.Plan) []string { - if !known { - return []string{"which builds it was last sent is not known — it was last sent before the " + - "mesh kept them, or sent a declaration by hand"} - } - var why []string - for _, m := range modules { - now := current[m] - was, carried := sent[m] - if carried && was == now.Commit { - continue - } - move := fmt.Sprintf("%s would move %sto %s", m, fromBuild(was, carried), buildName(now.Commit)) - if !now.RollOut { - why = append(why, move+", which its upgrade policy records rather than rolls out") - continue - } - if id := planStillToSend(plans, m, node); id != "" { - why = append(why, move+", which "+id+" has not sent it yet (one machine first)") - } - } - sort.Strings(why) - return why -} - -// planStillToSend is the open plan that has a module's new build still to send this machine, or empty: -// one holding the module that has neither finished sending it nor sent it here first, and has not -// failed it (novox/hq ADR 0218). The same reading rolledOutByAPlan makes for the whole module, made -// per machine. -func planStillToSend(plans []inventory.Plan, module, node string) string { - for _, p := range plans { - s, holds := p.Modules[module] - if !p.Open() || !holds { - continue - } - if s == nil { - return p.ID - } - if s.SentAt != nil || s.State == "failed" { - continue - } - first := false - for _, n := range s.First { - if n == node { - first = true - } - } - if !first { - return p.ID - } - } - return "" -} - -func fromBuild(was string, carried bool) string { - if !carried { - return "(never sent it) " - } - return "from " + buildName(was) + " " -} - -func buildName(commit string) string { - if commit == "" { - return "a build with no source" - } - return shortCommit(commit) -} - -// heldMachines reads, for each machine named, why a push that did not name it must not send it -// (heldBack), and answers only the machines held. A machine whose set cannot be worked out is left -// to the send, which says why. -func heldMachines(ctx context.Context, open *stores, names []string) (map[string][]string, error) { - // One reading of the mesh for every machine this looks at. - ctx = withInputCache(ctx) - out := map[string][]string{} - if len(names) == 0 { - return out, nil - } - inv := open.inventory - current, err := inv.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - plans, err := inv.OpenPlans(ctx) - if err != nil { - return nil, err - } - f, err := readMoveFacts(ctx, inv) - if err != nil { - return nil, err - } - for _, node := range names { - plan, _, err := resolveNode(ctx, open, node) - if err != nil { - continue - } - modules := make([]string, 0, len(plan.Modules)) - for _, m := range plan.Modules { - modules = append(modules, m.Module) - } - sent, known, err := inv.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - // A rebuild that put the same thing on the machine is no move (ADR 0236): read as the build it - // runs. - same := map[string]string{} - for m, was := range sent { - same[m] = was - if f.identical(m, was, current[m].Commit) { - same[m] = current[m].Commit - } - } - // A move under a module's own `record` that recreates nothing and moves no data goes as a module that rolls - // out goes (novox/hq ADR 0377): a walk's to send, not a person's. - here, copied := current, false - for m, was := range same { - if now, held := current[m]; held && now.OwnRecord() && !f.identical(m, was, now.Commit) && f.rolls(m, was) { - if !copied { - copied = true - here = make(map[string]inventory.CurrentBuild, len(current)) - for k, v := range current { - here[k] = v - } - } - now.RollOut = true - here[m] = now - } - } - why := heldBack(node, modules, same, known, here, plans) - // And a build no gate has seen is not carried by a send that does not judge it (ADR 0236). - for _, mv := range f.moves(node, modules, same, known, false) { - if planStillToSend(plans, mv.Module, node) == "" { - why = append(why, fmt.Sprintf("%s would move from %s to %s, which has passed no gate yet — a release "+ - "plan sends it, one machine first and then the rest at once, judged", mv.Module, buildName(mv.From), buildName(mv.To))) - } - } - if len(why) > 0 { - sort.Strings(why) - out[node] = why - } - } - return out, nil -} - -// sayHeld is what a push says about a machine it left behind on purpose: that it is behind, why it -// was not sent, that whatever else it is owed waits with it, and the command that sends it. -func sayHeld(w io.Writer, node string, why []string) { - fmt.Fprintf(w, "\n%s is behind and was not sent: %s. A push sends no build a policy or a plan "+ - "holds back to a machine it did not name (novox/hq ADR 0221), so anything else it is owed — a "+ - "grant from this push among it — waits with it. `push %s` sends it\n", - node, strings.Join(why, "; "), node) -} - -// flushBehind is the end of a named push: every other machine now behind is sent too, by name, over -// as many rounds as the sends take to settle (novox/hq issue 057, ADR 0083) — except a machine whose -// modules would move to a build a policy or a plan holds back, which is named and left (ADR 0221). -// -// `handled` is every machine already sent or already said; it is not considered again. Answers the -// machines that could not be composed, as refusals. -func flushBehind(ctx context.Context, open *stores, nodes []inventory.Node, handled map[string]bool, - compose func(held context.Context, node string) (sendable, error), d delivery, holder string, - keep keeping, w io.Writer) ([]string, error) { - inv := open.inventory - var refusals []string - // Bounded by the node count: a node is marked handled the round it is considered and is never - // considered twice, so the loop cannot run more than len(nodes) rounds. The bound is a guard - // against a logic error, not a real limit — if it were ever hit, that is a bug rather than a - // cascade legitimately still converging, so it is said rather than passed over in silence. - rounds := 0 - for { - would, err := wouldSend(ctx, open, nodes) - if err != nil { - return refusals, err - } - behind, err := inv.Waiting(ctx, would) - if err != nil { - return refusals, err - } - var also []string - for _, m := range behind { - if !handled[m.Node] { - also = append(also, m.Node) - } - } - if len(also) == 0 { - return refusals, nil - } - if rounds++; rounds > len(nodes) { - fmt.Fprintf(w, "\nstopped cascading after %d rounds with %s still behind — this "+ - "should not happen; run `push --behind` to finish\n", - rounds-1, strings.Join(also, ", ")) - return refusals, nil - } - sort.Strings(also) - held, err := heldMachines(ctx, open, also) - if err != nil { - return refusals, err - } - var sending []string - for _, name := range also { - // Every candidate this round is marked handled — the sent ones so they are not - // re-listed, the held ones because they stay held, and the refused ones so a machine - // that cannot be composed does not make the loop spin on it for ever. - handled[name] = true - if why, isHeld := held[name]; isHeld { - sayHeld(w, name, why) - continue - } - sending = append(sending, name) - } - if len(sending) == 0 { - continue - } - fmt.Fprintf(w, "\nthis push left %s behind — a provision granted from there, or a "+ - "declaration since changed; sending it too\n", strings.Join(sending, ", ")) - // Tolerantly, exactly as the named send: a machine that cannot be composed is collected as - // a refusal and reported at the end, and the others are still sent (novox/hq ADR 0066). - // Held for this round only, and after the last round's were given back, so two pushes - // cascading into each other's machines never each wait on the other. - refused, err := sendRound(ctx, open, sending, compose, d, holder, keep) - refusals = append(refusals, refused...) - if err != nil { - return refusals, err - } - } -} - -// composeForPush is how a push composes one machine: its set resolved, what it cannot host and what -// is left out of it said, and its declaration allocated. -func composeForPush(open *stores) func(held context.Context, node string) (sendable, error) { - return func(held context.Context, node string) (sendable, error) { - // Read afresh for each machine: the send before it made and recorded things. - held = freshInput(held) - plan, _, err := resolveNode(held, open, node) - if err != nil { - return sendable{}, err - } - reportUnhostable(node, plan) - reportKept(held, plan) - declared, err := composeSend(held, open, node) - if err == nil { - reportLeftOut(node, declared) - } - return declared, err - } -} diff --git a/cmd/mesh-controller/held_back_integration_test.go b/cmd/mesh-controller/held_back_integration_test.go deleted file mode 100644 index dc0b3d7d..00000000 --- a/cmd/mesh-controller/held_back_integration_test.go +++ /dev/null @@ -1,174 +0,0 @@ -//go:build integration - -package main - -import ( - "bytes" - "reflect" - "slices" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := time.Now().Add(-time.Hour) - aResolver(t, open, "c1c1c1c1c1", asked) - for _, node := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, node, "resolver"); err != nil { - t.Fatal(err) - } - } - // Recorded by a person's choice: the default rolls out since novox/hq ADR 0236. - if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{Why: "each machine checked by hand"}); err != nil { - t.Fatal(err) - } - compose := composeForPush(open) - d := &recordedDelivery{inv: inv} - if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil { - t.Fatal(err) - } - if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" { - t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err) - } - digestOfLaptop := func() string { - sent, err := inv.Outstanding(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - return sent - } - before := digestOfLaptop() - - // The change merges; the policy is record. `push anchor` sends the anchor... - aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute)) - d.declared = nil - if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, "", nil); err != nil { - t.Fatal(err) - } - // ...and its cascade leaves the laptop, saying so. - var said bytes.Buffer - d.declared = nil - refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", nil, &said) - if err != nil || len(refused) != 0 { - t.Fatalf("the cascade failed: %v %v", refused, err) - } - if len(d.declared) != 0 { - t.Fatalf("the cascade sent %v a build its policy records", d.declared) - } - if digestOfLaptop() != before { - t.Fatal("the laptop's last send moved: it was sent the held build") - } - for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2", - "upgrade policy records", "`push laptop` sends it"} { - if !strings.Contains(said.String(), want) { - t.Errorf("the push did not say %q:\n%s", want, said.String()) - } - } - - // Held and owed something else at once — a peer joined: still not sent, and both said: why it - // is held, and that what else it is owed waits with it. - aThirdMachine(t, open) - d.declared = nil - if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil { - t.Fatal(err) - } - d.declared = nil - said.Reset() - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, - compose, d, "", nil, &said); err != nil { - t.Fatal(err) - } - if len(d.declared) != 0 || digestOfLaptop() != before { - t.Fatalf("a held machine owed a consequence was sent: %v", d.declared) - } - if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") { - t.Fatalf("the push did not say both:\n%s", said.String()) - } - - // A policy that rolls out, and a build no gate has seen: still held — a cascade does not judge it - // (novox/hq ADR 0236). - if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil { - t.Fatal(err) - } - said.Reset() - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, - compose, d, "", nil, &said); err != nil { - t.Fatal(err) - } - if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") { - t.Fatalf("a cascade carried a build no gate has seen: %v\n%s", d.declared, said.String()) - } - // Once it passed a gate on some machine, the laptop is a consequence like any other, and sent. - if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: "build-c2", Module: "resolver", Commit: "c2c2c2c2c2", - Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil { - t.Fatal(err) - } - said.Reset() - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, - compose, d, "", nil, &said); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before { - t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String()) - } - if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" { - t.Fatalf("the new send did not record the new build: %v", builds) - } -} - -func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour)) - if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil { - t.Fatal(err) - } - compose := composeForPush(open) - d := &recordedDelivery{inv: inv} - if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil { - t.Fatal(err) - } - - // `push spare`, the machine just placed: the others' peers change with it. - aThirdMachine(t, open) - if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil { - t.Fatal(err) - } - d.declared = nil - var said bytes.Buffer - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) { - t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String()) - } - if strings.Contains(said.String(), "was not sent") { - t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String()) - } - - // A machine whose last send was not recorded — a declaration sent by hand — is held until named. - record, err := inv.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil { - t.Fatal(err) - } - d.declared = nil - said.Reset() - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil { - t.Fatal(err) - } - // The anchor, the hub, may still be settling from the machine placed above; the laptop is the - // question. - if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") { - t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String()) - } -} diff --git a/cmd/mesh-controller/held_back_test.go b/cmd/mesh-controller/held_back_test.go deleted file mode 100644 index 05e7c65a..00000000 --- a/cmd/mesh-controller/held_back_test.go +++ /dev/null @@ -1,168 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/overlay" - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq issue 259, ADR 0221: a push that did not name a machine does not send it a build its -// upgrade policy records rather than rolls out, nor one an open plan has not sent it yet. Anything -// else that moved is still a consequence the push sends (ADR 0083). -func TestAHeldBuildHoldsAMachineANamedPushDidNotName(t *testing.T) { - current := map[string]inventory.CurrentBuild{ - "resolver": {Commit: "c2c2c2c2c2"}, - "agent": {Commit: "a2", RollOut: true}, - "network": {}, - } - modules := []string{"network", "resolver", "agent"} - sent := map[string]string{"network": "", "resolver": "c1c1c1c1c1", "agent": "a2"} - - // A module whose policy records moved: held, naming it, both builds and why. - why := heldBack("laptop", modules, sent, true, current, nil) - if len(why) != 1 || !strings.Contains(why[0], "resolver would move from c1c1c1c1 to c2c2c2c2") || - !strings.Contains(why[0], "upgrade policy records") { - t.Fatalf("a recorded upgrade did not hold the machine: %v", why) - } - - // Nothing moved — what differs is a grant, a peer, a setting: not held (issue 057). - sent["resolver"] = "c2c2c2c2c2" - if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 { - t.Fatalf("a machine whose builds are all current was held: %v", why) - } - - // A module whose policy rolls out moved, and no plan holds it: sent, as before. - sent["agent"] = "a1" - if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 0 { - t.Fatalf("a rolled-out upgrade no plan holds was held: %v", why) - } - - // The last send's builds are not known: held whole. - if why := heldBack("laptop", modules, nil, false, current, nil); len(why) != 1 || - !strings.Contains(why[0], "not known") { - t.Fatalf("a machine whose last send was not recorded was not held: %v", why) - } - - // A module the machine was never sent, under a recording policy: held, and said so. - delete(sent, "resolver") - sent["agent"] = "a2" - if why := heldBack("laptop", modules, sent, true, current, nil); len(why) != 1 || - !strings.Contains(why[0], "resolver would move (never sent it) to c2c2c2c2") { - t.Fatalf("a module never sent under a recording policy: %v", why) - } -} - -// ADR 0218 meets ADR 0083: a plan waiting on its first machine has not sent the rest, and a push -// naming some other machine must not send them for it. -func TestAPlanWaitingOnItsFirstMachineHoldsTheRest(t *testing.T) { - at := time.Now() - current := map[string]inventory.CurrentBuild{"agent": {Commit: "a2", RollOut: true}} - sent := map[string]string{"agent": "a1"} - waiting := []inventory.Plan{{ID: "plan-7", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{ - "agent": {State: "built", First: []string{"ace"}, FirstAt: &at}}}} - - why := heldBack("g14", []string{"agent"}, sent, true, current, waiting) - if len(why) != 1 || !strings.Contains(why[0], "plan-7 has not sent it yet") { - t.Fatalf("a machine the plan has not reached was not held: %v", why) - } - // The first machine itself was sent by the plan: not held by it. - if why := heldBack("ace", []string{"agent"}, sent, true, current, waiting); len(why) != 0 { - t.Fatalf("the plan's first machine was held: %v", why) - } - // Built but not yet sent anywhere, or not yet built: the plan has it still to send. - for what, s := range map[string]*inventory.PlanModule{"built, unsent": {State: "built"}, "unasked": nil} { - plans := []inventory.Plan{{ID: "plan-8", State: inventory.PlanBuilding, - Modules: map[string]*inventory.PlanModule{"agent": s}}} - if why := heldBack("ace", []string{"agent"}, sent, true, current, plans); len(why) != 1 { - t.Errorf("%s: not held: %v", what, why) - } - } - // Sent everywhere, failed, or a plan no longer open: the plan holds nothing back. - for what, plans := range map[string][]inventory.Plan{ - "sent everywhere": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{ - "agent": {State: "built", First: []string{"ace"}, FirstAt: &at, SentAt: &at}}}}, - "failed": {{ID: "p", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{ - "agent": {State: "failed"}}}}, - "closed": {{ID: "p", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{ - "agent": {State: "built"}}}}, - } { - if why := heldBack("g14", []string{"agent"}, sent, true, current, plans); len(why) != 0 { - t.Errorf("%s: held: %v", what, why) - } - } -} - -// recordedDelivery sends nothing and records each send as the mesh does, so the next comparison -// reads the machine as current — and writes down which machines it declared. -type recordedDelivery struct { - inv *inventory.Inventory - declared []string -} - -func (r *recordedDelivery) grant(context.Context, []readyNode) error { return nil } - -func (r *recordedDelivery) declare(ctx context.Context, s readyNode, body []byte) (string, error) { - r.declared = append(r.declared, s.node) - return recordSent(ctx, r.inv, s.node, body, s.declared.Builds, s.declared.Epoch, sentRecordOf(ctx, s.declared)) -} - -// aResolver is a module built from a repository, at a commit, with something on the machine that -// says which build it is. -func aResolver(t *testing.T, open *stores, commit string, asked time.Time) { - t.Helper() - m := catalogue.Manifest{Module: "resolver", Version: "1", Resources: []map[string]any{ - {"id": "zones", "type": "file", "path": "/etc/resolver/zones", "content": "built from " + commit}, - }} - if err := open.inventory.RegisterModule(t.Context(), m, inventory.Source{ - Repository: "novox/mesh-catalog", Path: "modules/resolver", BuiltFrom: commit, Asked: asked}); err != nil { - t.Fatal(err) - } -} - -// A third machine on the private network: once it is sent, every other machine's peers change with -// it, which is a consequence a push must still send — no build moved. -func aThirdMachine(t *testing.T, open *stores) { - t.Helper() - ctx := t.Context() - record, err := open.inventory.AddNode(ctx, "spare") - if err != nil { - t.Fatal(err) - } - if err := open.inventory.SetPlace(ctx, "spare", "spare.example:51820", "here", false, "10.77.0.3"); err != nil { - t.Fatal(err) - } - reported, err := json.Marshal(map[string]any{"capabilities": []map[string]any{ - {"name": "container-runtime", "present": true}, {"name": "wireguard", "present": true}, - {"name": "systemd", "present": true}}}) - if err != nil { - t.Fatal(err) - } - var profile map[string]any - if err := json.Unmarshal(reported, &profile); err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordProfile(ctx, record.ID, profile); err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.Assign(ctx, "spare", overlay.Name); err != nil { - t.Fatal(err) - } -} - -// The issue as it happened, against the real stores: a change merged with the policy `record`, a push -// naming the anchor, and the laptop — running the same module — left with what it had, by name. - -// Issue 057's case is unchanged: a machine whose builds are all current and whose declaration moved -// for another reason is sent by a push that names someone else. diff --git a/cmd/mesh-controller/held_moves.go b/cmd/mesh-controller/held_moves.go deleted file mode 100644 index ce3fe0e6..00000000 --- a/cmd/mesh-controller/held_moves.go +++ /dev/null @@ -1,198 +0,0 @@ -package main - -import ( - "context" - "fmt" - "sort" - "strings" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A move held for a person is asked of the person (novox/hq issue 1732, issue 1320; ADR 0377, ADR 0259). -// -// **A wait only a person can end is said to that person at once.** ADR 0377 holds a recorded module's move that -// recreates a container or touches its data until a person pushes the machine. The walk said so in its note and -// ended "done", `status` named the machine behind, and nobody was asked: on 2026-10-21 mailu's new build waited on -// novox for 40 minutes behind a plan that read "done, 3 tier(s)", and on 2026-10-19 mssql waited on ace the same way. -// Row S24 raises one condition per machine that has such a move, for the operator, with the answer **Release**: the -// push of that machine, which the controller's own `push` verb performs on the operator's warrant (ADR 0259). It ends -// when the machine no longer waits, whoever pushed it. -// -// A hold a person chose is not asked again — a module they set to `record` with a why, and the bus, whose step has -// its own row (S17). - -// kindMovesHeld is S24's kind: machine..moves-held. -const kindMovesHeld = "moves-held" - -// heldFacts is one machine and the moves of recorded modules that wait there for a person. -type heldFacts struct { - node string - moves []heldMove -} - -// gatherHeldMoves is, per machine that holds at least one move no person chose to hold, all its held moves, by machine. -func gatherHeldMoves(ctx context.Context, inv *inventory.Inventory) ([]heldFacts, error) { - nodes, err := inv.Nodes(ctx) - if err != nil { - return nil, fmt.Errorf("the machines cannot be read: %w", err) - } - f, err := readMoveFacts(ctx, inv) - if err != nil { - return nil, err - } - var out []heldFacts - for _, n := range nodes { - moves, err := heldMovesOn(ctx, inv, f, n.Name) - if err != nil { - return nil, err - } - h := heldFacts{node: n.Name, moves: moves} - if len(h.asked()) > 0 { - out = append(out, h) - } - } - return out, nil -} - -// asked are the moves no person chose to hold, and chosen the ones a person did (a module set to record with a why, the bus). -func (h heldFacts) asked() []heldMove { return h.byChoice(false) } -func (h heldFacts) chosen() []heldMove { return h.byChoice(true) } - -func (h heldFacts) byChoice(chosen bool) []heldMove { - var out []heldMove - for _, m := range h.moves { - if m.Chosen == chosen { - out = append(out, m) - } - } - return out -} - -// modules are the held moves' modules, in order, of all of them. -func (h heldFacts) modules() []string { return moduleNames(h.moves) } - -func moduleNames(moves []heldMove) []string { - out := make([]string, 0, len(moves)) - for _, m := range moves { - out = append(out, m.Module) - } - return out -} - -// releaseAction is the one answer a held move offers: the push of the machine, approved by the operator. The push -// names every module whose move waits there as one that may move its data: the operator's Approve is the word -// ADR 0217 asks for, and the ask says so. Offered only where no person's own hold is on the machine too: a push -// composes every recorded module at its current build, so it would send that move as well, which the ask never named -// (ADR 0259 §6), and hold the whole machine if that module's data would move. -func (h heldFacts) releaseAction() conditions.Action { - return conditions.Action{Label: "Release", Verb: "mesh-controller.push", Level: conditions.LevelApprove, - Arguments: map[string]string{"node": h.node, "move": strings.Join(moduleNames(h.asked()), ","), "why": "", - "cause": conditions.CauseOperatorAnswer}} -} - -// heldWords are the plain words of a machine's held moves: which modules wait, why in general, and what an Approve does. -func heldWords(h heldFacts) (headline, explanation, needs string, resolved string) { - names := namesWords(moduleNames(h.asked()), 3) - headline = clipWords(conditions.Capital(names)+" waits for you on "+h.node, conditions.HeadlineMax) - recreates, data := false, false - for _, m := range h.asked() { - switch { - case strings.Contains(m.Why, "recreates"): - recreates = true - default: - data = true - } - } - why := "would change what it runs" - switch { - case recreates && data: - why = "would restart a container or touch its data" - case recreates: - why = "would restart one of its containers" - case data: - why = "would touch its files or data" - } - explanation = clipWords(fmt.Sprintf("The new version of %s %s on %s, so it does not go out with its delivery; "+ - "it waits for you. Release sends %s everything it is owed, these among it, and lets their data move where it "+ - "has to. Nothing is sent until you answer.", names, why, h.node, h.node), conditions.ExplanationMax) - needs = "release them " + FromMeshMCPServer - if chosen := h.chosen(); len(chosen) > 0 { - // A push of the machine would send the move a person chose to hold as well: no answer is offered here. - explanation = clipWords(fmt.Sprintf("The new version of %s %s on %s, so it does not go out with its delivery; "+ - "it waits for you. %s is also held there by your own choice, and a push of %s would send that too, so "+ - "nothing is offered here: the push is yours at the controller's terminal.", - names, why, h.node, conditions.Capital(namesWords(moduleNames(chosen), 3)), h.node), conditions.ExplanationMax) - needs = fmt.Sprintf("push %s at the controller's terminal, once you have decided about %s.", h.node, - namesWords(moduleNames(chosen), 3)) - } - resolved = "Resolved: " + names + " is no longer waiting on " + h.node - return -} - -// watchMovesHeld is S24: a machine that has moves of recorded modules waiting for a person, said to the operator with the -// answer that sends them. The condition ends when the machine no longer waits: it is no longer among the facts. -func watchMovesHeld(f *signalFacts) []conditions.Observation { - var out []conditions.Observation - for _, h := range f.held { - var said []string - for _, m := range h.moves { - said = append(said, fmt.Sprintf("%s %s → %s %s", m.Module, short(m.From), short(m.To), m.Why)) - } - headline, explanation, needs, resolved := heldWords(h) - if c := h.chosen(); len(c) > 0 { - said = append(said, "held by a person's own choice: "+strings.Join(moduleNames(c), ", ")) - } - var actions []conditions.Action - if len(h.chosen()) == 0 { - actions = []conditions.Action{h.releaseAction()} - } - out = append(out, conditions.Observation{Scope: conditions.ScopeMachine, ID: h.node, Kind: kindMovesHeld, - Machine: h.node, Severity: conditions.Warning, Resolver: conditions.ResolverOperator, - Summary: fmt.Sprintf("%s has %d move(s) of recorded modules waiting for a person's push: %s — `push %s` sends them", - h.node, len(h.moves), strings.Join(h.modules(), ", "), h.node), - Said: strings.Join(said, "; "), - Headline: headline, Explanation: explanation, Needs: needs, Resolved: resolved, - Actions: actions}) - } - return out -} - -// planHeld is, for a plan that ended or is running, the moves its modules carried that a machine still holds for a -// person: "module on machine", sorted. The plan's own record says the move waited when it was judged -// (PlanModule.Recorded); the machines say whether it still does, and for the build this plan carried, so a plan older -// than the build now held is not named. -func planHeld(p inventory.Plan, held []heldFacts) []string { - var out []string - for _, h := range held { - for _, m := range h.moves { - s := p.Modules[m.Module] - if s == nil || !strings.HasPrefix(s.Recorded[h.node], "waits:") || !sameCommit(s.Commit, m.To) { - continue - } - out = append(out, m.Module+" on "+h.node) - } - } - sort.Strings(out) - return out -} - -// heldSaid is what a plan's line adds when it carries a move that a machine holds for a person (novox/hq issue 1732): a -// plan whose module waits for a push is not plainly "done". -func heldSaid(p inventory.Plan, held []heldFacts) string { - waiting := planHeld(p, held) - if len(waiting) == 0 { - return "" - } - return fmt.Sprintf(" — %d held for a person: %s (the operator is asked)", len(waiting), strings.Join(waiting, ", ")) -} - -// heldNow reads the machines' held moves for a plan's line, nil when they cannot be read: the line then reads as before. -func heldNow(ctx context.Context, inv *inventory.Inventory) []heldFacts { - held, err := gatherHeldMoves(ctx, inv) - if err != nil { - return nil // empty-on-error: a plan's line then reads as it did before; the watchdog (S24) says it cannot read - } - return held -} diff --git a/cmd/mesh-controller/held_moves_integration_test.go b/cmd/mesh-controller/held_moves_integration_test.go deleted file mode 100644 index 55b65916..00000000 --- a/cmd/mesh-controller/held_moves_integration_test.go +++ /dev/null @@ -1,92 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func inventoryUpgrade(why string) inventory.Upgrade { return inventory.Upgrade{Why: why, By: "jochen"} } - -func mustMoveFacts(t *testing.T, rm *recordedMesh) moveFacts { - t.Helper() - f, err := readMoveFacts(t.Context(), rm.open.inventory) - if err != nil { - t.Fatal(err) - } - return f -} - -// 1732/AC8 (integration): a mesh running mailu at c1 on two machines, whose registered c2 recreates a container, is read -// as two machines each holding mailu's move for a person; a bundle-only c3 on top changes nothing of it; after a machine -// is sent c2 (as a push records it) it is no longer among them, and plans that carried c2 say nothing more of it. -func TestMachinesHoldingAMoveForAPersonAreReadFromTheRecords(t *testing.T) { - rm := aRecordedMesh(t, aMailu(true)) - c2 := aMailu(true) - resourceOf(&c2, "imap")["image"] = "registry.invalid/imap@sha256:2" - rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) - ctx := t.Context() - held, err := gatherHeldMoves(ctx, rm.open.inventory) - if err != nil { - t.Fatal(err) - } - if len(held) != 2 || held[0].node != "anchor" || held[1].node != "laptop" || len(held[0].moves) != 1 || - held[0].moves[0].Module != "mailu" || held[0].moves[0].To != "c2" || !strings.Contains(held[0].moves[0].Why, "recreates") { - t.Fatalf("held %+v", held) - } - if err := rm.open.inventory.RecordSent(ctx, nodeID(t, rm.open, "anchor"), "d-anchor-c2", map[string]string{"mailu": "c2"}); err != nil { - t.Fatal(err) - } - held, err = gatherHeldMoves(ctx, rm.open.inventory) - if err != nil || len(held) != 1 || held[0].node != "laptop" { - t.Fatalf("after anchor was sent c2: %+v (%v)", held, err) - } -} - -// 1732/AC9 (integration): a module a person set to record is no ask: they chose the hold. -func TestAHoldAPersonChoseIsNotAsked(t *testing.T) { - rm := aRecordedMesh(t, aMailu(false)) - c2 := aMailu(false) - c2.Bundles[0].Digest = "sha256:bbb" - rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) - if err := rm.open.inventory.SetUpgradeOf(t.Context(), "mailu", inventoryUpgrade("the migration week")); err != nil { - t.Fatal(err) - } - held, err := gatherHeldMoves(t.Context(), rm.open.inventory) - if err != nil || len(held) != 0 { - t.Fatalf("a machine holding only a person's own hold is asked: %+v (%v)", held, err) - } - if moves, _ := heldMovesOn(t.Context(), rm.open.inventory, mustMoveFacts(t, rm), "anchor"); len(moves) != 1 || !moves[0].Chosen { - t.Fatalf("status still names it, marked as chosen: %+v", moves) - } -} - -// 1742/AC4 (integration): a registered build naming an image of the store that the store lacks is read as a missing -// image on each machine that runs the module; one the store holds is not; and what the store holds is not asked twice. -func TestARegisteredBuildNamingAnImageTheStoreLacksIsRead(t *testing.T) { - m := aMailu(true) - resourceOf(&m, "imap")["image"] = "artifact-store://mailu/imap@sha256:" + strings.Repeat("e", 64) - rm := aRecordedMesh(t, m) - asked := 0 - lacking := func(_ context.Context, reference string) (bool, error) { asked++; return false, nil } - imagesHeld.Lock() - imagesHeld.held = map[string]bool{} - imagesHeld.Unlock() - got, err := gatherMissingImages(t.Context(), rm.open.inventory, lacking) - if err != nil || len(got) != 2 || got[0].Module != "mailu" || got[0].Node != "anchor" || got[1].Node != "laptop" { - t.Fatalf("missing %+v (%v)", got, err) - } - holding := func(_ context.Context, reference string) (bool, error) { asked++; return true, nil } - if got, err := gatherMissingImages(t.Context(), rm.open.inventory, holding); err != nil || len(got) != 0 { - t.Fatalf("an image the store holds is missing: %+v (%v)", got, err) - } - before := asked - if _, err := gatherMissingImages(t.Context(), rm.open.inventory, lacking); err != nil || asked != before { - t.Fatalf("an image the store was found to hold was asked about again (%d, %v)", asked-before, err) - } -} diff --git a/cmd/mesh-controller/held_moves_test.go b/cmd/mesh-controller/held_moves_test.go deleted file mode 100644 index 759837d7..00000000 --- a/cmd/mesh-controller/held_moves_test.go +++ /dev/null @@ -1,237 +0,0 @@ -package main - -import ( - "context" - "errors" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/asks" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// **A move held for a person is asked of the person** (novox/hq issue 1732, issue 1320; ADR 0377, ADR 0259). - -// aHeldMailu is novox holding mailu's new build because it would recreate a container: the walk's own words. -func aHeldMailu() heldFacts { - return heldFacts{node: "novox", moves: []heldMove{{Module: "mailu", From: "b2f5ebb2aaaa", To: "3259765dbbbb", - Why: "waits: recreates 1 of mailu's 12 container(s) (with a new image: automx)"}}} -} - -func heldConditionOn(f *signalFacts) conditions.Condition { - o := watchMovesHeld(f)[0] - return conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline, - Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions} -} - -// 1732/AC1: a machine holding a move of a recorded module for a person raises one condition for the operator, with -// Release as its only answer: the machine's push, approved, naming the modules whose moves wait there. -func TestAHeldMoveIsRaisedWithReleaseAsItsOnlyAction(t *testing.T) { - f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC)) - f.held = []heldFacts{aHeldMailu()} - got := watchMovesHeld(f) - if len(got) != 1 { - t.Fatalf("%+v", got) - } - o := got[0] - if o.Kind != "moves-held" || o.Scope != conditions.ScopeMachine || o.ID != "novox" || o.Severity != conditions.Warning || - o.Resolver != conditions.ResolverOperator || o.Key() != "machine.novox.moves-held" { - t.Fatalf("kind, scope, subject, severity or resolver: %+v (%s)", o, o.Key()) - } - if len(o.Actions) != 1 { - t.Fatalf("actions: %+v", o.Actions) - } - a := o.Actions[0] - if a.Label != "Release" || a.Verb != "mesh-controller.push" || a.Level != conditions.LevelApprove || - a.Arguments["node"] != "novox" || a.Arguments["move"] != "mailu" || a.Arguments["cause"] != conditions.CauseOperatorAnswer { - t.Fatalf("the action is not the push of novox: %+v", a) - } - if _, takes := a.Arguments["why"]; !takes { - t.Fatal("the push takes a why, which the asker fills in with the operator's answer") - } - for _, want := range []string{"mailu", "novox"} { - if !strings.Contains(o.Headline+o.Explanation, want) { - t.Errorf("the words do not name %q: %q / %q", want, o.Headline, o.Explanation) - } - } - if !strings.Contains(o.Said, "recreates 1 of mailu's 12") { - t.Errorf("the evidence does not carry the walk's own words: %q", o.Said) - } - if w := (conditions.Observation{Headline: o.Headline, Explanation: o.Explanation, Needs: o.Needs, Resolved: o.Resolved}); w.Headline == "" || len(o.Headline) > conditions.HeadlineMax { - t.Errorf("headline: %q", o.Headline) - } -} - -// 1732/AC2: a machine with nothing held raises nothing, so the condition ends the look after the move is sent. -func TestAMachineHoldingNothingRaisesNoHeldMoveCondition(t *testing.T) { - f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC)) - if got := watchMovesHeld(f); len(got) != 0 { - t.Fatalf("%+v", got) - } -} - -// 1732/AC3: the condition is asked on a channel at the approve level in words that say what Release does, and the -// operator's warrant calls the controller's own push with the machine and the modules, once. -func TestAHeldMovesReleaseIsAskedAndAWarrantPushesTheMachine(t *testing.T) { - f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC)) - f.held = []heldFacts{aHeldMailu()} - c := heldConditionOn(f) - r := newAskerRig(t) - r.open = []conditions.Condition{c} - if err := r.a.reconcile(context.Background()); err != nil { - t.Fatal(err) - } - sent := r.asksSent(t) - if len(sent) != 1 || len(sent[0].Options) != 1 { - t.Fatalf("asked %+v", sent) - } - o := sent[0].Options[0] - if o.Label != "Release" || o.Level != asks.Approve || o.Does != "the held builds are sent to novox" { - t.Errorf("the option is %+v", o) - } - if err := sent[0].Check(r.now); err != nil { - t.Errorf("the ask is refused as made: %v", err) - } - w := r.warrantFor(t, c.Key, "Release") - answerWith(t, r, w) - answerWith(t, r, w) - if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-controller.push@ ") || - !strings.Contains(r.called[0], `"node":"novox"`) || !strings.Contains(r.called[0], `"move":"mailu"`) || - !strings.Contains(r.called[0], `"cause":"`+conditions.CauseOperatorAnswer+`"`) || - !strings.Contains(r.called[0], "chose Release") { - t.Fatalf("called %v", r.called) - } -} - -// 1732/AC4: the controller may call its own push and upgrade verbs on a warrant: the grant names them. -func TestTheControllerMayActOnAWarrantForPushAndUpgrade(t *testing.T) { - for _, verb := range []string{"push", "upgrade"} { - granted := false - for _, v := range broker.VerbsTheControllerActsOnAWarrant { - granted = granted || (v.Seat == broker.ControllerSeat && v.Verb == verb) - } - if !granted { - t.Errorf("the controller may not act on a warrant for %s", verb) - } - } -} - -func aDoneMailuPlan() inventory.Plan { - return inventory.Plan{ID: "plan-1792585970446304628", Repository: "novox/mesh-catalog", State: inventory.PlanDone, - Tiers: [][]string{{"a"}, {"b"}, {"mailu"}}, - Modules: map[string]*inventory.PlanModule{"mailu": {State: "built", Commit: "3259765dbbbbcccc", - Recorded: map[string]string{"novox": "waits: recreates 1 of mailu's 12 container(s)"}}}} -} - -// 1732/AC5: a done plan that carries a move a machine still holds for a person says so on its line; once the machine -// no longer holds it, or holds another build of the module, the line says nothing more. -func TestADonePlanSaysWhatItHoldsForAPerson(t *testing.T) { - p := aDoneMailuPlan() - line := planLine(p, time.Date(2026, 10, 21, 16, 0, 0, 0, time.UTC)) + heldSaid(p, []heldFacts{aHeldMailu()}) - if !strings.Contains(line, "done, 3 tier(s)") || !strings.Contains(line, "1 held for a person: mailu on novox") { - t.Errorf("the line: %q", line) - } - if said := heldSaid(p, nil); said != "" { - t.Errorf("a plan whose moves were sent says %q", said) - } - other := aHeldMailu() - other.moves[0].To = "9999999ffff" - if said := heldSaid(p, []heldFacts{other}); said != "" { - t.Errorf("a plan older than the build now held says %q", said) - } - p.Modules["mailu"].Recorded = map[string]string{"novox": recordedRollsWhy} - if said := heldSaid(p, []heldFacts{aHeldMailu()}); said != "" { - t.Errorf("a move the plan sent says %q", said) - } -} - -// 1732/AC6: the builds held after a failed release are asked of the operator: the held-release condition offers -// Release, which calls the controller's own `upgrade` with release-backlog and nothing else, once, on a warrant. -func TestTheBuildsHeldAfterAFailedReleaseAreAskedAndReleasedOnAWarrant(t *testing.T) { - saved := backlogNow - t.Cleanup(func() { backlogNow = saved }) - backlogNow.held = "release-1792595552499982311 failed (failed on novox); what waits is released again by a person" - backlogNow.waiting = map[string][]inventory.CarriedMove{"novox": {{Module: "indexer"}, {Module: "nextcloud"}}} - got := backlogObservation() - if len(got) != 1 || len(got[0].Actions) != 1 { - t.Fatalf("raised %+v", got) - } - a := got[0].Actions[0] - if a.Label != "Release" || a.Verb != "mesh-controller.upgrade" || a.Level != conditions.LevelApprove || - a.Arguments["release-backlog"] != "true" || a.Arguments["cause"] != conditions.CauseOperatorAnswer { - t.Fatalf("the action: %+v", a) - } - o := got[0] - c := conditions.Condition{Key: o.Key(), Kind: o.Kind, Severity: o.Severity, Headline: o.Headline, - Explanation: conditions.Verdict(o.Needs, o.Explanation), Needs: o.Needs, Actions: o.Actions} - r := newAskerRig(t) - r.open = []conditions.Condition{c} - if err := r.a.reconcile(context.Background()); err != nil { - t.Fatal(err) - } - sent := r.asksSent(t) - if len(sent) != 1 || len(sent[0].Options) != 1 || sent[0].Options[0].Does != "the builds held after the failed delivery are released" { - t.Fatalf("asked %+v", sent) - } - w := r.warrantFor(t, c.Key, "Release") - answerWith(t, r, w) - answerWith(t, r, w) - if len(r.called) != 1 || !strings.HasPrefix(r.called[0], "mesh-controller.upgrade@ ") || - !strings.Contains(r.called[0], `"release-backlog":"true"`) || !strings.Contains(r.called[0], "chose Release") { - t.Fatalf("called %v", r.called) - } -} - -// 1732/AC7: a plan whose send waits for a gate while the builds are held after a failed release reads as waiting for -// a person to release them, not "building ... LATE"; with nothing held it still reads as the wait it is. -func TestAPlanWaitingForAGateBehindAFailedReleaseIsNotLate(t *testing.T) { - saved := backlogNow - t.Cleanup(func() { backlogNow = saved }) - now := time.Date(2026, 10, 21, 18, 50, 0, 0, time.UTC) - p := inventory.Plan{ID: "plan-1", Repository: "novox/mesh-sdk", Commit: "c0ffee00", State: inventory.PlanBuilding, - Tiers: [][]string{{"indexer"}}, TierEntered: now.Add(-100 * time.Minute), Created: now.Add(-110 * time.Minute), - Modules: map[string]*inventory.PlanModule{"indexer": {State: "built"}}, - Note: refusalNote(0, errors.New("a build waits there for its gate: novox"))} - backlogNow.held = "" - if line := planLine(p, now); !strings.Contains(line, "a build waits there for its gate: novox") { - t.Errorf("the wait is not said: %q", line) - } - backlogNow.held = "release-1 failed (failed on novox); what waits is released again by a person" - line := planLine(p, now) - if strings.Contains(line, "LATE") || strings.Contains(line, "building") || - !strings.Contains(line, "until a person releases it") { - t.Errorf("a plan held behind a failed release reads %q", line) - } -} - -// 1732/AC10: a machine that also holds a move a person chose to hold gets no Release, since a push would send that move -// too; the condition names both and says the push is the controller's terminal. A machine with only asked moves keeps it. -func TestAMachineAlsoHoldingAPersonsOwnHoldIsOfferedNoRelease(t *testing.T) { - f := calm(time.Date(2026, 10, 21, 17, 30, 0, 0, time.UTC)) - h := aHeldMailu() - h.moves = append(h.moves, heldMove{Module: "nftables", From: "n1aaaa", To: "n2bbbb", - Why: "waits: a person set it to record, and holds every move of it until they push", Chosen: true}) - f.held = []heldFacts{h} - got := watchMovesHeld(f) - if len(got) != 1 || len(got[0].Actions) != 0 { - t.Fatalf("%+v", got) - } - o := got[0] - text := o.Headline + o.Explanation + o.Needs + o.Summary + o.Said - for _, want := range []string{"mailu", "nftables", "terminal", "push novox"} { - if !strings.Contains(text, want) { - t.Errorf("does not say %q: %q", want, text) - } - } - w := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs} - if why, ok := conditions.PlainWords(w, "novox"); !ok { - t.Errorf("not plain: %s", why) - } - f.held = []heldFacts{aHeldMailu()} - if got := watchMovesHeld(f); len(got) != 1 || len(got[0].Actions) != 1 || got[0].Actions[0].Arguments["move"] != "mailu" { - t.Fatalf("a machine with only asked moves: %+v", got) - } -} diff --git a/cmd/mesh-controller/helpers_kept_integration_test.go b/cmd/mesh-controller/helpers_kept_integration_test.go new file mode 100644 index 00000000..92a3fb1e --- /dev/null +++ b/cmd/mesh-controller/helpers_kept_integration_test.go @@ -0,0 +1,80 @@ +//go:build integration + +package main + +import ( + "context" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/builderkeys" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// Helpers the integration tests of the deleted plan machinery shared with tests that remain. + +// noListener hears no node: these tests send merges only. +type noListener struct{ link.Listener } + +func becomes(t *testing.T, within time.Duration, what string, is func() bool) { + t.Helper() + deadline := time.Now().Add(within) + for time.Now().Before(deadline) { + if is() { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("%s did not happen within %s", what, within) +} + +// noBuildAsked once failed a test that asked build-agent for a build; nothing can now, so it says nothing. +func noBuildAsked(t *testing.T) { t.Helper() } + +func twoTiers(t *testing.T, open *stores) { + t.Helper() + for _, name := range []string{"a", "b"} { + if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: name, Version: "1"}, + inventory.Source{Repository: "novox/" + name, Seat: "git", Ref: "main", BuiltFrom: "c0ffee", Head: "c0ffee"}); err != nil { + t.Fatal(err) + } + } +} + +func inventoryHolds(t *testing.T, inv *inventory.Inventory, module string) (inventory.Entry, bool) { + t.Helper() + entries, err := inv.Catalogued(t.Context()) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if e.Manifest.Module == module { + return e, true + } + } + return inventory.Entry{}, false +} + +// planKeysOf is the indexer and the builder as a walk reads them, for a test: values by module and commit, keys by +// module. +type planKeysOf struct { + values map[string]sdkdelivery.Module // by module + "@" + commit + keys map[string]builderkeys.Key +} + +func (k planKeysOf) ModuleAt(_ context.Context, module, commit string) (sdkdelivery.Module, error) { + if v, ok := k.values[module+"@"+commit]; ok { + return v, nil + } + return sdkdelivery.Module{}, builderkeys.ErrNoValue +} + +func (k planKeysOf) Build(_ context.Context, module, fp string) (builderkeys.Key, error) { + if key, ok := k.keys[module]; ok && key.Fingerprint == fp { + return key, nil + } + return builderkeys.Key{Module: module, Fingerprint: fp}, nil +} diff --git a/cmd/mesh-controller/helpers_kept_test.go b/cmd/mesh-controller/helpers_kept_test.go new file mode 100644 index 00000000..ac44a2ea --- /dev/null +++ b/cmd/mesh-controller/helpers_kept_test.go @@ -0,0 +1,45 @@ +package main + +import ( + "context" + "encoding/json" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// Helpers the tests of the deleted plan machinery shared with tests that remain. + +func nodeID(t *testing.T, open *stores, name string) string { + t.Helper() + n, err := open.inventory.NodeByName(context.Background(), name) + if err != nil { + t.Fatal(err) + } + return n.ID +} + +func bodyKeys(t *testing.T, s sendable) map[string]any { + t.Helper() + raw, err := s.Body() + if err != nil { + t.Fatal(err) + } + var keys map[string]any + if err := json.Unmarshal(raw, &keys); err != nil { + t.Fatal(err) + } + return keys +} + +// theDeliveryClaims is a catalogue whose one module claiming the seat is today's mesh-delivery, for the tests that +// announce a merge from it. +func theDeliveryClaims(context.Context) ([]inventory.Entry, error) { + return []inventory.Entry{claimingTheSeat("mesh-delivery")}, nil +} + +func claimingTheSeat(module string) inventory.Entry { + return inventory.Entry{Manifest: catalogue.Manifest{Module: module, + Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}} +} diff --git a/cmd/mesh-controller/hold_test.go b/cmd/mesh-controller/hold_test.go deleted file mode 100644 index 818247f7..00000000 --- a/cmd/mesh-controller/hold_test.go +++ /dev/null @@ -1,47 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "errors" - "testing" - "time" -) - -// A cascade round gives its hold back on every way out: a declaration that cannot be marshalled -// and a send that fails leave nobody waiting for the node. -func TestASendRoundGivesItsHoldBackOnEveryWayOut(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - unmarshallable := func(context.Context, string) (sendable, error) { - return sendable{Resources: []map[string]any{{"id": "x", "bad": make(chan int)}}}, nil - } - plain := func(context.Context, string) (sendable, error) { - return sendable{Resources: []map[string]any{{"id": "x"}}}, nil - } - failing := &recordingDelivery{declareErr: errors.New("the broker went away")} - fine := &recordingDelivery{} - - for name, round := range map[string]func() error{ - "a body that cannot be marshalled": func() error { - _, err := sendRound(ctx, open, []string{"anchor"}, unmarshallable, fine, "", nil) - return err - }, - "a send that fails": func() error { - _, err := sendRound(ctx, open, []string{"anchor"}, plain, failing, "", nil) - return err - }, - } { - if err := round(); err == nil { - t.Fatalf("%s was not an error", name) - } - waiting, cancel := context.WithTimeout(ctx, 2*time.Second) - release, err := open.inventory.HoldNodes(waiting, []string{"anchor"}) - cancel() - if err != nil { - t.Fatalf("after %s the node is still held: %v", name, err) - } - release() - } -} diff --git a/cmd/mesh-controller/issue201_integration_test.go b/cmd/mesh-controller/issue201_integration_test.go deleted file mode 100644 index a138642c..00000000 --- a/cmd/mesh-controller/issue201_integration_test.go +++ /dev/null @@ -1,181 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" -) - -func core201(t *testing.T) *stores { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) - record := func(commit string, asked time.Time) inventory.Build { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"}) - b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit, - Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}} - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - return b - } - before := record("c1aaaaaaaaaa", old) - if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: before.Commit, - Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil { - t.Fatal(err) - } - record("c2bbbbbbbbbb", now) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil { - t.Fatal(err) - } - if err := inv.RestoreModule(ctx, before); err != nil { - t.Fatal(err) - } - wasMoves := machineMoves - machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { - modules, err := open.inventory.Assigned(ctx, node) - if err != nil { - return nil, err - } - sent, known, err := open.inventory.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - return f.moves(node, modules, sent, known, all), nil - } - t.Cleanup(func() { machineMoves = wasMoves }) - return open -} - -func TestReplay201(t *testing.T) { - open := core201(t) - ctx := t.Context() - k, _ := withConditionsInMemory(t) - - judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}}) - if kept, err := ungatedIn(judged, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { - t.Fatalf("a plan's send to the machine is made, the rest of it with it: %v %v", kept, err) - } - keeps, err := recordedKept(keepingRecorded(judged), open, "anchor") - if err != nil || keeps["mesh-controller"] != "c2bbbbbbbbbb" { - t.Fatalf("the send keeps the newer controller there: %v %v", keeps, err) - } - shelf := map[string]catalogue.Manifest{"mesh-controller": {Module: "mesh-controller", Version: "old"}} - if err := applyKept(keepingRecorded(judged), open, "anchor", keeps, shelf); err != nil || shelf["mesh-controller"].Version != "1" { - t.Fatalf("the send composes the kept build's manifest: %+v %v", shelf, err) - } - key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key() - if c, found, err := k.Get(ctx, key); err != nil || !found || !strings.Contains(c.Summary, "c2bbbbbb") { - t.Fatalf("the condition %s is raised, saying the build kept: %+v %v %v", key, c, found, err) - } - - putBack := keepingRecorded(withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true})) - if keeps, err := recordedKept(putBack, open, "anchor"); err != nil || len(keeps) != 0 { - t.Fatalf("the gate's own put-back of the controller carries it back: %v %v", keeps, err) - } - - settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}) - if _, found, _ := k.Get(ctx, key); !found { - t.Fatal("a send that kept the newer controller cleared the condition") - } - settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c1aaaaaaaaaa"}) - if _, found, _ := k.Get(ctx, key); found { - t.Fatal("a send carrying the mesh's record — a person's push — leaves the condition open") - } -} - -func TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend(t *testing.T) { - open := core201(t) - ctx := t.Context() - k, _ := withConditionsInMemory(t) - if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: "build-mesh-controller-c2bbbbbbbbbb", - Module: "mesh-controller", Commit: "c2bbbbbbbbbb", Previous: "c1aaaaaaaaaa", Machines: []string{"anchor"}, - Verdict: inventory.GateFailed, Rollback: inventory.NotRolledBack}); err != nil { - t.Fatal(err) - } - healer := withScope(ctx, sendScope{}) - if kept, err := ungatedIn(healer, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { - t.Fatalf("the healer's send is made: %v %v", kept, err) - } - if keeps, err := recordedKept(keepingRecorded(healer), open, "anchor"); err != nil || len(keeps) != 0 { - t.Fatalf("the failed controller was kept over the build put back: %v %v", keeps, err) - } - if open, _ := k.Open(ctx); len(open) != 0 { - t.Fatalf("a completed put-back raised a condition: %+v", open) - } -} - -func TestAGateJudgesTheOtherMovesBesideAKeptCore(t *testing.T) { - open := core201(t) - ctx := t.Context() - withConditionsInMemory(t) - inv := open.inventory - old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) - for _, b := range []struct { - commit string - at time.Time - }{{"a1aaaaaaaaaa", old}, {"a2bbbbbbbbbb", now}} { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-" + b.commit, Module: "app", Commit: b.commit, - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: b.at, At: b.at, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-" + b.commit}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: b.commit, Head: b.commit, - Asked: b.at}); err != nil { - t.Fatal(err) - } - } - if _, err := inv.Assign(ctx, "anchor", "app"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2", - map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a1aaaaaaaaaa"}); err != nil { - t.Fatal(err) - } - // The send as it is made: the controller kept at c2, the app moved to the record. - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - return names, open.inventory.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-3", - map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a2bbbbbbbbbb"}) - } - t.Cleanup(func() { sendRollout = wasSend }) - - moves, sent, _, err := gatedSend(ctx, open, "anchor", nil) - if err != nil || len(sent) != 1 { - t.Fatalf("the gated send is made: %v %v", sent, err) - } - if len(moves) != 1 || moves[0].Module != "app" || moves[0].To != "a2bbbbbbbbbb" { - t.Fatalf("the gate judges the app's move and not the kept controller: %+v", moves) - } - builds, _, err := inv.SentBuilds(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - g := &inventory.PlanGate{Carried: moves} - f := gateFacts{commits: judgedCommits(g, nil), sentBuilds: map[string]map[string]string{"anchor": builds}} - for _, m := range []string{"app", "mesh-controller"} { - if why, superseded := f.supersededOn(m, "anchor"); superseded { - t.Fatalf("the gate is superseded by its own send: %s", why) - } - } -} diff --git a/cmd/mesh-controller/issue201_test.go b/cmd/mesh-controller/issue201_test.go deleted file mode 100644 index 7bd7085c..00000000 --- a/cmd/mesh-controller/issue201_test.go +++ /dev/null @@ -1,93 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/lease" -) - -// novox/hq issue 201: on 2026-10-02 a plan sent the control node a controller older than the one running there, -// whose seat row the newer one had already widened; the older binary refused the row and the mesh went quiet for -// ten minutes. No send but a person's push and a gate's own put-back moves the controller or the node-engine back -// behind the build its machine was last sent: the send keeps it there, and carries the rest. - -// askedIn answers a build from a fixed table: module@commit → when it was asked; failed names those that failed -// their gate. -func askedIn(table map[string]time.Time, failed ...string) func(module, commit string) (askedBuild, bool, error) { - return func(module, commit string) (askedBuild, bool, error) { - at, known := table[module+"@"+commit] - b := askedBuild{asked: at} - for _, f := range failed { - if f == module+"@"+commit { - b.failedGate = true - } - } - return b, known, nil - } -} - -func TestACoreModuleIsKeptOverAnOlderRecord(t *testing.T) { - old, now := t0, t0.Add(time.Minute) - table := map[string]time.Time{ - "mesh-controller@c1": old, "mesh-controller@c2": now, - "mesh-host@h1": old, "mesh-host@h2": now, - "app@a1": old, "app@a2": now, - } - sent := map[string]string{"mesh-controller": "c2", "mesh-host": "h2", "app": "a2"} - record := map[string]string{"mesh-controller": "c1", "mesh-host": "h1", "app": "a1"} - found, err := coreDowngrades("anchor", sent, record, sendScope{judged: map[string]bool{"anchor": true}}, askedIn(table)) - if err != nil { - t.Fatal(err) - } - if len(found) != 2 || found[0].module != "mesh-controller" || found[0].component != lease.ComponentController || - found[0].sent != "c2" || found[1].module != "mesh-host" || found[1].component != lease.ComponentEngine { - t.Fatalf("the controller and the node-engine over an older record are kept, and nothing else: %+v", found) - } - if said := found[0].String(); !strings.Contains(said, "anchor was last sent mesh-controller c2") || - !strings.Contains(said, "`push anchor`") || !strings.Contains(said, "`rebuild mesh-controller`") { - t.Fatalf("it says the machine, the build kept and the two ways out: %s", said) - } - - // The record forward, the same build, or a build the records do not hold: nothing kept. - for _, c := range [][2]string{{"c1", "c2"}, {"c2", "c2"}, {"c2", "c9"}, {"c9", "c1"}} { - found, err := coreDowngrades("anchor", map[string]string{"mesh-controller": c[0]}, - map[string]string{"mesh-controller": c[1]}, sendScope{}, askedIn(table)) - if err != nil || len(found) != 0 { - t.Fatalf("sent %s, record %s: no downgrade proved, nothing kept: %+v %v", c[0], c[1], found, err) - } - } - - // A sent build that failed its gate is never ahead: the record put back before it is sent. - if found, _ := coreDowngrades("anchor", sent, record, sendScope{}, askedIn(table, "mesh-controller@c2")); len(found) != 1 || - found[0].module != "mesh-host" { - t.Fatalf("a controller that failed its gate was kept over the build put back: %+v", found) - } - - // A gate's own put-back carries its own modules back, and nothing else. - putBack := sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true} - if found, _ := coreDowngrades("anchor", sent, record, putBack, askedIn(table)); len(found) != 1 || found[0].module != "mesh-host" { - t.Fatalf("a put-back of the controller carries it back, and not the node-engine: %+v", found) - } - if found, _ := coreDowngrades("anchor", sent, record, sendScope{modules: map[string]bool{"mesh-controller": true}}, - askedIn(table)); len(found) != 2 { - t.Fatalf("a send for the controller that is no put-back carries it back: %+v", found) - } -} - -// core201 is a mesh whose control node, anchor, was sent the newer controller c2 while the mesh's record of the -// controller is the build before it, c1, which passed its gate when it rolled out. - -// TestReplay201 replays the race on a store: the control node runs the newer controller and the mesh's record is -// the build before it. A plan's send there — the node-engine's plan of 2026-10-02 — goes, keeps the newer -// controller, and says so as a condition; the gate's own put-back carries it back; a send that carries the record, -// as a person's push does, clears the condition. - -// The review of #273: a gate failed the newer controller, put the record back, and the put-back's own send was -// refused. The next automatic send — a healer's, which judges nothing — completes the put-back: a build that -// failed its gate is never kept. - -// Review 91 of #273: a gated send to a machine whose controller is kept over an older record leaves the kept -// controller out of what its gate judges. Judged as moving back to the record, the gate would read the send's own -// kept controller as another build sent there after it (issue 352) and be superseded, judging nothing. diff --git a/cmd/mesh-controller/issue348_integration_test.go b/cmd/mesh-controller/issue348_integration_test.go deleted file mode 100644 index a723bda0..00000000 --- a/cmd/mesh-controller/issue348_integration_test.go +++ /dev/null @@ -1,199 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" - "github.com/novox/mesh-controller/internal/link" -) - -func TestReplay348(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv, k := open.inventory, conditionsFrom - say := func(at time.Time, n *link.NetworkHealth) { - t.Helper() - if err := stateHealth(ctx, inv, k, "anchor", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil { - t.Fatal(err) - } - } - network := func() (conditions.Condition, bool) { - t.Helper() - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - for _, c := range list { - if c.Key == "machine.anchor.network" { - return c, true - } - } - return conditions.Condition{}, false - } - - // 10:58:45 — the second failing look: raised. - say(h0, networkSaying(namesRefused(link.StateUnhealthy, 2))) - raised, ok := network() - if !ok { - t.Fatal("the resolver refusing on the control node raised nothing") - } - time.Sleep(5 * time.Millisecond) - sent := time.Now().UTC() - time.Sleep(5 * time.Millisecond) - - // 10:59:42 — the restarted engine's first statement: one look failed, a second decides. - say(h0.Add(time.Minute), networkSaying(namesRefused(link.StateUnknown, 1))) - if _, ok := network(); !ok { - t.Fatal("a statement that judged nothing yet cleared the condition: the restarted engine's first look " + - "said the fault was gone while it still refused") - } - // 11:00:23 — its second look: unhealthy again, the same raising. - say(h0.Add(2*time.Minute), networkSaying(namesRefused(link.StateUnhealthy, 2))) - again, ok := network() - if !ok || !again.Raised.Equal(raised.Raised) || again.Count != 1 { - t.Fatalf("the same raising was not kept: raised %s (first %s), count %d", again.Raised, raised.Raised, again.Count) - } - - // The gate on the control node, for a build sent after the fault began. - open2, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - f := gateFacts{judged: true, open: open2} - if w := aboutTheMachine("anchor", []string{"mesh-host"}, sent, f); w.whole != "" || len(w.on) != 0 { - t.Fatalf("a fault from before the send held the build: %+v", w) - } - - // Decided healthy: cleared. - say(h0.Add(3*time.Minute), networkSaying(link.NetworkPart{Part: link.PartNames, State: link.StateHealthy, Since: h0})) - if c, ok := network(); ok { - t.Fatalf("a statement that decides the names healthy left %s open", c.Key) - } -} - -// namesRefused is the control node's names part as its node-engine said it in the outage: its own -// resolver, at its own address, refusing. -func namesRefused(state string, streak int) link.NetworkPart { - p := link.NetworkPart{Part: link.PartNames, State: state, Since: h0, Streak: streak} - if state == link.StateUnhealthy { - p.Reason = "1 of its 2 resolvers do not answer as the mesh's do" - p.Said = "10.77.0.1 — anchor.internal (IPv4): read udp 10.77.0.1:35244->10.77.0.1:53: read: connection refused" - p.Toward = []string{"10.77.0.1"} - } - return p -} - -func networkSaying(parts ...link.NetworkPart) *link.NetworkHealth { - state := link.StateHealthy - for _, p := range parts { - switch { - case p.State == link.StateUnhealthy: - state = link.StateUnhealthy - case p.State == link.StateUnknown && state == link.StateHealthy: - state = link.StateUnknown - } - } - return &link.NetworkHealth{State: state, Since: h0, Parts: parts} -} - -// A fault there at the send, cleared and reopened after it, is not raised since the send; one that cleared -// before the send and came back after it is — a send that breaks a recovered machine fails its gate (review -// of mesh-controller PR 179, A2). Read through OpenAt, by both of the gate's readings. -func TestAFaultThatFlappedAfterTheSendIsNotTheSendsAndOneThatRecoveredBeforeItIs(t *testing.T) { - since := h0 - network := func(first time.Time, gaps ...conditions.Gap) conditions.Condition { - raised := since.Add(time.Minute) - if len(gaps) > 0 { - raised = gaps[len(gaps)-1].Reopened - } - return conditions.Condition{Key: "machine.anchor.network", Kind: kindMachineNetwork, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor"}, - Summary: "anchor's network is not healthy", Source: sourceNetwork, First: first, Gaps: gaps, Raised: raised} - } - held := func(c conditions.Condition) bool { - return aboutTheMachine("anchor", []string{"mesh-controller"}, since, gateFacts{judged: true, - open: []conditions.Condition{c}}).whole != "" - } - // The day's case: raised before the send, cleared 8 s after it, reopened 49 s after it. - flapped := network(since.Add(-49*time.Second), - conditions.Gap{Cleared: since.Add(8 * time.Second), Reopened: since.Add(49 * time.Second)}) - if held(flapped) { - t.Fatal("a fault there at the send, flapping after it, held the machine") - } - // Recovered before the send, broken again after it: the send's. - recovered := network(since.Add(-time.Hour), - conditions.Gap{Cleared: since.Add(-30 * time.Second), Reopened: since.Add(20 * time.Second)}) - if !held(recovered) { - t.Fatal("a machine recovered at the send and broken after it passed the gate") - } - // An older gap, before the send, and the fault there at the send: not the send's. - twice := network(since.Add(-time.Hour), - conditions.Gap{Cleared: since.Add(-50 * time.Minute), Reopened: since.Add(-45 * time.Minute)}, - conditions.Gap{Cleared: since.Add(10 * time.Second), Reopened: since.Add(30 * time.Second)}) - if held(twice) { - t.Fatal("a fault there at the send, with an older gap, held the machine") - } - // Raised after the send, never cleared: the send's. - if !held(network(time.Time{})) { - t.Fatal("a fault raised after the send held nothing") - } - // And a module's own, through judgeHealth. - at := since.Add(2 * time.Minute) - g := gateFacts{judged: true, now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", - Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{}, - served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}, - open: []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{ - Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Summary: "failing", - First: since.Add(-time.Hour), Raised: since.Add(time.Minute), - Gaps: []conditions.Gap{{Cleared: since.Add(5 * time.Second), Reopened: since.Add(time.Minute)}}}}} - if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); strings.HasPrefix(why, "raised since it was sent") { - t.Fatalf("a module's own fault there at the send: %s", why) - } - g.open[0].Gaps[0].Cleared = since.Add(-5 * time.Second) - if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); !strings.HasPrefix(why, "raised since it was sent") { - t.Fatalf("a module's own fault, recovered at the send and back after it, was not counted: %s", why) - } -} - -// Only an undecided part holds a condition that names it; a condition about another part clears, and a -// statement unknown as a whole holds every part (review of PR 179, A4). Pure. -func TestAnUndecidedPartHoldsOnlyWhatNamesIt(t *testing.T) { - f := netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateUnknown, inventory.NetworkPart{Part: link.PartNames, State: link.StateUnknown, Streak: 1}, - inventory.NetworkPart{Part: link.PartRoute, State: link.StateHealthy}), - "laptop": aNetwork(link.StateHealthy, inventory.NetworkPart{Part: link.PartNames, State: link.StateHealthy}), - "spare": aNetwork(link.StateStarting, inventory.NetworkPart{Part: link.PartTunnel, State: link.StateHealthy}), - "other": aNetwork(link.StateStarting, inventory.NetworkPart{Part: link.PartTunnel, State: link.StateStarting}), - }) - u := undecidedParts(f) - if !u["anchor"][link.PartNames] || u["anchor"][link.PartRoute] || u["laptop"] != nil || !u["spare"]["*"] || - !u["other"][link.PartTunnel] || u["other"]["*"] { - t.Fatalf("undecided: %v", u) - } - about := func(machine, said string, also ...string) conditions.Condition { - return conditions.Condition{Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: machine, - Machine: machine, Also: also}, Evidence: []conditions.Evidence{{Said: said}}} - } - for _, c := range []struct { - c conditions.Condition - held bool - }{ - {about("anchor", "names since 2026-10-09 10:58:45 UTC: 10.77.0.1 — refused"), true}, - {about("anchor", "route since 2026-10-09 10:58:45 UTC: no default route"), false}, - {about("laptop", "names since 2026-10-09 10:58:45 UTC: refused"), false}, - {about("spare", "route since …: no default route"), true}, - {about("hub", "anchor: names: refused", "anchor"), true}, - {about("hub", "anchor: tunnel: no handshake", "anchor"), false}, - } { - if got := heldUndecided(c.c, u); got != c.held { - t.Errorf("%s %q held %v, want %v", c.c.Subject.Machine, c.c.Evidence[0].Said, got, c.held) - } - } -} diff --git a/cmd/mesh-controller/issue352_integration_test.go b/cmd/mesh-controller/issue352_integration_test.go deleted file mode 100644 index 1f7c399e..00000000 --- a/cmd/mesh-controller/issue352_integration_test.go +++ /dev/null @@ -1,227 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "errors" - "reflect" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestReplay352(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn) - t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging) -} - -func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) { - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - advancePlans(ctx, b.open) // anchor is sent, and the fixture reports it applied - // 16:31:37 — another plan sends anchor a newer declaration, which it has not reported on. - carried, _, _ := inv.SentBuilds(ctx, "anchor") - if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-newer", carried); err != nil { - t.Fatal(err) - } - reports, _ := inv.LastReports(ctx) - for _, r := range reports { - if r.Node == "anchor" && r.Current { - t.Fatal("the fixture's newer send reads as reported") - } - } - gateEvery, gateBound = 0, 0 // past the bound at once: before the fix, "has not reported" fails it here - for i := 0; i < 4; i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - if p.State == inventory.PlanFailed || strings.Contains(p.Note, "has not reported") { - t.Fatalf("the release failed on the newer send: %s %s", p.State, p.Note) - } - if g := p.Release.Gate; g != nil && (g.Sent == nil || g.Sent["anchor"].Digest == "") { - t.Fatalf("the gate does not keep what it sent: %+v", g) - } - if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("app's pass on anchor was not kept: %+v %v %v", v, found, err) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { - t.Fatalf("app was put back to %s", current["app"].Commit) - } -} - -func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - advancePlans(ctx, b.open) // the release judges app c2 on anchor - _, _, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}}) - if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") { - t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err) - } - if len(b.sent) != 1 { - t.Fatalf("sent %v", b.sent) - } -} - -func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - inv := g.open.inventory - advancePlans(ctx, g.open) // anchor is sent app c2 first - if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-c3", map[string]string{"app": "c3"}); err != nil { - t.Fatal(err) - } - gateEvery = 0 - advancePlans(ctx, g.open) - p := g.plan(t) - if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") { - t.Fatalf("the plan is %s: %s", p.State, p.Note) - } - // Nothing put back: the registered build stands, the build is not marked, and the plan's gate made no - // rollback (a release may walk what the other send left waiting on anchor; that is not a put-back). - if r := p.Modules["app"].Gate.Rollback; r != "" { - t.Fatalf("a superseded judging made a rollback: %q", r) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { - t.Fatalf("app was put back to %s", current["app"].Commit) - } - if failed, _ := inv.GateFailed(ctx, "build-2"); failed { - t.Fatal("a superseded build was marked failed") - } -} - -func testASendThatMovesTheModuleSupersedesTheJudging(t *testing.T) { - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - advancePlans(ctx, b.open) - // Another send moves app on anchor to a build this gate does not judge. - carried, _, _ := inv.SentBuilds(ctx, "anchor") - carried["app"] = "c3" - if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-c3", carried); err != nil { - t.Fatal(err) - } - gateEvery = 0 - advancePlans(ctx, b.open) - p := b.release(t) - if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") { - t.Fatalf("the release is %s: %s", p.State, p.Note) - } - if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: a superseded judging puts nothing back", b.sent) - } - if _, found, _ := inv.GateOf(ctx, "build-app-c2"); found { - t.Fatal("a superseded judging kept a verdict") - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { - t.Fatalf("app was put back to %s", current["app"].Commit) - } -} - -func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - keeper, _ := withConditionsInMemory(t) - told := &conditions.Told{} - was := doctorFrom - doctorFrom = &doctor{open: open, keeper: keeper, teller: told} - t.Cleanup(func() { doctorFrom = was }) - wasSend := sendRollout - var sent [][]string - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - sent = append(sent, names) - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - - build := func(id, commit, digest string, asked time.Time) inventory.Build { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: commit}) - b := inventory.Build{ID: id, Module: "mesh-controller", Commit: commit, Repository: "novox/mesh-controller", Path: ".", - Manifest: manifest, Asked: asked, At: asked, Made: []inventory.Artifact{{Name: "controller", Kind: catalogue.ArtifactBundle, - Reference: "mesh-artifact://mesh-controller/controller/blobs/sha256:" + digest}}} - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: commit}, - inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Path: ".", BuiltFrom: commit, Head: commit, Asked: asked}); err != nil { - t.Fatal(err) - } - return b - } - previous := build("build-old", "ef26d4cb", strings.Repeat("1", 64), time.Now().Add(-2*time.Hour)) - failed := build("build-new", "e6b00e2e", strings.Repeat("2", 64), time.Now().Add(-time.Minute)) - if versionOfBuild(previous) != strings.Repeat("1", 12) { - t.Fatalf("the build's version is %q", versionOfBuild(previous)) - } - applied, err := inv.SchemaApplied(ctx) - if err != nil || applied < 87 { - t.Fatalf("the store's schema reaches %d (%v)", applied, err) - } - // The build before never recorded what it reads: not proved, refused. - if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "never recorded") { - t.Fatalf("an unknown reach: %v %q", ok, why) - } - // It reads less than the store has: refused, naming both. - if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied-1); err != nil { - t.Fatal(err) - } - if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "is at") { - t.Fatalf("a reach behind the store: %v %q", ok, why) - } - // Through the gate: the failed build is marked, nothing is put back, nothing is sent, the condition is urgent. - at := time.Now().Add(-5 * time.Minute) - state := &inventory.PlanModule{Build: failed.ID, Commit: failed.Commit, Previous: previous.Commit, First: []string{"anchor"}, FirstAt: &at} - p := inventory.Plan{ID: "plan-352", Repository: "novox/mesh-controller", Branch: "main", Commit: failed.Commit, Created: at, - State: inventory.PlanRolling, Tiers: [][]string{{"mesh-controller"}}, Modules: map[string]*inventory.PlanModule{"mesh-controller": state}} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - gateFailed(ctx, open, &p, "mesh-controller", state, []string{"anchor"}, "not healthy within 10m0s of its apply") - if state.Gate.Rollback != inventory.NotRolledBack || !strings.Contains(p.Note, "NOT put back") || !strings.Contains(p.Note, "the current build is kept") { - t.Fatalf("rollback %q: %s", state.Gate.Rollback, p.Note) - } - if len(sent) != 0 { - t.Fatalf("sent %v: nothing is put back", sent) - } - if current, _ := inv.CurrentBuilds(ctx); current["mesh-controller"].Commit != failed.Commit { - t.Fatalf("the module was put back to %s", current["mesh-controller"].Commit) - } - if marked, _ := inv.GateFailed(ctx, failed.ID); !marked { - t.Fatal("the failed build is not marked failed at its gate") - } - open2, _ := keeper.Open(ctx) - var found bool - for _, c := range open2 { - if c.Kind == kindRollbackFailed && c.Severity == conditions.Urgent && strings.Contains(c.Summary, "current build is kept") { - found = true - } - } - if !found { - t.Fatalf("no urgent rollback-failed condition saying the current build is kept: %+v", open2) - } - // Reaching the store: allowed. - if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied); err != nil { - t.Fatal(err) - } - if why, ok := controllerSchemaAllows(ctx, inv, previous); !ok { - t.Fatalf("a build that reads the whole schema was refused: %q", why) - } - // A build with no bundle to know it by: refused. - if why, ok := controllerSchemaAllows(ctx, inv, inventory.Build{Commit: "x"}); ok || !strings.Contains(why, "names no bundle") { - t.Fatalf("a build without a bundle: %v %q", ok, why) - } -} diff --git a/cmd/mesh-controller/issue352_test.go b/cmd/mesh-controller/issue352_test.go deleted file mode 100644 index 86e8cea5..00000000 --- a/cmd/mesh-controller/issue352_test.go +++ /dev/null @@ -1,156 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" -) - -// novox/hq issue 352: on 2026-10-09 a release's gate on the control node read the machine's report against a -// newer send another plan had just made there — not against its own send — and failed three builds the -// machine had reported healthy ("has not reported on what it was sent"), put them back on every machine, -// to a controller older than the store's schema, and that controller then judged the newer plan's -// controller passed from the put-back build's health. - -// TestReplay352 replays the walk on the backlog fixture: the release sends anchor and anchor reports; -// another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module -// to another build supersedes the judging: no verdict, nothing put back. - -// A plan's own first send waits while a release judges the same module on that machine with another build. - -// A merge plan's judging is superseded the same way: another send moved its module on the first machine. - -// A report is on the gate's own send: the declaration itself, or one sequenced after it; a gate kept -// without its send reads the report against the send made last, as before. Pure. -func TestAReportIsHeldAgainstTheGatesOwnSend(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - sent := inventory.SentDeclaration{Digest: "d-490", Sequence: 490} - for _, c := range []struct { - r inventory.Reported - want bool - }{ - {inventory.Reported{Declared: "d-490", Current: false}, true}, - {inventory.Reported{Declared: "d-491", ReportedSequence: 491, Current: true}, true}, - {inventory.Reported{Declared: "d-489", ReportedSequence: 489, Current: false}, false}, - {inventory.Reported{Declared: "other", ReportedSequence: 490}, true}, // the same sequence, said by another digest - {inventory.Reported{Declared: "d-495", ReportedSequence: 495, Current: true}, true}, // the last send: this one or a later one - {inventory.Reported{Declared: "", Current: false}, false}, - } { - if got := sent.ReportsOn(c.r); got != c.want { - t.Errorf("%+v on %+v: %v", c.r, sent, got) - } - } - byDigest := inventory.SentDeclaration{Digest: "d-1"} - if !byDigest.ReportsOn(inventory.Reported{Declared: "d-1"}) || byDigest.ReportsOn(inventory.Reported{ReportedSequence: 5}) || - !byDigest.ReportsOn(inventory.Reported{Current: true}) { - t.Fatal("a send kept without a sequence is matched by its digest and by the last send alone") - } - f := gateFacts{sent: map[string]inventory.SentDeclaration{"anchor": sent}} - if !f.reportedOn("anchor", inventory.Reported{Declared: "d-490"}) || f.reportedOn("anchor", inventory.Reported{Declared: "d-1"}) { - t.Fatal("a gate that kept its send read the report against something other than it") - } - if !f.reportedOn("laptop", inventory.Reported{Current: true}) || f.reportedOn("laptop", inventory.Reported{Current: false}) { - t.Fatal("a gate that did not keep its send does not read the report against the send made last") - } - // Through the merge plan's first-machine wait too. - at := time.Now() - state := inventory.PlanModule{First: []string{"anchor"}, FirstAt: &at, - Gate: &inventory.PlanGate{Machines: []string{"anchor"}, Sent: map[string]inventory.SentDeclaration{"anchor": sent}}} - reports := []inventory.Reported{{Node: "anchor", At: &at, Outcome: inventory.OutcomeApplied, Current: false, Declared: "d-490"}} - if step := nextRollout(state, []string{"anchor", "laptop"}, false, reports, at.Add(time.Minute), time.Hour); step.waiting != "" || step.failed != "" { - t.Fatalf("the first machine's report on the plan's own send read as none: %+v", step) - } - reports[0].Declared = "d-480" - if step := nextRollout(state, []string{"anchor", "laptop"}, false, reports, at.Add(time.Minute), time.Hour); step.waiting == "" { - t.Fatalf("a report on an older send read as the plan's: %+v", step) - } -} - -// A gate judges only the build the machine was last sent: last sent another build of the module, the -// judging is superseded, whatever the machine reports. Pure. -func TestAGateJudgesOnlyTheBuildTheMachineWasLastSent(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - at := time.Now() - f := gateFacts{now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", Outcome: inventory.OutcomeApplied, - At: &at, Current: true}}, engines: map[string]string{}, served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}, - commits: map[string]string{"mesh-controller": "e6b00e2e"}, sentBuilds: map[string]map[string]string{"anchor": {"mesh-controller": "ef26d4cb"}}} - taken := at.Add(-30 * time.Second) - f.holder = &lease.Holder{Taken: taken, Health: &lease.Health{Ready: true}} - h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f) - if h != healthSuperseded || !strings.Contains(why, "ef26d4cb") || !strings.Contains(why, "e6b00e2e") { - t.Fatalf("a controller build the machine no longer runs: %v %q", h, why) - } - f.sentBuilds["anchor"]["mesh-controller"] = "e6b00e2e" - if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f); h == healthSuperseded { - t.Fatalf("the build sent read as another: %q", why) - } - delete(f.sentBuilds, "anchor") - if h, why := judgeHealth("mesh-controller", lease.ComponentController, catalogue.Manifest{}, "anchor", at.Add(-time.Minute), f); h == healthSuperseded { - t.Fatalf("a machine whose send is not known read as superseded: %q", why) - } - g := &inventory.PlanGate{To: "c2", Carried: []inventory.CarriedMove{{Module: "late", Node: "anchor", To: "c5"}}} - if got := judgedCommits(g, []judged{{"app", "anchor"}, {"late", "anchor"}}); got["app"] != "c2" || got["late"] != "c5" { - t.Fatalf("judged commits %v", got) - } -} - -// A move of another build of a module to a machine where a release or a plan is judging that module -// waits for that judging; the same build to that machine is already there. Pure. -func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - at := time.Now() - release := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{ - Gate: &inventory.PlanGate{Machines: []string{"novox"}, Carried: []inventory.CarriedMove{ - {Module: "mesh-controller", Node: "novox", From: "ef26d4cb", To: "2913c54c"}}}}} - merge := inventory.Plan{ID: "plan-1", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{ - "app": {First: []string{"anchor"}, FirstAt: &at, Commit: "c2", Gate: &inventory.PlanGate{Machines: []string{"anchor"}}}}} - f := moveFacts{plans: []inventory.Plan{release, merge}} - for _, c := range []struct { - module, node, to, want string - }{ - {"mesh-controller", "novox", "e6b00e2e", "release-1"}, // the day's case: a newer controller to the judged machine - {"mesh-controller", "novox", "2913c54c", ""}, // the same build: already there - {"mesh-controller", "ace", "2913c54c", "release-1"}, // another machine while the first is judged - {"mesh-host", "novox", "x", ""}, // a module the release does not carry - {"app", "anchor", "c2", ""}, - {"app", "anchor", "c3", "plan-1"}, - {"app", "laptop", "c2", "plan-1"}, - } { - if got := f.walkedBy(c.module, c.node, c.to); got != c.want { - t.Errorf("%s %s to %s: walked by %q, want %q", c.module, c.to, c.node, got, c.want) - } - } - release.Release.Gate.Verdict = inventory.GatePassed - merge.Modules["app"].Gate.Verdict = inventory.GatePassed - if f.walkedBy("mesh-controller", "novox", "e6b00e2e") != "" || f.walkedBy("app", "laptop", "c3") != "" { - t.Fatal("a passed judging still holds a move") - } - release.Release.Gate.Verdict = "" - f.plans[0].State = inventory.PlanSuperseded - if f.walkedBy("mesh-controller", "novox", "e6b00e2e") != "" { - t.Fatal("a closed release still holds a move") - } -} - -// The controller is never put back to a build that reaches less of the store's schema than the store -// has, or to one that never said what it reaches: the current build is kept, and the condition says so. - -// The lease's holder names the build the declaration told it it is, and the version stamp only without one. -func TestTheHolderNamesTheBuildTheDeclarationToldIt(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - t.Setenv(RunningBuildVar, " ad62528c47c7 ") - if h := holderOf("x"); h.Build != "ad62528c47c7" { - t.Fatalf("the holder's build is %q", h.Build) - } - t.Setenv(RunningBuildVar, "") - if h := holderOf("x"); h.Build != version { - t.Fatalf("without a declared version the holder's build is %q", h.Build) - } - if reach, err := schemaReach(); err != nil || reach < 87 { - t.Fatalf("this build's reach is %d (%v)", reach, err) - } -} diff --git a/cmd/mesh-controller/issue459_integration_test.go b/cmd/mesh-controller/issue459_integration_test.go deleted file mode 100644 index 14c9acc0..00000000 --- a/cmd/mesh-controller/issue459_integration_test.go +++ /dev/null @@ -1,293 +0,0 @@ -//go:build integration - -package main - -import ( - "errors" - "reflect" - "slices" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestAStopWithoutAMergeWalksTheOthersAgain(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/cat1aaaaaaaa") - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - cat := catalogueMerge("cat1aaaaaaaa", "app", t0) - one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) - hear(t, open, cat, t0.Add(time.Second)) - hear(t, open, one, t0.Add(6*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - if len(ws) != 1 || !ws[0].Open() { - t.Fatalf("no open walk to stop: %+v", ws) - } - stopped := ws[0] - p, said, err := stopWalkWithout(ctx, open.inventory, stopped.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute)) - if err != nil { - t.Fatal(err) - } - if p.State != inventory.PlanFailed || p.Delivery.Stopped == "" || p.Delivery.WalkedAgainBy == "" || - !reflect.DeepEqual(p.Delivery.LeftOut, []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: cat.Commit}}) { - t.Fatalf("the stopped walk reads %s %+v", p.State, p.Delivery) - } - if said.WalkedAgainBy != p.Delivery.WalkedAgainBy || !reflect.DeepEqual(said.WalkedAgain, []string{"novox/one@" + one.Commit}) { - t.Fatalf("the stop answered %+v", said) - } - again, err := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy) - if err != nil || len(again) != 1 || again[0].Commit != one.Commit { - t.Fatalf("the fresh batch answers %+v (%v); want the other repository's merge alone", again, err) - } - if left, _ := open.inventory.MergesOf(ctx, stopped.ID); len(left) != 1 || left[0].Commit != cat.Commit { - t.Fatalf("the stopped walk still answers %+v; want the left-out merge alone", left) - } - cutAt(t, open, t0.Add(4*time.Minute)) - fresh, err := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy) - if err != nil || fresh.Batch() || !fresh.Open() { - t.Fatalf("the fresh batch was not cut into a walk: %s %v", fresh.State, err) - } - if _, in := fresh.Modules["one"]; !in { - t.Fatalf("the fresh walk does not build one: %v", fresh.Modules) - } - if _, in := fresh.Modules["app"]; in { - t.Fatalf("the fresh walk builds app, whose merge was left out: %v", fresh.Modules) - } - if len(fresh.Delivery.Merges) != 1 || fresh.Delivery.Merges[0].Commit != one.Commit { - t.Fatalf("the fresh walk answers %+v", fresh.Delivery.Merges) - } - // A stopped walk starts no search of its own (ADR 0276 decision 3 is for a failed one). - cutAt(t, open, t0.Add(5*time.Minute)) - if alone, _ := open.inventory.AloneMerges(ctx); len(alone) != 0 { - t.Fatalf("a merge of the stopped walk waits to be walked alone: %+v", alone) - } -} - -func TestAStopWithoutAMergeEndsTheMergesThatContainIt(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/cat1aaaaaaaa") - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - first := catalogueMerge("cat1aaaaaaaa", "app", t0) - later := catalogueMerge("cat2cccccccc", "notes", t0.Add(10*time.Second)) - one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), first.Head) - hear(t, open, first, t0.Add(time.Second)) - hear(t, open, one, t0.Add(6*time.Second)) - hear(t, open, later, t0.Add(11*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - walk := ws[0] - - if _, _, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "x", - []string{"novox/mesh-catalog@ffffffffffff"}, t0.Add(3*time.Minute)); err == nil { - t.Fatal("a stop leaving out a merge the walk does not answer was taken") - } - if p, _ := open.inventory.PlanByID(ctx, walk.ID); !p.Open() { - t.Fatalf("a refused stop ended the walk: %s", p.State) - } - - p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + first.Commit}, t0.Add(3*time.Minute)) - if err != nil { - t.Fatal(err) - } - want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: first.Commit}, - {Repository: "novox/mesh-catalog", Commit: later.Commit, Contains: first.Commit}} - if !reflect.DeepEqual(p.Delivery.LeftOut, want) || !reflect.DeepEqual(said.LeftOut, want) { - t.Fatalf("left out %+v, said %+v; want %+v", p.Delivery.LeftOut, said.LeftOut, want) - } - if !strings.Contains(p.Note, later.Commit[:8]) || !strings.Contains(p.Note, p.Delivery.WalkedAgainBy) { - t.Fatalf("the stopped walk's note does not name the merge ended with it and the fresh batch: %q", p.Note) - } - again, _ := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy) - if len(again) != 1 || again[0].Commit != one.Commit { - t.Fatalf("walked again: %+v; want the other repository's merge alone", again) - } - - // Nothing left once the other repository's merge is left out too: the fresh walk is stopped, and nothing more. - cutAt(t, open, t0.Add(4*time.Minute)) - fresh, _ := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy) - q, said, err := stopWalkWithout(ctx, open.inventory, fresh.ID, "mesh-delivery for jochen", "that one too", - []string{"novox/one@" + one.Commit}, t0.Add(5*time.Minute)) - if err != nil || q.State != inventory.PlanFailed || q.Delivery.WalkedAgainBy != "" || said.WalkedAgainBy != "" { - t.Fatalf("a stop leaving nothing to walk: %s %+v %+v %v", q.State, q.Delivery, said, err) - } - if _, bs := walks(t, open); len(bs) != 0 { - t.Fatalf("a batch of nothing was made: %+v", bs) - } -} - -func TestAStopWalksTheOthersAgainInTheOpenBatch(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/cat1aaaaaaaa") - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - cat := catalogueMerge("cat1aaaaaaaa", "app", t0) - one1 := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) - hear(t, open, cat, t0.Add(time.Second)) - hear(t, open, one1, t0.Add(6*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - walk := ws[0] - one2 := repoMerge("one", "one2cccccccc", t0.Add(3*time.Minute)) - hear(t, open, one2, t0.Add(3*time.Minute)) - _, bs := walks(t, open) - if len(bs) != 1 { - t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs)) - } - waiting := bs[0] - p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second)) - if err != nil { - t.Fatal(err) - } - if p.Delivery.WalkedAgainBy != waiting.ID || said.WalkedAgainBy != waiting.ID { - t.Fatalf("walked again by %q (said %q); want the open batch %s", p.Delivery.WalkedAgainBy, said.WalkedAgainBy, waiting.ID) - } - if _, bs := walks(t, open); len(bs) != 1 { - t.Fatalf("%d batches; want the open one alone", len(bs)) - } - b, _ := open.inventory.PlanByID(ctx, waiting.ID) - if b.Delivery.Batch == nil || !b.Delivery.Batch.Closed || !slices.Contains(b.Delivery.Batch.Withheld, "app") { - t.Fatalf("the open batch reads %+v; want its window closed and app withheld", b.Delivery.Batch) - } - // Its window closed by the stop, not by the clock: cut at once. - cutAt(t, open, t0.Add(3*time.Minute+11*time.Second)) - fresh, _ := open.inventory.PlanByID(ctx, waiting.ID) - if fresh.Batch() || fresh.CommitOf("novox/one") != one2.Commit { - t.Fatalf("the batch is %s at %q; want it cut, walking the newer commit", fresh.State, fresh.CommitOf("novox/one")) - } - var answered []string - for _, m := range fresh.Delivery.Merges { - answered = append(answered, m.Commit+">"+m.Carried) - } - if !slices.Contains(answered, one1.Commit+">"+one2.Commit) || !slices.Contains(answered, one2.Commit+">") { - t.Fatalf("the walk answers %v; want the earlier merge carried by the later", answered) - } -} - -func TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/cat1aaaaaaaa") - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - cat := catalogueMerge("cat1aaaaaaaa", "app", t0) - one1 := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) - hear(t, open, cat, t0.Add(time.Second)) - hear(t, open, one1, t0.Add(6*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - walk := ws[0] - sent := t0.Add(150 * time.Second) - for name, m := range walk.Modules { - if name == "app" || name == "one" { - m.First, m.FirstAt = []string{"anchor"}, &sent - } - } - if err := open.inventory.SavePlan(ctx, &walk); err != nil { - t.Fatal(err) - } - cat2 := catalogueMerge("cat2dddddddd", "notes", t0.Add(3*time.Minute)) - hear(t, open, cat2, t0.Add(3*time.Minute)) - _, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second)) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(said.Kept, []inventory.PlanKept{{Module: "app", Machines: []string{"anchor"}}}) { - t.Fatalf("kept %+v; want app on anchor alone (one is walked again)", said.Kept) - } - p, _ := open.inventory.PlanByID(ctx, walk.ID) - if !reflect.DeepEqual(p.Delivery.Kept, said.Kept) || !strings.Contains(p.Note, "app on anchor") { - t.Fatalf("the stopped walk's record does not name what stays: %+v %q", p.Delivery.Kept, p.Note) - } - if len(said.StillCarriedBy) != 1 || !strings.Contains(said.StillCarriedBy[0], cat2.Commit) { - t.Fatalf("still carried by %v; want the later catalogue merge waiting in the batch", said.StillCarriedBy) - } - if _, err := retryPlan(ctx, open, walk.ID); err == nil || !strings.Contains(err.Error(), "was stopped without") { - t.Fatalf("a walk stopped without some merges was retried, or refused for another reason: %v", err) - } -} - -func TestAStopWithoutAMergeLeavesItOutOfABatch(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - cat := catalogueMerge("cat1aaaaaaaa", "app", t0) - one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) - hear(t, open, cat, t0.Add(time.Second)) - hear(t, open, one, t0.Add(6*time.Second)) - _, bs := walks(t, open) - b := bs[0] - r, said, err := stopWalkWithout(ctx, open.inventory, b.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(10*time.Second)) - if err != nil { - t.Fatal(err) - } - if said.WalkedAgainBy != b.ID || said.Stopped == b.ID { - t.Fatalf("said %+v; want a stopped record of its own, and the batch walking the others", said) - } - if left, _ := open.inventory.MergesOf(ctx, said.Stopped); len(left) != 1 || left[0].Commit != cat.Commit { - t.Fatalf("the stopped record holds %+v", left) - } - if stopped, _ := open.inventory.PlanByID(ctx, said.Stopped); stopped.State != inventory.PlanFailed || - stopped.Delivery.Stopped == "" || stopped.Delivery.WalkedAgainBy != b.ID { - t.Fatalf("the stopped record reads %s %+v", stopped.State, stopped.Delivery) - } - _ = r - cutAt(t, open, t0.Add(3*time.Minute)) - w, _ := open.inventory.PlanByID(ctx, b.ID) - if _, in := w.Modules["app"]; in || w.Batch() { - t.Fatalf("the batch's walk is %s and builds %v; want it cut without app", w.State, w.Modules) - } - // A batch whose every merge is left out is stopped itself. - hear(t, open, catalogueMerge("cat3eeeeeeee", "notes", t0.Add(4*time.Minute)), t0.Add(4*time.Minute)) - _, bs = walks(t, open) - last := bs[0] - if _, said, err := stopWalkWithout(ctx, open.inventory, last.ID, "mesh-delivery for jochen", "x", - []string{"novox/mesh-catalog@cat3eeeeeeee"}, t0.Add(4*time.Minute+time.Second)); err != nil || said.Stopped != last.ID { - t.Fatalf("a batch left with nothing: %+v %v", said, err) - } - if p, _ := open.inventory.PlanByID(ctx, last.ID); p.State != inventory.PlanFailed || p.Batch() { - t.Fatalf("the emptied batch is %s", p.State) - } -} - -func TestSavingPlansAndMovingMergesIsOneAct(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - endsItsLine(t) - ctx := t.Context() - hear(t, open, catalogueMerge("cat1aaaaaaaa", "app", t0), t0.Add(time.Second)) - _, bs := walks(t, open) - stale := bs[0] - moved := stale - moved.Note = "written by somebody else" - if err := open.inventory.SavePlan(ctx, &moved); err != nil { - t.Fatal(err) - } - fresh := inventory.Plan{ID: "plan-fresh", Repository: "novox/mesh-catalog", Branch: "main", Commit: "x", Created: t0, - State: inventory.PlanAssembling, Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}} - err := open.inventory.SavePlansMoving(ctx, []*inventory.Plan{&fresh, &stale}, - []inventory.MergeMove{{Repository: "novox/mesh-catalog", Commit: "cat1aaaaaaaa", Plan: "plan-fresh"}}) - if !errors.Is(err, inventory.ErrPlanMoved) { - t.Fatalf("a stale plan in the act: %v", err) - } - if _, err := open.inventory.PlanByID(ctx, "plan-fresh"); err == nil { - t.Fatal("the new plan was written though the act was refused") - } - if m, _, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", "cat1aaaaaaaa"); m.Plan != stale.ID { - t.Fatalf("the merge moved to %q though the act was refused", m.Plan) - } -} diff --git a/cmd/mesh-controller/issue459_test.go b/cmd/mesh-controller/issue459_test.go deleted file mode 100644 index 9c79df8c..00000000 --- a/cmd/mesh-controller/issue459_test.go +++ /dev/null @@ -1,158 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "reflect" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq issue 459, ADR 0299: since the merge window one walk carries every merge of its batch, so a stop of one -// group member through `delivery-stop` ended the walk of every other delivery in it. The stop now names the merges -// to leave out: the walk is ended, and every other merge it answered is walked again in a fresh batch without them -// — the stopped merge, and each later merge of its repository's branch, which contains it and ends with it. - -// endsItsLine ends, as the test ends, a line the controller's output left open (a walk's ask prints " tier N: " and -// the asked build the rest, which the tests' fake ask never prints): `go test -v` would put the test's "--- PASS" -// after it, and the store-tests step of merge-check.sh, which reads that marker at a line's start, would say the -// test never ran. -func endsItsLine(t *testing.T) { - t.Cleanup(func() { fmt.Println() }) -} - -// leaved is a kept merge of a repository, made at a moment. -func leaved(repository, commit string, made time.Time) inventory.BatchedMerge { - return inventory.BatchedMerge{Repository: repository, Branch: "main", Commit: commit, Merged: made, Heard: made} -} - -// The stopped merge is left out, and so is each later merge of its repository's branch, naming the merge it -// contains; an earlier merge of that repository and every other repository's merge are walked again. -func TestLeftOutAreTheStoppedMergeAndTheMergesThatContainIt(t *testing.T) { - c1 := leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0) - c2 := leaved("novox/mesh-catalog", "c2bbbbbbbbbb", t0.Add(10*time.Second)) - c3 := leaved("novox/mesh-catalog", "c3cccccccccc", t0.Add(20*time.Second)) - o1 := leaved("novox/one", "o1dddddddddd", t0.Add(5*time.Second)) - again, out, err := leaveOut([]inventory.BatchedMerge{c1, o1, c2, c3}, []string{"novox/Mesh-Catalog@c2bbbbbbbbbb"}) - if err != nil { - t.Fatal(err) - } - var walked []string - for _, m := range again { - walked = append(walked, m.Commit) - } - if !reflect.DeepEqual(walked, []string{c1.Commit, o1.Commit}) { - t.Fatalf("walked again: %v; want the earlier catalogue merge and the other repository's", walked) - } - want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: c2.Commit}, - {Repository: "novox/mesh-catalog", Commit: c3.Commit, Contains: c2.Commit}} - if !reflect.DeepEqual(out, want) { - t.Fatalf("left out: %+v; want %+v", out, want) - } - // A merge the walk does not answer is refused, naming it: nothing ends. - if _, _, err := leaveOut([]inventory.BatchedMerge{c1, o1}, []string{"novox/mesh-catalog@ffffffffffff"}); err == nil || - !strings.Contains(err.Error(), "ffffffffffff") { - t.Fatalf("a merge the walk does not answer was left out: %v", err) - } - if _, _, err := leaveOut([]inventory.BatchedMerge{c1}, []string{"no-commit"}); err == nil { - t.Fatal("a merge without its commit was taken") - } -} - -// moving is a kept merge that moved modules when it was heard. -func moving(m inventory.BatchedMerge, modules ...string) inventory.BatchedMerge { - m.Event, _ = json.Marshal(keptMerge{Moves: modules}) - return m -} - -// A merge of another repository that moves a module the stopped merge moves is left out too, naming the modules -// it shares: the module is built at its branch, which holds the stopped merge (review of #232). And so on, until -// none more is: a merge sharing a module with that one is left out as well. -func TestAMergeSharingAStoppedMergesModuleIsLeftOut(t *testing.T) { - stopped := moving(leaved("novox/mesh-catalog", "c1aaaaaaaaaa", t0), "app") - reader := moving(leaved("novox/one", "o1bbbbbbbbbb", t0.Add(time.Second)), "app", "one") - next := moving(leaved("novox/two", "t1cccccccccc", t0.Add(2*time.Second)), "one") - free := moving(leaved("novox/three", "h1dddddddddd", t0.Add(3*time.Second)), "three") - again, out, err := leaveOut([]inventory.BatchedMerge{stopped, reader, next, free}, []string{"novox/mesh-catalog@c1aaaaaaa"}) - if err != nil { - t.Fatal(err) - } - if len(again) != 1 || again[0].Commit != free.Commit { - t.Fatalf("walked again: %+v; want the merge that shares nothing alone", again) - } - want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: stopped.Commit}, - {Repository: "novox/one", Commit: reader.Commit, Contains: stopped.Commit, Shares: []string{"app"}}, - {Repository: "novox/two", Commit: next.Commit, Contains: stopped.Commit, Shares: []string{"one"}}} - if !reflect.DeepEqual(out, want) { - t.Fatalf("left out %+v; want %+v", out, want) - } -} - -// A walk withholds a module a stopped merge moved where it would build it only as a dependent; one a merge of its -// own batch moves is built, and empty tiers go. -func TestAWalkWithholdsAStoppedMergesDependents(t *testing.T) { - plan := inventory.Plan{Tiers: [][]string{{"one"}, {"app"}, {"notes", "app2"}}, - Modules: map[string]*inventory.PlanModule{"one": {}, "app": {}, "notes": {}, "app2": {}}} - got := withhold(&plan, []string{"app", "notes", "absent"}, []string{"notes"}) - if !reflect.DeepEqual(got, []string{"app"}) || !reflect.DeepEqual(plan.Tiers, [][]string{{"one"}, {"notes", "app2"}}) { - t.Fatalf("withheld %v, tiers %v", got, plan.Tiers) - } - if _, in := plan.Modules["app"]; in { - t.Fatal("a withheld module is still in the plan") - } -} - -// The seat verb passes the merges to leave out to the command. -func TestDeliveryStopTakesTheMergesToLeaveOut(t *testing.T) { - got, err := argvFor("delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen", - "without": "novox/mesh-lab@b1,novox/mesh-lab@b2"}) - want := []string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen", - "--without", "novox/mesh-lab@b1,novox/mesh-lab@b2"} - if err != nil || !reflect.DeepEqual(got, want) { - t.Fatalf("delivery-stop with without → %v %v, wanted %v", got, err, want) - } -} - -// A walk of the catalogue's merge and another repository's is stopped without the catalogue's: it ends stopped, -// names the batch that walks the other merge again and the merge it left out, and that batch is cut into a walk -// that builds the other repository's module alone. - -// A later merge of the stopped merge's repository contains it: it is left out too, named with the merge it -// contains, and only the other repository's merge is walked again. With nothing left to walk, the walk is only -// ended. A merge the walk does not answer is refused, and nothing ends. - -// A merge heard while the walk ran waits in the open batch, at a later commit of the same repository. The merge -// walked again joins that batch, never a batch of its own that would walk the older commit after the newer one -// (review of #232): its window closes, and its walk is at the newer commit, answering both. - -// A stop names what the stopped walk already sent and nothing walks again, on which machines; and a later merge of -// the stopped merge's repository that waits in a batch, which will deliver the stopped change. A stopped walk -// that walked its merges again is refused a retry, saying why. - -// A batch not yet cut is stopped without a merge: the merge goes to a stopped record of its own, the batch goes on -// with the others and withholds the stopped merge's modules; with every merge left out, the batch is stopped. - -// The stopped walk, the batch and the merges moved are one act (ADR 0299 decision 7): a plan written by somebody -// else since it was read refuses all of it, and no merge moves, no plan is written. - -// What a stopped walk leaves on the machines names every machine its sends reached (review of #232): a module sent -// with no first machine reached every machine running it; one sent first reached those, and the rest it was sent -// to; a module a merge walked again moves is sent again, and is not named. -func TestKeptNamesEveryMachineASendReached(t *testing.T) { - at := t0 - p := inventory.Plan{Modules: map[string]*inventory.PlanModule{ - "together": {SentAt: &at}, - "first": {First: []string{"anchor"}, FirstAt: &at, Rest: map[string]inventory.SentDeclaration{"laptop": {}}}, - "again": {SentAt: &at}, - "unsent": {}, - }} - got := keptOf(p, []string{"again"}, map[string][]string{"together": {"laptop", "anchor"}, "again": {"anchor"}}) - want := []inventory.PlanKept{{Module: "first", Machines: []string{"anchor", "laptop"}}, - {Module: "together", Machines: []string{"anchor", "laptop"}}} - if !reflect.DeepEqual(got, want) { - t.Fatalf("kept %+v; want %+v", got, want) - } -} diff --git a/cmd/mesh-controller/issue529_test.go b/cmd/mesh-controller/issue529_test.go deleted file mode 100644 index 444d33ab..00000000 --- a/cmd/mesh-controller/issue529_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "strings" - "testing" - "time" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq issue 529: a walk waiting because its delivery is held for a person said only "its walk waits for -// mesh-delivery's word — `plans go … --why …` starts it by hand", pointing a reader at a hand start instead of at -// the person's decision. The line names what mesh-delivery says holds the walk: the held delivery, its reason and -// `mesh-delivery.release`; or the delivery not at its turn. - -func waitingWalk(now time.Time) inventory.Plan { - return inventory.Plan{ID: "plan-1791793277221379309", Repository: "novox/mesh-sdk", Branch: "main", - Commit: "9747f3e1b1ac", State: inventory.PlanBuilding, Tiers: [][]string{{"mesh-sdk"}}, - Created: now.Add(-9 * time.Minute), Modules: map[string]*inventory.PlanModule{}, - Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}} -} - -func TestAWalkWaitingOnAHeldDeliveryNamesTheHold(t *testing.T) { - now := time.Date(2026, 10, 12, 8, 35, 0, 0, time.UTC) - p := waitingWalk(now) - held := "novox/mesh-sdk@d41d255f28ad held since 2026-10-12T08:23:00Z: its group adr-0307-state-history's composed " + - "check did not pass for the heads that merged — a person decides: `mesh-delivery.release` (id " + - "novox/mesh-sdk@d41d255f28ad, why) lets it go, `mesh-delivery.stop` ends it" - waits := walkWaits{p.ID: {{ID: "novox/mesh-sdk@d41d255f28ad", State: "held", Walk: p.ID, Says: held}}} - line := planLineSaid(p, now, tierAtLeast, nil, waits) - for _, want := range []string{"held for a person", "composed check did not pass", "mesh-delivery.release"} { - if !strings.Contains(line, want) { - t.Fatalf("the line does not say %q: %q", want, line) - } - } - if strings.Contains(line, "starts it by hand") { - t.Fatalf("a held walk's line points at a hand start: %q", line) - } - // Told nothing — an older mesh-delivery, or none answering — it reads as before. - if line := planLineSaid(p, now, tierAtLeast, nil, nil); !strings.Contains(line, "starts it by hand") { - t.Fatalf("told nothing, the line reads %q", line) - } -} - -func TestAWalkWaitingOnADeliveryNotAtItsTurnSaysSo(t *testing.T) { - now := time.Date(2026, 10, 12, 8, 57, 0, 0, time.UTC) - p := waitingWalk(now) - waits := walkWaits{p.ID: {{ID: "novox/mesh-sdk@9747f3e1b1ac", State: "published", Walk: p.ID, - Says: "not let go yet: novox/mesh-tools@3f0c1a6cec18 (published) carried by it not at its turn"}}} - line := planLineSaid(p, now, tierAtLeast, nil, waits) - if !strings.Contains(line, "not at its turn") || !strings.Contains(line, "novox/mesh-tools@3f0c1a6cec18") { - t.Fatalf("the line reads %q", line) - } - // No hand start offered: it would skip the group's order (review of issue 529). - if strings.Contains(line, "plans go") { - t.Fatalf("a walk not at its turn points at a hand start: %q", line) - } -} - -// A walk nothing holds — the nzbgeek case, its merges' deliveries ended as nothing-to-move — is said so, and the hand -// start is the way on. -func TestAWalkHeldByNothingSaysWhyAndTheHandStart(t *testing.T) { - now := time.Date(2026, 10, 11, 20, 30, 0, 0, time.UTC) - p := waitingWalk(now) - waits := walkWaits{p.ID: {{ID: "novox/mesh-media-catalog@cdcdcdcd0000", State: "unheld", Walk: p.ID, - Says: "held by nothing: its merges' deliveries ended as nothing-to-move (novox/mesh-media-catalog@abababab0000), " + - "yet the walk moves nzbgeek, nzbget — nothing gives it its word"}}} - line := planLineSaid(p, now, tierAtLeast, nil, waits) - if !strings.Contains(line, "ended as nothing-to-move") || !strings.Contains(line, "plans go "+p.ID) { - t.Fatalf("the line reads %q", line) - } -} - -// What holds each waiting walk is asked of mesh-delivery's `stalled` with `walks`, a verb the controller's grant -// already names; only lines naming a walk are kept, by walk. -func TestWhatHoldsTheWaitingWalksIsAskedOfTheDeliveryOwner(t *testing.T) { - was := askDeliveryOwner - defer func() { askDeliveryOwner = was }() - var asked map[string]any - askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) { - if verb != "stalled" { - t.Fatalf("asked %s", verb) - } - asked = args - return json.Marshal([]stalledLine{{ID: "novox/a@1", State: "held", Walk: "plan-1", Says: "held: why"}, - {ID: "novox/b@2", State: "held", Says: "a delivery past its bound, no walk"}}) - } - got := walksWaiting(t.Context(), nil, true) - if asked["walks"] != true { - t.Fatalf("stalled was asked with %v", asked) - } - if len(got) != 1 || len(got["plan-1"]) != 1 || got["plan-1"][0].Says != "held: why" { - t.Fatalf("the waiting walks are %+v", got) - } - if got := walksWaiting(t.Context(), nil, false); got != nil { - t.Fatalf("with no holder on record it asked: %+v", got) - } -} diff --git a/cmd/mesh-controller/issue611_integration_test.go b/cmd/mesh-controller/issue611_integration_test.go deleted file mode 100644 index 31aa2acb..00000000 --- a/cmd/mesh-controller/issue611_integration_test.go +++ /dev/null @@ -1,54 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestAWalkThatMovedNothingSaysWhichModulesMatchedWhatRuns(t *testing.T) { - open := windowed(t) - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - inv := open.inventory - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-one", Repository: "novox/one", Ref: "main", - Module: "one", Commit: "c0", On: "builder", Asked: t0.Add(-time.Hour), - Sources: []inventory.BuildSource{{Paths: []string{"main.go", "go.mod"}}}}); err != nil { - t.Fatal(err) - } - readme := repoMerge("one", "one1aaaaaaaa", t0) - readme.Paths = []string{"README.md"} - hear(t, open, readme, t0.Add(time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - if len(ws) != 1 { - t.Fatalf("walks: %+v", ws) - } - w := ws[0] - if w.State != inventory.PlanDone || len(w.Tiers) != 0 || len(w.Modules) != 0 { - t.Fatalf("the walk reads %s, tiers %v, modules %v; want done with nothing to walk", w.State, w.Tiers, w.Modules) - } - if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "holds none of the 1 changed file(s)") { - t.Fatalf("the walk does not say one matched what runs: %+v", w.Delivery) - } - if !strings.Contains(w.Note, "unchanged") || !strings.Contains(w.Note, "one") { - t.Errorf("the walk's note does not name it: %q", w.Note) - } - - // A merge that changes the build source is walked, and says nothing unchanged. - code := repoMerge("one", "one2bbbbbbbb", t0.Add(3*time.Minute)) - hear(t, open, code, t0.Add(3*time.Minute+time.Second)) - cutAt(t, open, t0.Add(6*time.Minute)) - ws, _ = walks(t, open) - if len(ws) != 2 { - t.Fatalf("walks: %+v", ws) - } - if _, built := ws[0].Modules["one"]; !built || ws[0].Delivery == nil || len(ws[0].Delivery.Unchanged) != 0 { - t.Fatalf("the walk of a change to main.go reads modules %v, delivery %+v", ws[0].Modules, ws[0].Delivery) - } -} diff --git a/cmd/mesh-controller/issue611_test.go b/cmd/mesh-controller/issue611_test.go deleted file mode 100644 index 2851dc55..00000000 --- a/cmd/mesh-controller/issue611_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package main - -import ( - "reflect" - "sort" - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// novox/hq issue 611: mesh-controller #293 changed one test file. Its delivery plan, made while another walk built -// build-agent and route-proxy (issue 606), read them whole and named them; at the cut their recorded build sources -// held none of its files, so its walk moved nothing and its record named no module. mesh-delivery could not tell -// "matched what runs" from "never built" and stopped the delivery, asking the operator about a change already on -// every machine. The walk now says, per module, that its build matched what runs (delivery.unchanged, ADR 0267 §5). - -// The shape of #293: a test file of the controller's repository. The controller, built from the root, and the two -// modules packaging the repository each recorded a build source holding none of it: each is said unchanged, with -// why; a module of another repository is not named. -func TestAMergeSaysTheModulesWhoseBuildSourceItLeftUnchanged(t *testing.T) { - const catalogueRepo = "http://forge.internal:20000/novox/mesh-catalog.git" - const controllerRepo = "http://forge.internal:20000/novox/mesh-controller.git" - entries := []inventory.Entry{ - fromRepo("mesh-controller", controllerRepo, ""), - fromRepo("build-agent", catalogueRepo, "modules/build-agent"), - fromRepo("route-proxy", catalogueRepo, "modules/route-proxy"), - fromRepo("gitea", catalogueRepo, "modules/gitea"), - } - read := map[string][]inventory.ReadRepository{ - "mesh-controller": {{Own: true, Paths: []string{"go.mod", "module.json", "cmd/mesh-controller/", "internal/builder/"}}}, - "build-agent": { - {Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"go.mod", "cmd/mesh-builder/", "internal/builder/"}}, - {Own: true, Paths: []string{"modules/build-agent/Dockerfile", "modules/build-agent/module.json"}}, - }, - "route-proxy": { - {Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"go.mod", "modules/route-proxy/cmd/route-proxy/"}}, - {Own: true, Paths: []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}}, - }, - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "8f1aec0", - Paths: []string{"internal/builder/replay363_test.go"}} - if moved := reachOfMerge(m, entries, read, sharedRepositoryEdges).Moved(); len(moved) != 0 { - t.Fatalf("the fixture moves %v: a test file is in no build source", moved) - } - got := unchangedOfMerge(m, entries, read) - var names []string - for n, why := range got { - names = append(names, n) - if !strings.Contains(why, "holds none of") || !strings.Contains(why, "internal/builder/replay363_test.go") || - !strings.Contains(why, "the build the mesh holds is its build") { - t.Errorf("%s is said unchanged as %q", n, why) - } - } - if want := []string{"build-agent", "mesh-controller", "route-proxy"}; !reflect.DeepEqual(sortedOf(names), want) { - t.Fatalf("said unchanged: %v; want %v", sortedOf(names), want) - } - - // A module whose build source holds a changed file is moved, never said unchanged. - touched := m - touched.Paths = []string{"internal/builder/builder.go"} - if got := unchangedOfMerge(touched, entries, read); got["mesh-controller"] != "" || got["build-agent"] != "" { - t.Errorf("a module the merge moves is said unchanged: %v", got) - } - if got := unchangedOfMerge(touched, entries, read); got["route-proxy"] == "" { - t.Errorf("the route proxy, whose build source holds none of it, is not said unchanged: %v", got) - } - // With no build source recorded, the module is read whole and moved: nothing says it unchanged. - unsaid := map[string][]inventory.ReadRepository{"build-agent": {{Repository: "novox/mesh-controller", Ref: "main"}}} - if got := unchangedOfMerge(m, entries, unsaid); len(got) != 0 { - t.Errorf("modules with no recorded build source are said unchanged: %v", got) - } - // Files not all said: every module is moved, none said unchanged. - truncated := m - truncated.PathsTruncated = true - if got := unchangedOfMerge(truncated, entries, read); len(got) != 0 { - t.Errorf("a merge whose files were not all said says %v unchanged", got) - } -} - -// A module the walk builds anyway — another merge's, or a dependent — is not said unchanged: the modules map keeps -// meaning what the walk builds, and the two never name one module. -func TestAWalkSaysUnchangedOnlyWhatItDoesNotBuild(t *testing.T) { - unchanged := map[string]string{"build-agent": "why a", "route-proxy": "why b"} - got := unchangedLeft(unchanged, map[string]*inventory.PlanModule{"build-agent": {}}) - if !reflect.DeepEqual(got, map[string]string{"route-proxy": "why b"}) { - t.Fatalf("left %v", got) - } - if got := unchangedLeft(map[string]string{"x": "y"}, map[string]*inventory.PlanModule{"x": {}}); got != nil { - t.Fatalf("left %v; want none", got) - } -} - -// Against the store, the path a real merge takes: a merge whose module recorded a build source that holds none -// of its files is cut into a walk done with nothing to walk, whose record names the module unchanged and why. - -func sortedOf(names []string) []string { - out := append([]string{}, names...) - sort.Strings(out) - return out -} diff --git a/cmd/mesh-controller/issue648_test.go b/cmd/mesh-controller/issue648_test.go deleted file mode 100644 index af1ba7d5..00000000 --- a/cmd/mesh-controller/issue648_test.go +++ /dev/null @@ -1,90 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A tier whose module is no move — made from the source every machine running it runs, so nothing is -// sent — is not waited on for a report after its marking (novox/hq issue 648). Its machines answered the -// send that last carried that build before the walk got there, and nothing will make them report again: -// the walk asks the next tier in the same step, rather than "waiting for app on anchor, laptop" for ever. -func TestANoMoveTierWaitsForNoNewReport(t *testing.T) { - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - inv := g.open.inventory - - // app holds the build seat, so a module of the next tier is built by it; since novox/hq ADR 0360 §3 that gates - // nothing. - for _, m := range []catalogue.Manifest{ - {Module: "app", Version: "c2", Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}}, - {Module: "web", Version: "w1"}, - } { - if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", - Path: "modules/" + m.Module, BuiltFrom: m.Version, Head: m.Version, Asked: time.Now().Add(-time.Minute)}); err != nil { - t.Fatal(err) - } - } - // What anchor and laptop were last sent (c1) was made from the source the plan's build of c2 is. - for _, b := range []inventory.Build{ - {ID: "build-1s", Module: "app", Commit: "c1", SourceFingerprint: "src1:app", - Asked: time.Now().Add(-30 * time.Second), At: time.Now().Add(-30 * time.Second)}, - {ID: "build-2s", Module: "app", Commit: "c2", SourceFingerprint: "src1:app", Asked: time.Now(), At: time.Now()}, - } { - b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - // Each machine answered that send, applied, before the walk's tier was built. - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.RecordDoing(ctx, nodeID(t, g.open, n), inventory.Doing{Node: n, - Outcome: inventory.OutcomeApplied, Declared: "d-" + n + "-c1", Applied: 1, - At: time.Now().Add(-10 * time.Second)}); err != nil { - t.Fatal(err) - } - } - built := time.Now().UTC() - p := g.plan(t) - p.Tiers = [][]string{{"app"}, {"web"}} - p.Modules["app"].Build, p.Modules["app"].BuiltAt = "build-2s", &built - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - - advancePlans(ctx, g.open) - - if len(g.sent) != 0 { - t.Fatalf("a no-move build was sent: %v", g.sent) - } - p = g.plan(t) - if s := p.Modules["app"]; s == nil || !strings.HasPrefix(s.Why, "no move") { - t.Fatalf("the build was not marked no move: %+v", s) - } - if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") { - t.Fatalf("the walk waits on a no-move tier for a report nothing will cause: tier %d, %s %q", p.Tier, p.State, p.Note) - } - - // A machine that has not answered its last send is not waited on either: a built-by edge gates no tier - // (novox/hq ADR 0360 §3). Before, the walk waited for app on laptop. - p.Tier, p.State, p.Note = 0, inventory.PlanRolling, "" - if err := inv.RecordSent(ctx, nodeID(t, g.open, "laptop"), "d-laptop-later", map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - advancePlans(ctx, g.open) - p = g.plan(t) - if p.Tier < 1 || strings.Contains(p.Note, "waiting for app") { - t.Fatalf("the walk waits on a built-by edge: tier %d, %q", p.Tier, p.Note) - } -} diff --git a/cmd/mesh-controller/kept_helpers.go b/cmd/mesh-controller/kept_helpers.go new file mode 100644 index 00000000..46e747a5 --- /dev/null +++ b/cmd/mesh-controller/kept_helpers.go @@ -0,0 +1,252 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/micro" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "regexp" + "strconv" + "strings" + "time" +) + +// Small helpers the plan machinery once held and other code still reads. + +// silentCallBound is how long a call to the bus may take before it is called unanswered. +var silentCallBound = 20 * time.Second + +// clipWords keeps a plain sentence within a bound, at a word. +func clipWords(s string, n int) string { + if len(s) <= n { + return s + } + cut := strings.LastIndex(s[:n-1], " ") + if cut <= 0 { + cut = n - 1 + } + return s[:cut] + "…" +} + +// orWords is s, or the words for when it is empty. +func orWords(s, none string) string { + if s == "" { + return none + } + return s +} + +func boundedCall[T any](ctx context.Context, what string, call func(context.Context) (T, error)) (T, error) { + ctx, cancel := context.WithTimeout(ctx, silentCallBound) + defer cancel() + type answer struct { + v T + err error + } + done := make(chan answer, 1) + go func() { v, err := call(ctx); done <- answer{v, err} }() + select { + case a := <-done: + return a.v, a.err + case <-ctx.Done(): + var zero T + return zero, fmt.Errorf("%s: no answer from the bus within %s: %w", what, silentCallBound, ctx.Err()) + } +} + +func sameCommitID(a, b string) bool { + if a == "" || b == "" { + return false + } + if a == b { + return true + } + n := min(len(a), len(b)) + return n >= 7 && a[:n] == b[:n] +} + +// wordsZone is the zone a stalled line's times are said in: the controller's local zone, as every other time in its +// messages. A seam a test replaces, so that no test writes time.Local, which every goroutine of the package reads. +var wordsZone = func() *time.Location { return time.Local } + +// wordsNow is the time the words are said at, which says whether a time needs its day; a seam a test replaces. +var wordsNow = time.Now + +var ( + planIDWord = regexp.MustCompile(`\bplan-[0-9]+\b`) + walkWd = regexp.MustCompile(`\bwalks\b`) + walkOne = regexp.MustCompile(`\bwalk\b`) +) + +// plainRefusalOr is plainRefusal with the sentence to say when the refusal's own words cannot be said plainly. +func plainRefusalOr(refusal, fallback string) string { + r := planIDWord.ReplaceAllString(refusal, "the plan") + r = walkWd.ReplaceAllString(r, "deliveries") + r = walkOne.ReplaceAllString(r, "delivery") + r = strings.TrimSpace(r) + if _, ok := conditions.Plain(r); !ok { + return fallback + } + return r +} + +// askDeliveryOwner asks the holder of the mesh-delivery seat one of the verbs the controller is granted; +// a seam a test replaces. The holder's own refusal is an error naming it. +var askDeliveryOwner = func(ctx context.Context, conn *nats.Conn, verb string, args map[string]any) (json.RawMessage, error) { + granted := false + for _, v := range broker.VerbsTheControllerAsksTheDeliveryOwner { + granted = granted || v.Verb == verb + } + if !granted { + return nil, fmt.Errorf("the controller asks %s.%s, which its grant does not name", catalogue.DeliverySeat, verb) + } + if conn == nil { + return nil, errors.New("this controller is not on the bus") + } + answer, err := link.AskMeshSeatTool(ctx, conn, catalogue.DeliverySeat, verb, args, deliveryOwnerWithin) + if err != nil { + return nil, err + } + if answer.Error != "" { + return nil, fmt.Errorf("%s.%s refused: %s", catalogue.DeliverySeat, verb, answer.Error) + } + return unwrapToolResult(answer.Result), nil +} + +// deliveryOwnerWithin is how long the owner is given to answer. +var deliveryOwnerWithin = 10 * time.Second + +// unwrapToolResult is a tool's answer whatever the runtime wrapped it in: the JSON itself, or the +// protocol's content list holding it as text. +func unwrapToolResult(raw json.RawMessage) json.RawMessage { + var wrapped struct { + Content []struct { + Text string `json:"text"` + } `json:"content"` + } + if json.Unmarshal(raw, &wrapped) == nil && len(wrapped.Content) > 0 && json.Valid([]byte(wrapped.Content[0].Text)) { + return json.RawMessage(wrapped.Content[0].Text) + } + return raw +} + +// servedOnTheBus asks the bus's discovery what every machine's node tools answer and which modules' +// tools are served where. A variable so a test needs no runtime. +var servedOnTheBus = func(ctx context.Context, conn *nats.Conn) (map[string]served, error) { + inbox := conn.NewRespInbox() + sub, err := conn.SubscribeSync(inbox) + if err != nil { + return nil, err + } + defer func() { _ = sub.Unsubscribe() }() + if err := conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil { + return nil, fmt.Errorf("asking the bus who serves what: %w", err) + } + out := map[string]served{} + add := func(node string) served { + s, ok := out[node] + if !ok { + s = served{tools: map[string]bool{}} + } + return s + } + deadline := time.Now().Add(discoveryPatience) + for time.Now().Before(deadline) { + wait, cancel := context.WithTimeout(ctx, discoveryQuiet) + msg, err := sub.NextMsgWithContext(wait) + cancel() + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + break + } + var info micro.Info + if json.Unmarshal(msg.Data, &info) != nil { + continue + } + if info.Name == broker.RuntimeModule { + node := info.Metadata["node"] + if node == "" { + node = info.ID + } + s := add(node) + s.runtime = true + out[node] = s + } + for _, e := range info.Endpoints { + if e.Metadata["kind"] != "tool" || e.Metadata["module"] == "" { + continue + } + node := e.Metadata["node"] + if node == "" { + node = info.ID + } + s := add(node) + s.tools[e.Metadata["module"]] = true + out[node] = s + } + } + return out, nil +} + +// served is what one machine's node tools answered the bus's discovery with. +type served struct { + runtime bool + tools map[string]bool +} + +// theGraph is what the planner reads. +func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry, map[string][]inventory.ReadRepository, + []inventory.Edge, error) { + entries, err := inv.Catalogued(ctx) + if err != nil { + return nil, nil, nil, err + } + read, err := readForPlanning(ctx, inv) + if err != nil { + return nil, nil, nil, err + } + edges, err := inv.Dependencies(ctx) + if err != nil { + return nil, nil, nil, err + } + return entries, read, edges, nil +} + +// refusedWord is the word a build's failure says the reader did not know; empty when it says none. +func refusedWord(failed string) string { + if m := unknownWord.FindStringSubmatch(failed); m != nil { + return m[1] + } + return "" +} + +// unknownWord is how the strict reading of a manifest says a word is not one it knows: encoding/json's own words, which +// the build-agent's answer carries whole. The build's answer has no typed field for it (Failed is the builder's words). +var unknownWord = regexp.MustCompile(`json: unknown field "([^"]+)"`) + +// durationSetting reads a duration as a setting says it: "90s", "10m", or a number of seconds. +func durationSetting(v any) (time.Duration, bool) { + switch t := v.(type) { + case string: + if d, err := time.ParseDuration(strings.TrimSpace(t)); err == nil && d >= 0 { + return d, true + } + if n, err := strconv.ParseFloat(strings.TrimSpace(t), 64); err == nil && n >= 0 { + return time.Duration(n * float64(time.Second)), true + } + case float64: + if t >= 0 { + return time.Duration(t * float64(time.Second)), true + } + } + return 0, false +} diff --git a/cmd/mesh-controller/lean_walk_integration_test.go b/cmd/mesh-controller/lean_walk_integration_test.go deleted file mode 100644 index bc5ebd47..00000000 --- a/cmd/mesh-controller/lean_walk_integration_test.go +++ /dev/null @@ -1,141 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "slices" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) { - judgeLeavesAsTheCore(t) - b := aBacklogOnThree(t) - ctx := t.Context() - said := events(t) - for i := 0; i < 12; i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - for _, n := range []string{"desk", "laptop", "anchor"} { - if !slices.Contains(*said, "send "+n) { - t.Fatalf("%s was not sent: %v", n, *said) - } - } - if slices.Contains(*said, "judge") { - t.Fatalf("a machine was judged by its health: %v", *said) - } - if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) - } -} - -func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) { - judgeLeavesAsTheCore(t) - b := aBacklogOnThree(t) - ctx := t.Context() - inv := b.open.inventory - b.broken["desk"] = true // desk fails what it is sent - advancePlans(ctx, b.open) - for i := 0; i < 12 && b.release(t).Open(); i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") { - t.Fatalf("the others have no verdict of their own: %+v", p.Release) - } - if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) - } - if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" { - t.Fatalf("desk was put back: %v", sent) - } -} - -func TestAReleaseSendKeptAndMadeIsNotMadeAgainAtOnce(t *testing.T) { - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - p, err := releaseBacklog(ctx, b.open, "") - if err != nil || p == nil { - t.Fatalf("no release opened: %v", err) - } - if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending { - t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) - } - if err := inv.SavePlan(ctx, p); err != nil { - t.Fatal(err) - } - // The send is made, and the controller is replaced before it keeps it. - if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 { - t.Fatalf("anchor was sent again: %v", b.sent) - } - g := b.release(t).Release.Gates["anchor"] - if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { - t.Fatalf("anchor is not judged on the send made: %+v", g) - } -} - -func TestAMachineSilentPastItsBoundIsLeftAndSaidAtOnce(t *testing.T) { - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - reporting := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing - b.sent = append(b.sent, names) - current, _ := open.inventory.CurrentBuilds(ctx) - return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep", - map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit}) - } - return reporting(ctx, open, names) - } - t.Cleanup(func() { sendRollout = reporting }) - for i := 0; i < 8; i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - g := p.Release.Gates["laptop"] - if !p.Open() || g == nil || g.Verdict != "" || g.Silent { - t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g) - } - since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) - g.Since = &since - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - p = b.release(t) - // anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7). - if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed { - t.Fatal("a silent machine's build was marked failed") - } - if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { - t.Fatalf("laptop was put back: %v", sent) - } - obs, err := silentObservation(ctx, inv) - if err != nil { - t.Fatal(err) - } - if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) { - t.Fatalf("the silence is not said: %+v", obs) - } -} diff --git a/cmd/mesh-controller/lean_walk_test.go b/cmd/mesh-controller/lean_walk_test.go deleted file mode 100644 index 5f6ad330..00000000 --- a/cmd/mesh-controller/lean_walk_test.go +++ /dev/null @@ -1,116 +0,0 @@ -package main - -import ( - "slices" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq ADR 0360 §4, §5: a walk ends on its machines' reports — every machine sent has reported on its send, -// or is silent past ApplySilentAfter — and names a failed or refused apply; it never waits for a health reading. -func TestAWalkEndsOnItsMachinesReports(t *testing.T) { - now := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC) - sent := now.Add(-time.Minute) - long := now.Add(-inventory.ApplySilentAfter - time.Minute) - at := now.Add(-10 * time.Second) - p := inventory.Plan{Modules: map[string]*inventory.PlanModule{ - "app": {Rest: map[string]inventory.SentDeclaration{ - "ace": {Digest: "d-ace", At: sent}, - "novox": {Digest: "d-novox", At: sent}, - "shanks": {Digest: "d-shanks", At: sent}, - "laptop": {Digest: "d-laptop", At: long}, - }}, - "proxy": {Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: sent}}}, - }} - reports := []inventory.Reported{ - {Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"}, - {Node: "novox", Outcome: inventory.OutcomeFailed, At: &at, Declared: "d-novox"}, - {Node: "shanks", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-before"}, - } - waiting, failed, silent := walkReports(p, reports, now, nil) - if !slices.Equal(waiting, []string{"shanks"}) || !slices.Equal(failed, []string{"novox"}) || - !slices.Equal(silent, []string{"laptop"}) { - t.Fatalf("waiting %v, failed %v, silent %v", waiting, failed, silent) - } - reports[2].Declared = "d-shanks" - if waiting, _, _ := walkReports(p, reports, now, nil); len(waiting) != 0 { - t.Fatalf("the walk still waits once every machine reported: %v", waiting) - } -} - -// withTheGate turns the first-machine gate on for a test of its own path: kept for the walks a controller before -// novox/hq ADR 0360 started. -func withTheGate(t *testing.T) { - t.Helper() - was := gateFirstMachine - gateFirstMachine = true - t.Cleanup(func() { gateFirstMachine = was }) -} - -// Every machine heard from is sent at once and passed on its applied report, with no judging (novox/hq ADR 0360 §7); -// before, the first machine was judged before the rest were sent (ADR 0353 §1). - -// A machine that reports its apply failed is named and fails the release, with nothing put back, and every other -// machine goes on to its own report (novox/hq ADR 0360 §5, §7); before, it was put back (ADR 0353 §3). - -// A release's send kept before it was made, and made, is judged by the controller that takes over without being -// made again (ADR 0353 §8), with every machine sent at once (ADR 0360 §7). - -// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put -// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4); its step also sent anchor, so the release is done, not failed (ADR 0360 §7). - -// Issue 840 (novox/hq ADR 0362 §1-5): a walk's end counts a machine's send of a provisioned module applied only once -// the engine and the machine's provisioner client both said so. -func TestAWalkEndWaitsForTheProvisioner(t *testing.T) { - now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC) - at := now.Add(-10 * time.Second) - plan := func(sentAt time.Time) inventory.Plan { - return inventory.Plan{Modules: map[string]*inventory.PlanModule{"app": {Rest: map[string]inventory.SentDeclaration{ - "ace": {Digest: "d-ace", At: sentAt, Provisioned: map[string]string{"app": "t1"}}, - "g14": {Digest: "d-g14", At: sentAt, Provisioned: map[string]string{"app": "t1"}}, - }}}} - } - reports := []inventory.Reported{ - {Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"}, - {Node: "g14", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-g14"}, - } - says := map[string]inventory.ProvReport{} - prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok } - applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: at} - - // (a) the engine applied, the client has not said: the end waits on both. - waiting, failed, silent := walkReports(plan(now.Add(-time.Minute)), reports, now, prov) - if !slices.Equal(waiting, []string{"ace", "g14"}) || len(failed)+len(silent) != 0 { - t.Fatalf("a: waiting %v, failed %v, silent %v", waiting, failed, silent) - } - // ace's client applied, g14's failed on that build (b): ace done, g14 a failed apply naming the reason. - says["ace/app"] = applied - says["g14/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: at} - waiting, failed, silent = walkReports(plan(now.Add(-time.Minute)), reports, now, prov) - if len(waiting) != 0 || len(silent) != 0 || len(failed) != 1 || !strings.HasPrefix(failed[0], "g14 (") || - !strings.Contains(failed[0], "no value") { - t.Fatalf("b: waiting %v, failed %v, silent %v", waiting, failed, silent) - } - // (c) no client report past ApplySilentAfter: named silent, never applied. - delete(says, "g14/app") - waiting, failed, silent = walkReports(plan(now.Add(-inventory.ApplySilentAfter-time.Minute)), reports, now, prov) - if len(waiting) != 0 || len(failed) != 0 || !slices.Equal(silent, []string{"g14"}) { - t.Fatalf("c: waiting %v, failed %v, silent %v", waiting, failed, silent) - } - // both said it: nothing waits. - says["g14/app"] = applied - if waiting, failed, silent = walkReports(plan(now.Add(-time.Minute)), reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 { - t.Fatalf("both said: waiting %v, failed %v, silent %v", waiting, failed, silent) - } - // (d) a send handing nothing to the provisioner is the engine's report alone, as before. - bare := plan(now.Add(-time.Minute)) - bare.Modules["app"].Rest = map[string]inventory.SentDeclaration{"ace": {Digest: "d-ace", At: now.Add(-time.Minute)}} - if waiting, failed, silent = walkReports(bare, reports, now, func(string, string) (inventory.ProvReport, bool) { - return inventory.ProvReport{}, false - }); len(waiting)+len(failed)+len(silent) != 0 { - t.Fatalf("d: waiting %v, failed %v, silent %v", waiting, failed, silent) - } -} diff --git a/cmd/mesh-controller/lone_merge_test.go b/cmd/mesh-controller/lone_merge_test.go deleted file mode 100644 index e5850d76..00000000 --- a/cmd/mesh-controller/lone_merge_test.go +++ /dev/null @@ -1,338 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "errors" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// The tests of novox/hq issue 525's window skip: a merge waits for the merge window only when a batch assembling -// holds a merge related to it, or its delivery group may have a member still to merge (ADR 0304 decision 1, ADR -// 0306 decisions 2 and 3). Walks stay one at a time. - -// A merge from no head that shares nothing, with no walk open, is cut into its walk when it is heard: no window. -func TestALoneMergeSharingNothingIsCutAtOnce(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - hear(t, open, lone(repoMerge("one", "c1", t0)), t0) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 0 || !ws[0].Open() || ws[0].Commit != "c1" { - t.Fatalf("a lone merge sharing nothing: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs)) - } - if ws[0].Times == nil || ws[0].Times.WindowClosed == nil || !ws[0].Times.WindowClosed.Equal(t0) { - t.Fatalf("its window closed at %+v; want when it was heard, %s", ws[0].Times, t0) - } -} - -// A merge related to the batch assembling joins it and waits for the window, which its coming restarts: by its -// repository, by a module in common (a dependent of what the batch moves), by an edge of the module graph between -// their modules (one that widens nothing), or by its delivery group's head. -func TestARelatedMergeJoinsTheBatchAndWaitsForTheWindow(t *testing.T) { - ctx := t.Context() - for _, c := range []string{"same repository", "a module in common", "an edge of the graph", "the same group head"} { - t.Run(c, func(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/c1") - asksWithPaths(t) - inv := open.inventory - first := repoMerge("one", "c1", t0) - second := lone(repoMerge("two", "c2", t0.Add(10*time.Second))) - why := map[string]string{"same repository": "both change novox/one", "a module in common": "both move two", - "an edge of the graph": "two depends on builder (built-by)", - "the same group head": "both are members of group " + first.Head}[c] - switch c { - case "same repository": - second = lone(repoMerge("one", "c2", t0.Add(10*time.Second))) - case "a module in common": - // two is built on one: a merge of one moves two too. - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "two", Version: "1", - Build: &catalogue.Build{On: []catalogue.BuildsOn{{Arg: "BASE", Module: "one"}}}}, - inventory.Source{Repository: "novox/two", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - case "an edge of the graph": - // two is built by the build machine: an edge that orders and widens nothing. - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "builder", Version: "1", - Claims: []catalogue.Claim{{Name: "mesh-build-machine", Scope: catalogue.ScopeMesh}}}, - inventory.Source{Repository: "novox/builder", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - first = repoMerge("builder", "c1", t0) - case "the same group head": - second = fromHead(repoMerge("two", "c2", t0.Add(10*time.Second)), first.Head) - } - hear(t, open, first, t0) - said := captured(t, func() error { - return (following{open: open}).hearMerge(ctx, second, t0.Add(10*time.Second)) - }) - if !strings.Contains(said, "("+why+")") { - t.Fatalf("the merge was not said related by %q: %s", why, said) - } - ws, bs := walks(t, open) - if len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 { - t.Fatalf("the related merge: %d walk(s), batches %+v", len(ws), bs) - } - if bs[0].Delivery.Batch.CutAtOnce { - t.Fatal("a batch a related merge joined is cut at once") - } - cutAt(t, open, t0.Add(99*time.Second)) - if ws, _ := walks(t, open); len(ws) != 0 { - t.Fatalf("cut before the window the related merge restarted closed: %+v", ws) - } - cutAt(t, open, t0.Add(100*time.Second)) - ws, bs = walks(t, open) - if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 { - t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs)) - } - }) - } -} - -// A merge from a head whose delivery group has a member still to merge waits for the window, even with no batch -// assembling, and says why; its sibling joins the same batch by their head (ADR 0306 decision 2). A merge from a -// head that names no group shares nothing and is cut at once, as one from no head is. -func TestAMergeOfAGroupWaitsEvenAlone(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - states := map[string]string{"novox/one": "ready", "novox/two": "ready", "novox/elsewhere": "ready"} - groupStates(t, "feat/x", states) - hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/x"), t0) - ws, bs := walks(t, open) - if len(ws) != 0 || len(bs) != 1 || bs[0].Delivery.Batch.CutAtOnce || windowClosed(bs[0], t0) { - t.Fatalf("a group's merge alone: %d walk(s), batches %+v", len(ws), bs) - } - if words := batchWords(bs[0], t0); !strings.Contains(words, "group feat/x has a member still to merge: novox/elsewhere (ready), novox/two (ready)") { - t.Fatalf("it reads %q", words) - } - // Its members merge: the batch is held for the last, and the window runs again from that merge. - delete(states, "novox/elsewhere") - states["novox/two"] = "merged" - hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(5*time.Second)), "feat/x"), t0.Add(5*time.Second)) - if _, bs := walks(t, open); len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 { - t.Fatalf("the group's second member did not join the first: %+v", bs) - } - cutAt(t, open, t0.Add(94*time.Second)) - if ws, _ := walks(t, open); len(ws) != 0 { - t.Fatalf("cut before its window closed: %+v", ws) - } - cutAt(t, open, t0.Add(95*time.Second)) - ws, bs = walks(t, open) - if len(ws) != 1 || len(bs) != 0 || ws[0].CommitOf("novox/one") != "c1" || ws[0].CommitOf("novox/two") != "c2" { - t.Fatalf("at its window: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs)) - } -} - -// A merge from a head that names no group, sharing nothing, with no walk open, is cut at once. -func TestAHeadedMergeInNoGroupIsCutAtOnce(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - hear(t, open, fromHead(repoMerge("one", "c1", t0), "fix/alone"), t0) - if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 || !ws[0].Open() { - t.Fatalf("a headed merge in no group: %d walk(s), %d batch(es)", len(ws), len(bs)) - } -} - -// When mesh-delivery cannot say a head's group, the merge waits for the window, the safe side, and its batch says -// why. -func TestAGroupNobodyCanSayWaits(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - was := deliveryGroupOf - deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { - return nil, false, errors.New("nothing serves mesh-delivery.groups") - } - t.Cleanup(func() { deliveryGroupOf = was }) - hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/x"), t0) - ws, bs := walks(t, open) - if len(ws) != 0 || len(bs) != 1 || bs[0].Delivery.Batch.CutAtOnce { - t.Fatalf("a merge whose group nobody could say: %d walk(s), batches %+v", len(ws), bs) - } - if words := batchWords(bs[0], t0); !strings.Contains(words, "mesh-delivery did not say whether group feat/x has a "+ - "member still to merge (nothing serves mesh-delivery.groups): it waits for the window, the safe side") { - t.Fatalf("it reads %q", words) - } -} - -// A member counts as still to merge only while its pull request is open, and not once this controller keeps its -// merge: a group whose other members are merged or closed makes nobody wait. -func TestAGroupsMergedOrClosedMembersMakeNobodyWait(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - ctx := t.Context() - members := []groupMember{{Repository: "novox/one", State: "ready"}, {Repository: "novox/two", State: "published"}, - {Repository: "novox/three", State: "stopped"}} - was := deliveryGroupOf - deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { return members, true, nil } - t.Cleanup(func() { deliveryGroupOf = was }) - m := fromHead(repoMerge("one", "c1", t0), "feat/x") - if why, err := groupWait(ctx, open.inventory, m, true, false); err != nil || why != "" { - t.Fatalf("the group's other members are merged or closed, yet: %q %v", why, err) - } - members = append(members, groupMember{Repository: "novox/four", State: "checked"}) - if why, _ := groupWait(ctx, open.inventory, m, true, false); !strings.Contains(why, "novox/four (checked)") { - t.Fatalf("an open member: %q", why) - } - // novox/four's merge is kept in a batch already: merged, whatever mesh-delivery says yet. - hear(t, open, fromHead(repoMerge("two", "c4", t0), "feat/x"), t0) - members[3].Repository = "novox/two" - if why, _ := groupWait(ctx, open.inventory, m, true, false); why != "" { - t.Fatalf("a member this controller keeps the merge of still waits: %q", why) - } - if got := memberOf("novox/mesh-sdk@d41d255f28ad (held)"); got != (groupMember{Repository: "novox/mesh-sdk", State: "held"}) { - t.Fatalf("a member reads %+v", got) - } -} - -// A merge related to two batches assembling folds the newer into the older, which answers every merge of both -// and names nothing twice; the folded one names the batch that took it over. -func TestAMergeRelatedToTwoBatchesFoldsThem(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "feat/a", "feat/b") - asksWithPaths(t) - ctx := t.Context() - hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/a"), t0) - hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(5*time.Second)), "feat/b"), t0.Add(5*time.Second)) - _, bs := walks(t, open) - if len(bs) != 2 { - t.Fatalf("two merges sharing nothing: %d batch(es)", len(bs)) - } - older, newer := bs[1], bs[0] // newest first - // Of one's repository and from two's head: related to both. - hear(t, open, fromHead(repoMerge("one", "c3", t0.Add(10*time.Second)), "feat/b"), t0.Add(10*time.Second)) - _, bs = walks(t, open) - if len(bs) != 1 || bs[0].ID != older.ID || len(bs[0].Delivery.Merges) != 3 { - t.Fatalf("after a merge related to both: batches %+v; want %s answering three merges", bs, older.ID) - } - folded, err := open.inventory.PlanByID(ctx, newer.ID) - if err != nil || folded.State != inventory.PlanSuperseded || folded.Delivery == nil || - folded.Delivery.TakenOverBy != older.ID { - t.Fatalf("the newer batch reads %s %+v (%v)", folded.State, folded.Delivery, err) - } - if left, _ := open.inventory.MergesOf(ctx, newer.ID); len(left) != 0 { - t.Fatalf("the folded batch still answers %+v", left) - } - cutAt(t, open, t0.Add(100*time.Second)) - ws, bs := walks(t, open) - var cut []inventory.Plan - for _, w := range ws { - if w.State != inventory.PlanSuperseded { - cut = append(cut, w) - } - } - if len(cut) != 1 || len(bs) != 0 || cut[0].ID != older.ID || cut[0].CommitOf("novox/one") != "c3" || - cut[0].CommitOf("novox/two") != "c2" { - t.Fatalf("the folded batch's walk: %+v, %d batch(es)", cut, len(bs)) - } -} - -// With a walk open that it is related to (the same repository, issue 973), a batch cut at once still waits for -// it, said queued behind it (ADR 0276 decision 4, ADR 0407 decision 4); it is cut once that walk ended, with no -// window to wait for. -func TestACutAtOnceBatchQueuesBehindAnOpenWalk(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/c1") - asksWithPaths(t) - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(90*time.Second)) - ws, _ := walks(t, open) - if len(ws) != 1 || !ws[0].Open() { - t.Fatalf("no walk open: %+v", ws) - } - first := ws[0] - heard := t0.Add(100 * time.Second) - hear(t, open, lone(repoMerge("one", "c2", heard)), heard) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID || - !bs[0].Delivery.Batch.CutAtOnce { - t.Fatalf("a lone merge during a walk: %d walk(s), batches %+v", len(ws), bs) - } - if !strings.HasPrefix(batchWords(bs[0], heard), "queued behind "+first.ID) { - t.Fatalf("it reads %q", batchWords(bs[0], heard)) - } - first.State = inventory.PlanDone - if err := open.inventory.SavePlan(t.Context(), &first); err != nil { - t.Fatal(err) - } - cutAt(t, open, heard.Add(time.Second)) - ws, bs = walks(t, open) - if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" { - t.Fatalf("the queued batch was not cut once the walk ended: %+v %+v", ws, bs) - } - if ws[0].Times == nil || ws[0].Times.WindowClosed == nil || !ws[0].Times.WindowClosed.Equal(heard) { - t.Fatalf("its window closed at %+v; want when its merge was heard, %s", ws[0].Times, heard) - } -} - -// A stopped walk's other merges are walked again in a batch they are related to, and a batch of their own when -// they share nothing with the one assembling (ADR 0299, ADR 0304 decision 1): never in a batch of another -// change, whose window the stop would close. -func TestAStopWalksTheOthersAgainInABatchOfTheirOwnWhenTheyShareNothing(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/cat1aaaaaaaa") - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - cat := catalogueMerge("cat1aaaaaaaa", "app", t0) - one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) - hear(t, open, cat, t0.Add(time.Second)) - hear(t, open, one, t0.Add(6*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - walk := ws[0] - // Related to the walk by its repository (queued behind it), and to nothing the stop walks again. - hear(t, open, catalogueMerge("cat2cccccccc", "notes", t0.Add(3*time.Minute)), t0.Add(3*time.Minute)) - _, bs := walks(t, open) - if len(bs) != 1 { - t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs)) - } - other := bs[0] - _, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second)) - if err != nil { - t.Fatal(err) - } - if said.WalkedAgainBy == "" || said.WalkedAgainBy == other.ID { - t.Fatalf("walked again by %q; want a batch of its own, not %s", said.WalkedAgainBy, other.ID) - } - b, _ := open.inventory.PlanByID(ctx, other.ID) - if b.Delivery.Batch == nil || b.Delivery.Batch.Closed || len(b.Delivery.Merges) != 1 { - t.Fatalf("the other change's batch reads %+v; want it untouched by the stop", b.Delivery) - } -} - -// A catalogue batch's cut folds in every walk waiting for its delivery's word (cutBatch's carry), so what that -// walk moves is the batch's too: a lone merge related to the waiting walk joins the batch assembling and waits for -// its window, rather than being cut at once as sharing nothing with what the batch's own merges move. -func TestALoneMergeRelatedToAWalkTheCutFoldsJoinsTheBatch(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/c3") - asksWithPaths(t) - ctx := t.Context() - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(90*time.Second)) - if ws, _ := walks(t, open); len(ws) != 1 || !ws[0].Waiting() { - t.Fatalf("the walk does not wait for its word: %+v", ws) - } - hear(t, open, repoMerge("three", "c3", t0.Add(2*time.Minute)), t0.Add(2*time.Minute)) - hear(t, open, lone(repoMerge("one", "c2", t0.Add(2*time.Minute+5*time.Second))), t0.Add(2*time.Minute+5*time.Second)) - _, bs := walks(t, open) - if len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 || bs[0].Delivery.Batch.CutAtOnce { - t.Fatalf("a lone merge related to the waiting walk: batches %+v", bs) - } -} diff --git a/cmd/mesh-controller/machine_network_integration_test.go b/cmd/mesh-controller/machine_network_integration_test.go deleted file mode 100644 index 0a18cf2d..00000000 --- a/cmd/mesh-controller/machine_network_integration_test.go +++ /dev/null @@ -1,266 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "os" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -func TestARewrittenResolverFileIsRaisedFromTheStatementAndClearedWhenWrittenBack(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv, k := open.inventory, conditionsFrom - say := func(at time.Time, n *link.NetworkHealth) { - t.Helper() - if err := stateHealth(ctx, inv, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil { - t.Fatal(err) - } - } - openKeys := func() []string { - t.Helper() - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - var keys []string - for _, c := range list { - keys = append(keys, c.Key) - } - return keys - } - healthy := &link.NetworkHealth{State: link.StateHealthy, Since: h0, Parts: []link.NetworkPart{ - {Part: link.PartResolvConf, State: link.StateHealthy, Since: h0}}} - rewritten := &link.NetworkHealth{State: link.StateUnhealthy, Since: h0.Add(time.Minute), Parts: []link.NetworkPart{ - {Part: link.PartResolvConf, State: link.StateUnhealthy, Reason: rewrittenByVPN.Reason, Said: rewrittenByVPN.Said, - Writer: "FortiClient", Owner: "networkmanager", Since: h0.Add(time.Minute)}, - {Part: link.PartNames, State: link.StateUnhealthy, Reason: namesThroughVPN.Reason, Said: namesThroughVPN.Said, - Toward: namesThroughVPN.Toward, Since: h0.Add(time.Minute)}}} - - say(h0, healthy) - if keys := openKeys(); len(keys) != 0 { - t.Fatalf("a healthy network raised %v", keys) - } - say(h0.Add(time.Minute), rewritten) - if keys := openKeys(); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" { - t.Fatalf("the rewrite raised %v", keys) - } - kept, had, err := inv.HealthOf(ctx, "laptop") - if err != nil || !had || kept.Network == nil || kept.Network.Parts[0].Writer != "FortiClient" { - t.Fatalf("the statement's network was not kept: %+v %v", kept.Network, err) - } - if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "FortiClient") || - !strings.Contains(lines, "unhealthy resolv-conf") { - t.Fatalf("node show says:\n%s", lines) - } - say(h0.Add(2*time.Minute), healthy) - if keys := openKeys(); len(keys) != 0 { - t.Fatalf("written back, still open: %v", keys) - } - // An engine older than the judging says no network: nothing raised, and node show says it is not known. - say(h0.Add(3*time.Minute), nil) - kept, had, _ = inv.HealthOf(ctx, "laptop") - if keys := openKeys(); len(keys) != 0 || kept.Network != nil { - t.Fatalf("an older engine: %v %+v", keys, kept.Network) - } - if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "older than that judging") { - t.Fatalf("node show says:\n%s", lines) - } -} - -func TestTheDrillsStatementsRaiseAndClearTheRewrite(t *testing.T) { - raw, err := os.ReadFile("testdata/network-drill.json") - if err != nil { - t.Fatal(err) - } - var said []link.Health - if err := json.Unmarshal(raw, &said); err != nil { - t.Fatal(err) - } - open := aMesh(t) - ctx := t.Context() - k := conditionsFrom - var raisedAt []int - for i, h := range said { - if err := stateHealth(ctx, open.inventory, k, "laptop", h, h.At); err != nil { - t.Fatal(err) - } - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - for _, c := range list { - if c.Key != "machine.laptop.networkmanager.rewritten" || !strings.Contains(c.Summary, "(FortiClient)") { - t.Fatalf("statement %d raised %s: %s", i, c.Key, c.Summary) - } - raisedAt = append(raisedAt, i) - } - } - // Five statements: healthy, healthy, one failing look (still healthy), unhealthy, written back. - if len(raisedAt) != 1 || raisedAt[0] != 3 { - t.Fatalf("the rewrite was open after statements %v; want after the fourth alone", raisedAt) - } -} - -func aNetwork(state string, parts ...inventory.NetworkPart) *inventory.NetworkHealth { - for i := range parts { - if parts[i].State == "" { - parts[i].State = link.StateUnhealthy - } - parts[i].Since = h0 - } - return &inventory.NetworkHealth{State: state, Since: h0, Parts: parts} -} - -func TestAResolverFileRewrittenIsOneFindingNamingItsWriterAndWhatItCosts(t *testing.T) { - obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateHealthy), - "laptop": aNetwork(link.StateUnhealthy, rewrittenByVPN, namesThroughVPN), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" { - t.Fatalf("want one finding, the rewrite, got %v", keys) - } - o := obs[0] - for _, want := range []string{"laptop", "rewritten by another program (FortiClient)", "mesh names do not resolve", - "next reconcile"} { - if !strings.Contains(o.Summary, want) { - t.Errorf("the summary does not say %q: %s", want, o.Summary) - } - } - if strings.Contains(o.Summary, "172.16.") || strings.Contains(o.Summary, "/etc/") { - t.Errorf("an address or a path reached the summary: %s", o.Summary) - } - if !strings.Contains(o.Said, "172.16.5.5") || o.Severity != conditions.Warning || o.Machine != "laptop" { - t.Errorf("the evidence or the severity is wrong: %+v", o) - } -} - -func TestOneMachineFailingTowardAHealthyHubIsItsOwn(t *testing.T) { - obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateHealthy), - "laptop": aNetwork(link.StateUnhealthy, tunnelDown), - "spare": aNetwork(link.StateHealthy), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" { - t.Fatalf("want the laptop's own, got %v", keys) - } - if obs[0].Severity != conditions.Warning { - t.Fatalf("a laptop's own tunnel is a warning, got %s", obs[0].Severity) - } -} - -func TestTwoMachinesThatCannotReachTheHubAreSaidOnceAtTheHub(t *testing.T) { - obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateHealthy), - "laptop": aNetwork(link.StateUnhealthy, tunnelDown), - "spare": aNetwork(link.StateUnhealthy, tunnelDown), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.unreachable" { - t.Fatalf("want one condition at the hub, got %v", keys) - } - o := obs[0] - if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop, spare") || len(o.Also) != 2 { - t.Fatalf("the hub's condition is %+v", o) - } -} - -func TestWhatPointsAtASilentHubIsHeldUnderItsSilence(t *testing.T) { - f := netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, tunnelDown)}) - f.silent["anchor"] = true - if obs := networkObservations(f); len(obs) != 0 { - t.Fatalf("the hub's silence says it; got %v", keysOf(obs)) - } -} - -func TestAHubWhoseOwnNetworkIsUnhealthyListsWhoCannotReachIt(t *testing.T) { - obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateUnhealthy, noRoute), - "laptop": aNetwork(link.StateUnhealthy, tunnelDown), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.network" { - t.Fatalf("want the hub's own, holding the laptop's, got %v", keys) - } - if o := obs[0]; o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop cannot reach it") { - t.Fatalf("the hub's condition is %+v", o) - } -} - -func TestOneMachineFailingAHealthyMeshResolverIsItsOwnAndTwoAreTheResolvers(t *testing.T) { - silentResolver := inventory.NetworkPart{Part: link.PartNames, Reason: "1 of its 2 resolvers do not answer as the mesh's do", - Said: "10.77.0.3 — no answer within 1s", Toward: []string{"10.77.0.3"}} - obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateHealthy), "spare": aNetwork(link.StateHealthy), - "laptop": aNetwork(link.StateUnhealthy, silentResolver), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" { - t.Fatalf("one machine alone: want its own, got %v", keys) - } - obs = networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateUnhealthy, silentResolver), "spare": aNetwork(link.StateHealthy), - "laptop": aNetwork(link.StateUnhealthy, silentResolver), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.spare.unreachable" { - t.Fatalf("two machines: want it said once at the resolver's machine, got %v", keys) - } - if !strings.Contains(obs[0].Summary, "its resolver") { - t.Fatalf("the resolver's machine is said %s", obs[0].Summary) - } -} - -func TestTheControlNodeAndTheBusAreUrgent(t *testing.T) { - f := netFacts(map[string]*inventory.NetworkHealth{"anchor": aNetwork(link.StateUnhealthy, rewrittenByVPN)}) - if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent { - t.Fatalf("the control node's rewritten file: %+v", obs) - } - bus := inventory.NetworkPart{Part: link.PartBus, Reason: "the bus cannot be reached", Said: "no link"} - f = netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, bus, noRoute)}) - if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent { - t.Fatalf("the bus unreachable from the laptop: %+v", obs) - } -} - -func TestAnEngineThatSaysNothingOfItsNetworkRaisesNothing(t *testing.T) { - f := netFacts(map[string]*inventory.NetworkHealth{"laptop": nil, "anchor": aNetwork(link.StateHealthy)}) - if obs := networkObservations(f); len(obs) != 0 { - t.Fatalf("got %v", keysOf(obs)) - } -} - -// The gate: a resolver file another program rewrote waits the judging rather than failing it at the -// bound; the same, when the send moved the module whose file it is, is that module's; a machine's own -// network fault holds the machine as a whole, as before. -func TestTheGateWaitsOnARewriteItDidNotMakeAndHoldsTheOwnerOnOneItMoved(t *testing.T) { - since := h0 - rewrite := conditions.Condition{Key: "machine.laptop.networkmanager.rewritten", Kind: kindNetworkRewritten, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop.networkmanager", Machine: "laptop"}, - Summary: "the resolver file was rewritten", Raised: since.Add(time.Minute), Source: sourceNetwork} - f := gateFacts{judged: true, open: []conditions.Condition{rewrite}} - if w := aboutTheMachine("laptop", []string{"letta"}, since, f); w.waiting == "" || w.whole != "" || len(w.on) != 0 { - t.Fatalf("a rewrite the send did not make: %+v", w) - } - if w := aboutTheMachine("laptop", []string{"networkmanager", "letta"}, since, f); w.on["networkmanager"] == "" || - w.on["letta"] != "" || w.waiting != "" { - t.Fatalf("a rewrite of the file a moved module owns: %+v", w) - } - own := conditions.Condition{Key: "machine.laptop.network", Kind: kindMachineNetwork, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"}, - Summary: "laptop's network is not healthy", Raised: since.Add(time.Minute), Source: sourceNetwork} - if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true, - open: []conditions.Condition{own}}); w.whole == "" || w.waiting != "" { - t.Fatalf("the machine's own network: %+v", w) - } - unreachable := conditions.Condition{Key: "machine.anchor.unreachable", Kind: kindNetworkUnreachable, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor", Also: []string{"laptop", "spare"}}, - Summary: "anchor cannot be reached", Raised: since.Add(time.Minute), Source: sourceNetwork} - if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true, - open: []conditions.Condition{unreachable}}); w.waiting == "" || w.whole != "" { - t.Fatalf("a machine that cannot reach the hub: %+v", w) - } -} diff --git a/cmd/mesh-controller/machine_units_integration_test.go b/cmd/mesh-controller/machine_units_integration_test.go deleted file mode 100644 index fd9017e0..00000000 --- a/cmd/mesh-controller/machine_units_integration_test.go +++ /dev/null @@ -1,125 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" -) - -func TestAFailedUnitIsTheModulesConditionAndTheMachinesOwnAreOneFinding(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv, k := open.inventory, conditionsFrom - openConditions := func() map[string]conditions.Condition { - t.Helper() - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - out := map[string]conditions.Condition{} - for _, c := range list { - out[c.Key] = c - } - return out - } - say := func(at time.Time, h link.Health) { - t.Helper() - if err := stateHealth(ctx, inv, k, "laptop", h, at); err != nil { - t.Fatal(err) - } - } - - say(h0, desktop(h0)) - got := openConditions() - machine, raised := got["machine.laptop.units"] - if !raised || len(got) != 1 { - t.Fatalf("the machine's own failed units are one finding at once (the engine looked twice): %v", got) - } - for _, unit := range []string{"mnt-recalbox.mount", "storage-media.mount", "greenclip.service"} { - if !strings.Contains(machine.Summary, unit) { - t.Errorf("the finding names %s: %s", unit, machine.Summary) - } - } - if strings.Contains(machine.Summary, "openrazer") || machine.Severity != conditions.Warning { - t.Errorf("the module's unit is the module's, and the machine's a warning: %+v", machine) - } - - say(h0.Add(time.Minute), desktop(h0.Add(time.Minute))) - got = openConditions() - module, raised := got["module.openrazer.laptop.unhealthy"] - if !raised || !strings.Contains(module.Summary, "openrazer-daemon.service") || - !strings.Contains(module.Summary, "account's own service manager") { - t.Fatalf("the module's failed unit raises its condition on the second statement, naming the unit: %+v", got) - } - - kept, had, err := inv.HealthOf(ctx, "laptop") - if err != nil || !had || kept.Units == nil || len(kept.Units.Failed) != 3 { - t.Fatalf("the statement's units were not kept: %+v %v", kept.Units, err) - } - lines := strings.Join(unitsLines(kept, had, h0), "\n") - for _, want := range []string{"degraded", "storage-media.mount (system) — no module places it", - "openrazer-daemon.service — openrazer's"} { - if !strings.Contains(lines, want) { - t.Errorf("node show does not say %q:\n%s", want, lines) - } - } - - // An older engine says nothing of its units: nothing cleared on its word. - older := desktop(h0.Add(2 * time.Minute)) - older.Units = nil - say(h0.Add(2*time.Minute), older) - if _, still := openConditions()["machine.laptop.units"]; !still { - t.Fatal("a statement saying nothing of units cleared the machine's finding") - } - - // The operator mends the mounts and removes the leftover link; the module's unit still fails. - mended := desktop(h0.Add(3 * time.Minute)) - mended.Units.Failed = []link.FailedUnit{} - say(h0.Add(3*time.Minute), mended) - got = openConditions() - if _, still := got["machine.laptop.units"]; still { - t.Fatalf("no failed unit of the machine's own, still open: %v", got) - } - if _, still := got["module.openrazer.laptop.unhealthy"]; !still { - t.Fatalf("the module's unit still fails, and its condition stands: %v", got) - } - - // And the module's unit mended: running, nothing open. - well := link.Health{Contract: link.ReadinessContract, At: h0.Add(4 * time.Minute), - Units: &link.UnitsHealth{State: link.UnitsRunning, Failed: []link.FailedUnit{}}} - say(h0.Add(4*time.Minute), well) - if got = openConditions(); len(got) != 0 { - t.Fatalf("a running service manager leaves nothing open: %v", got) - } -} - -// desktop is the workstation as found on 2026-10-08, with the mesh's names: the Razer daemon's packaged -// user unit, the module's; two mounts of the machine's own table and a unit a removed package left -// behind, the machine's. -func desktop(at time.Time) link.Health { - return link.Health{Contract: link.ReadinessContract, At: at, - Resources: []link.ResourceHealth{{Module: "openrazer", Resource: "openrazer.daemon", Kind: link.KindUnit, - Target: "openrazer-daemon.service", State: link.StateUnhealthy, - Reason: "failed in the account's own service manager (exit-code)", Since: h0, Streak: 2}}, - Units: &link.UnitsHealth{State: link.UnitsDegraded, Failed: []link.FailedUnit{ - {Unit: "mnt-recalbox.mount", Scope: "system", Load: "loaded", Result: "exit-code", Since: h0}, - {Unit: "storage-media.mount", Scope: "system", Load: "loaded", Result: "timeout", Since: h0}, - {Unit: "greenclip.service", Scope: "user", Load: "not-found", Result: "start-limit-hit", Since: h0}, - }}} -} - -func TestTheGateNeverHoldsASendOnTheMachinesOwnUnits(t *testing.T) { - since := h0 - units := conditions.Condition{Key: "machine.laptop.units", Kind: kindMachineUnits, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"}, - Summary: "laptop's service manager is degraded", Raised: since.Add(time.Minute), Source: sourceUnits} - if w := aboutTheMachine("laptop", []string{"openrazer"}, since, gateFacts{judged: true, - open: []conditions.Condition{units}}); w.whole != "" || w.waiting != "" || len(w.on) != 0 { - t.Fatalf("the machine's own failed units held a send: %+v", w) - } -} diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 37e6ceaf..c7a70520 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -63,15 +63,6 @@ func run() error { // 45 §6), so the next controller takes it at once rather than after its age. defer theLease.release() - // Every walk a verb keeps is said on the bus, whichever verb kept it (novox/hq ADR 0308 decision 1): the save - // that starts or ends a walk keeps its mark, and no later save says it again. The serving controller says its - // own saves. - if args[0] != "serve" { - said, done := sayingWhatAVerbKept(ctx) - inventory.PlanSaved = said - defer done() - } - switch args[0] { case "switch": return switchCommand(ctx, args[1:]) @@ -94,7 +85,8 @@ func run() error { return rehearseCommand(ctx, args[1:]) case "builds": return buildsCommand(ctx, args[1:]) - case "forget-checks", "queue", "cancel", "clear", "kill", "replay", "pause", "resume": + case "forget-checks", "queue", "cancel", "clear", "kill", "replay", "pause", "resume", "plans", "plan", "push", "deliver", + "delivery-stop", "delivery-backups-again", "delivery-walks", "delivery-order", "delivery", "upgrade": return removedByName(args[0]) case "try": return tryCommand(ctx, args[1:]) @@ -113,11 +105,6 @@ func run() error { return movePlacementCommand(ctx, args[1:]) case "move-config": return moveConfigCommand(ctx, args[1:]) - case "plans": - return plansCommand(ctx, args[1:]) - case "delivery": - // The verbs the delivery's owner asks with (novox/hq ADR 0239). - return deliveryCommand(ctx, args[1:]) case "pin": return pinCommand(ctx, args[1:], true) case "unpin": @@ -140,13 +127,9 @@ func run() error { // The facts snapshot a merge check is fed (novox/hq to-be 45 §9). case "facts": return factsCommand(ctx, args[1:]) - case "upgrade": - return upgradeCommand(ctx, args[1:]) // The bus as a planned step (novox/hq to-be 45 §8, ADR 0236). case "bus": return busCommand(ctx, args[1:]) - case "declare": - return declare(ctx, args[1:]) case "overlay": return overlayCommand(ctx, args[1:]) case "module": @@ -165,10 +148,6 @@ func run() error { return secretCommand(ctx, args[1:]) case "operator": return operatorCommand(ctx, args[1:]) - case "plan": - return planCommand(ctx, args[1:]) - case "push": - return pushCommand(ctx, args[1:]) case "rollout": return rolloutCommand(ctx, args[1:]) case "seats": diff --git a/cmd/mesh-controller/members_file_test.go b/cmd/mesh-controller/members_file_test.go deleted file mode 100644 index 9c5b0309..00000000 --- a/cmd/mesh-controller/members_file_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "os" - "slices" - "strings" - "testing" -) - -// aLongGroup is members as JSON over 200 KB: 1666 paths of the shape of mesh-delivery #5's change (novox/hq issue 1272). -func aLongGroup(t *testing.T) string { - t.Helper() - var paths []string - for i := 0; i < 1666; i++ { - paths = append(paths, fmt.Sprintf("modules/mesh-delivery-forge-bridge/vendor/github.com/nats-io/nats-server/v2/server/certstore/certstore_windows_file_%04d.go", i)) - } - body, err := json.Marshal([]map[string]any{{"id": "novox/mesh-delivery#5", "paths": paths}, {"id": "novox/mesh-catalog#374"}}) - if err != nil { - t.Fatal(err) - } - if len(body) < 200<<10 { - t.Fatalf("the members are only %d bytes", len(body)) - } - return string(body) -} - -func readBackFromTheCommand(t *testing.T, argv []string, members string) { - t.Helper() - for _, a := range argv { - if len(a) > 64<<10 { - t.Errorf("an argument of %d bytes is handed to the command", len(a)) - } - } - i := slices.Index(argv, "--members-file") - if i < 0 { - t.Fatalf("the command does not name a members file: %.200v", argv) - } - raw, err := os.ReadFile(argv[i+1]) - if err != nil || string(raw) != members { - t.Errorf("the file does not hold the whole members list (%d bytes, %v)", len(raw), err) - } - removeMembersFile(argv) - if _, err := os.Stat(argv[i+1]); !os.IsNotExist(err) { - t.Errorf("the file is still there after the command: %v", err) - } -} - -func TestDeliveryOrderHandsALongMembersListInAFile(t *testing.T) { // 1272/AC2 - members := aLongGroup(t) - argv, err := argvFor("delivery-order", map[string]any{"members": members}) - if err != nil { - t.Fatal(err) - } - if argv[0] != "delivery" || argv[1] != "order" { - t.Errorf("the command is %.100v", strings.Join(argv, " ")) - } - readBackFromTheCommand(t, argv, members) -} - -func TestAShortMembersListIsStillTheArgumentItWas(t *testing.T) { // 1272/AC3 - members := `[{"id":"novox/a#1"},{"id":"novox/b#2"}]` - argv, err := argvFor("delivery-order", map[string]any{"members": members}) - want := []string{"delivery", "order", "--members", members} - if err != nil || !slices.Equal(argv, want) { - t.Errorf("a short list gives %v, %v, want %v", argv, err, want) - } -} - -func TestTheCommandReadsTheMembersFromTheFileAsFromTheArgument(t *testing.T) { // 1272/AC1 - members := `[{"id":"novox/a#1"}]` - argv, _ := membersFlags(strings.Repeat(" ", membersInArgument) + members) - defer removeMembersFile(argv) - if len(argv) != 2 || argv[0] != "--members-file" { - t.Fatalf("a long list gives %v", argv) - } - if st, err := os.Stat(argv[1]); err != nil || st.Mode().Perm() != 0o600 { - t.Errorf("the file is %v, %v", st, err) - } -} diff --git a/cmd/mesh-controller/merge_bus_error_integration_test.go b/cmd/mesh-controller/merge_bus_error_integration_test.go deleted file mode 100644 index 81a56d04..00000000 --- a/cmd/mesh-controller/merge_bus_error_integration_test.go +++ /dev/null @@ -1,75 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "errors" - "sync/atomic" - "testing" - "time" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/chain" - "github.com/novox/mesh-controller/internal/link" -) - -// withoutJetStream is a bucket whose reads fail the way a bus answers while its streams are away. -type withoutJetStream struct { - chain.Backend - away atomic.Bool -} - -func (w *withoutJetStream) Get(ctx context.Context, key string) (chain.Entry, bool, error) { - if w.away.Load() { - return chain.Entry{}, false, nats.ErrJetStreamNotEnabled - } - return w.Backend.Get(ctx, key) -} - -func (w *withoutJetStream) Keys(ctx context.Context) ([]string, error) { - if w.away.Load() { - return nil, nats.ErrJetStreamNotEnabled - } - return w.Backend.Keys(ctx) -} - -func (w *withoutJetStream) Watch(ctx context.Context) (<-chan chain.Entry, error) { - if w.away.Load() { - return nil, nats.ErrJetStreamNotEnabled - } - return w.Backend.Watch(ctx) -} - -func (w *withoutJetStream) WatchFiltered(ctx context.Context, filter string) (<-chan chain.Entry, error) { - if w.away.Load() { - return nil, nats.ErrJetStreamNotEnabled - } - return w.Backend.WatchFiltered(ctx, filter) -} - -// **A merge heard while the bus answers "jetstream not enabled" is held, recorded nowhere, and recorded when handed -// over again** (issue 1583, mesh-catalog #404): hearMerge answers ErrTryAgain, so the bus does not acknowledge the -// announcement, and the merge joins a batch when the bus is back. -func TestAMergeHeardWhileTheBusHasNoJetStreamIsHeldAndRecordedWhenHandedOverAgain(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - assignments := &withoutJetStream{Backend: chain.NewMemory()} - open.inventory.UseChain(chain.New(chain.NewMemory(), chain.NewMemory(), assignments, func(context.Context) error { return nil })) - merge := catalogueMerge("cat2bbbbbbbb", "app", t0) - - assignments.away.Store(true) - err := (following{open: open}).hearMerge(ctx, merge, t0.Add(time.Second)) - if !errors.Is(err, link.ErrTryAgain) { - t.Fatalf("a merge heard while the bus had no jetstream answered %v, which the bus takes as done", err) - } - - assignments.away.Store(false) - hear(t, open, merge, t0.Add(10*time.Second)) - if _, bs := walks(t, open); len(bs) != 1 { - t.Fatalf("the merge handed over again made %d batch(es)", len(bs)) - } -} diff --git a/cmd/mesh-controller/merge_bus_error_test.go b/cmd/mesh-controller/merge_bus_error_test.go deleted file mode 100644 index cf660e25..00000000 --- a/cmd/mesh-controller/merge_bus_error_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package main - -import ( - "errors" - "testing" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/link" -) - -// **A merge the controller cannot act on is held, whatever the error** (issue 1583, after 921): the bus acknowledges an -// announcement only when hearMerge answers without ErrTryAgain, and a merge heard once is not announced again. The bus -// answering "jetstream not enabled" (mesh-catalog #404, 2026-10-21) is no outage of a kind notNow knows, and took the merge. -func TestAMergeThatFailsOnAnyErrorIsHeldAndNotTaken(t *testing.T) { - for name, cause := range map[string]error{ - "jetstream not enabled": nats.ErrJetStreamNotEnabled, - "a bucket missing": errors.New("the bucket mesh-controller_builds is not on the bus"), - "no responders": nats.ErrNoResponders, - } { - held := heldMerge(cause) - if !errors.Is(held, link.ErrTryAgain) || !errors.Is(held, cause) { - t.Errorf("%s answers %v, which the bus takes as done", name, held) - } - } - if heldMerge(nil) != nil { - t.Error("a merge that was heard answers an error") - } - already := link.ErrTryAgain - if heldMerge(already) != already { - t.Error("a merge already held is wrapped again") - } -} diff --git a/cmd/mesh-controller/merge_on_the_bus_test.go b/cmd/mesh-controller/merge_on_the_bus_test.go deleted file mode 100644 index 3cfc5780..00000000 --- a/cmd/mesh-controller/merge_on_the_bus_test.go +++ /dev/null @@ -1,181 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "os" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -// The merge path whole, against a real bus and the store (issue 973 and issue 921, novox/hq ADR 0407 decisions 1 -// and 3; ADR 0391's proof): the forge's merged event is published on the subject the controller consumes, the -// controller's server hands it over and the window keys are on the bus. Store-backed: skips without one. - -// noListener hears no node: these tests send merges only. -type noListener struct{ link.Listener } - -// theMergePathOn serves the controller's consume loop and its merge window on a test bus, and returns a way to -// publish the forge's merged event. -func theMergePathOn(t *testing.T, open *stores) func(link.SourceMoved) { - t.Helper() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - server := link.ConnectNats(js, nil, noListener{}) - if err := server.Follows(following{open: open}); err != nil { - t.Fatal(err) - } - ctx, stop := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { defer close(done); _ = server.Serve(ctx) }() - if err := startMergeWindow(ctx, open, js); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { - stop() - <-done - lookingAtBatches.Wait() - setWindowClock(nil) - windowLook.Store(nil) - }) - // The forge's merges are taken from a consumer made when the server starts serving: a merge published before it exists - // is not delivered to it. With the chain read from the bus the set-up above is quick enough to lose that race, so the - // publisher waits for the consumer. - becomes(t, 10*time.Second, "the merges' consumer being made", func() bool { - _, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName) - return err == nil - }) - return func(m link.SourceMoved) { - t.Helper() - body, err := json.Marshal(m) - if err != nil { - t.Fatal(err) - } - if _, err := js.Context().Publish(broker.ControllerFollows[3], body); err != nil { - t.Fatal(err) - } - } -} - -func becomes(t *testing.T, within time.Duration, what string, is func() bool) { - t.Helper() - deadline := time.Now().Add(within) - for time.Now().Before(deadline) { - if is() { - return - } - time.Sleep(50 * time.Millisecond) - } - t.Fatalf("%s did not happen within %s", what, within) -} - -func mergeKept(open *stores, repo, commit string) bool { - _, known, err := open.inventory.MergeOf(context.Background(), repo, commit) - return err == nil && known -} - -// Two merges of one repository heard while another controller holds the plans, and a rollout of that repository -// open, are both recorded once the plans are free (handed over again by the bus, never dropped), join one batch, -// and the next rollout carries both: no person released anything (issue 921). -func TestMergesHeardWhileAnotherHolderHasThePlansAreRecordedAndCarried(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - was, wasPoll := inventory.HoldWaitFor, inventory.HoldPoll - inventory.HoldWaitFor, inventory.HoldPoll = time.Second, 20*time.Millisecond - t.Cleanup(func() { inventory.HoldWaitFor, inventory.HoldPoll = was, wasPoll }) - // A window of a second: a merge joining a batch opens its window again (ADR 0407 decision 1), so the batch the - // two held merges make is cut when that window has closed and the rollout before it has ended, not before. - settings := t.TempDir() + "/merge-window.json" - if err := os.WriteFile(settings, []byte(`{"merge-window": "1s", "merge-window-at-most": "10m"}`), 0o600); err != nil { - t.Fatal(err) - } - t.Setenv(mergeWindowFileVar, settings) - publish := theMergePathOn(t, open) - ctx := t.Context() - - publish(repoMerge("one", "c1", time.Now())) - becomes(t, 15*time.Second, "the first rollout opening", func() bool { - ws, _ := walks(t, open) - return len(ws) == 1 && ws[0].Open() - }) - ws, _ := walks(t, open) - first := ws[0] - - release, err := open.inventory.HoldPlans(ctx, true) // the other controller works the plans - if err != nil { - t.Fatal(err) - } - publish(repoMerge("one", "c2", time.Now())) - publish(repoMerge("one", "c3", time.Now())) - time.Sleep(2500 * time.Millisecond) // both met the hold and were given back to the bus - if mergeKept(open, "novox/one", "c2") || mergeKept(open, "novox/one", "c3") { - t.Fatal("a merge was recorded while another holder had the plans") - } - release() - becomes(t, 30*time.Second, "both merges recorded after the hold", func() bool { - return mergeKept(open, "novox/one", "c2") && mergeKept(open, "novox/one", "c3") - }) - - first.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &first); err != nil { - t.Fatal(err) - } - // The rollout ended: the look that cuts what waited behind it, once the batch's window is closed too. - var bs []inventory.Plan - becomes(t, 15*time.Second, "the next rollout being cut with both merges", func() bool { - advanceHeld(ctx, open) - ws, bs = walks(t, open) - return len(bs) == 0 && len(ws) == 2 - }) - if ws[0].CommitOf("novox/one") != "c3" || len(ws[0].Delivery.Merges) != 2 { - t.Fatalf("the next rollout does not carry both merges: commit %q, merges %+v", ws[0].CommitOf("novox/one"), - ws[0].Delivery.Merges) - } -} - -// A batch's window closes by the expiry of its key on the bus, heard by the watcher, and the batch is cut with -// no cutter and no call to cut (ADR 0407 decision 1): a window of a second, a merge that waits for it. -func TestABatchWindowClosesByTheExpiryOfItsKey(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - everyHeadWaits(t) - settings := t.TempDir() + "/merge-window.json" - if err := os.WriteFile(settings, []byte(`{"merge-window": "1s", "merge-window-at-most": "10m"}`), 0o600); err != nil { - t.Fatal(err) - } - t.Setenv(mergeWindowFileVar, settings) - publish := theMergePathOn(t, open) - - start := time.Now() - publish(repoMerge("one", "c1", start)) - becomes(t, 10*time.Second, "the merge waiting in a batch", func() bool { - _, bs := walks(t, open) - return len(bs) == 1 && bs[0].State == inventory.PlanAssembling - }) - if ws, _ := walks(t, open); len(ws) != 0 { - t.Fatalf("the batch was cut before its window closed: %+v", ws) - } - becomes(t, 15*time.Second, "the batch being cut by its key's expiry", func() bool { - ws, bs := walks(t, open) - return len(ws) == 1 && len(bs) == 0 - }) - if took := time.Since(start); took < time.Second { - t.Fatalf("the batch was cut after %s, before its window of a second was up", took) - } -} diff --git a/cmd/mesh-controller/merge_unsaved_integration_test.go b/cmd/mesh-controller/merge_unsaved_integration_test.go deleted file mode 100644 index 095930b9..00000000 --- a/cmd/mesh-controller/merge_unsaved_integration_test.go +++ /dev/null @@ -1,71 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "errors" - "sync/atomic" - "testing" - "time" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/chain" - "github.com/novox/mesh-controller/internal/link" -) - -// refusing is a bucket that refuses to write while it is told to. -type refusing struct { - chain.Backend - refuse atomic.Bool -} - -func (r *refusing) Create(ctx context.Context, key string, value []byte) (uint64, error) { - if r.refuse.Load() { - return 0, nats.ErrTimeout - } - return r.Backend.Create(ctx, key, value) -} - -func (r *refusing) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) { - if r.refuse.Load() { - return 0, nats.ErrTimeout - } - return r.Backend.Update(ctx, key, value, revision) -} - -// **A merge whose plan cannot be saved is held, recorded nowhere, and recorded when it is handed over again** (ADR 0407 -// decision 3, issue 1092): hearMerge answers ErrTryAgain — the bus leaves the announcement unacknowledged and hands it -// over again — and the merge is in no record until the plan holding it is written. -func TestAMergeWhosePlanCannotBeSavedIsHeldAndRecordedWhenHandedOverAgain(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - rollouts := &refusing{Backend: chain.NewMemory()} - open.inventory.UseChain(chain.New(chain.NewMemory(), rollouts, chain.NewMemory(), func(context.Context) error { return nil })) - merge := catalogueMerge("cat1aaaaaaaa", "app", t0) - - rollouts.refuse.Store(true) - err := (following{open: open}).hearMerge(ctx, merge, t0.Add(time.Second)) - if !errors.Is(err, link.ErrTryAgain) { - t.Fatalf("a merge whose plan could not be saved answered %v, which the bus takes as done", err) - } - if _, known, err := open.inventory.MergeOf(ctx, "novox/mesh-catalog", merge.Commit); err != nil || known { - t.Fatalf("the merge is kept though its plan was not: %v %v", known, err) - } - if _, bs := walks(t, open); len(bs) != 0 { - t.Fatalf("a batch was kept though its write failed: %+v", bs) - } - - rollouts.refuse.Store(false) - hear(t, open, merge, t0.Add(10*time.Second)) - _, bs := walks(t, open) - if len(bs) != 1 { - t.Fatalf("the merge handed over again made %d batch(es)", len(bs)) - } - if m, known, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", merge.Commit); !known || m.Plan != bs[0].ID { - t.Fatalf("the merge is %+v %v, and the batch %s", m, known, bs[0].ID) - } -} diff --git a/cmd/mesh-controller/merge_window_bus.go b/cmd/mesh-controller/merge_window_bus.go deleted file mode 100644 index bd721841..00000000 --- a/cmd/mesh-controller/merge_window_bus.go +++ /dev/null @@ -1,204 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" -) - -// The merge window's clock is on the bus (novox/hq ADR 0407 decision 1, issue 973): an open batch has a key -// `.quiet` that lives the window and is made again by each merge into the batch, and a key `.most` -// made once when the batch opens that lives the window's most. The first of the two to expire closes the batch, -// and the watcher hears it: no timer polls the batches. The store still keeps the times (ClosesAt, AtMost) as the -// record of when a window closed, which `plans` reads; the keys are what wakes the controller. - -// windowClock is the bus's side of the merge window. -type windowClock interface { - // Arm makes a batch's keys: its quiet key again, to live quiet; its most key if it has none, to live most. - Arm(ctx context.Context, batch string, quiet, most time.Duration) error - // Forget takes a batch's keys away: it was cut, or closed, and nothing is to wake the controller for it. - Forget(ctx context.Context, batch string) -} - -// windowLook is the look a window's expiry takes: the watcher's, and the fallback timer's. -var windowLook atomic.Pointer[func(batch string)] - -var ( - windowMu sync.RWMutex - windowKeyed windowClock // nil in a process that is not the serving controller -) - -// setWindowClock makes clock the merge window's; nil takes it away. -func setWindowClock(c windowClock) { - windowMu.Lock() - defer windowMu.Unlock() - windowKeyed = c -} - -func windowClockOf() windowClock { - windowMu.RLock() - defer windowMu.RUnlock() - return windowKeyed -} - -// windowDeadlines are how long a batch's two keys are to live from now: what is left of its window and of its -// most, each at least a second (the least a key can live). Not armed for a batch whose window is closed already -// (by a stop, by its time, or because it is cut at once) or that is no batch. -func windowDeadlines(b inventory.Plan, now time.Time) (quiet, most time.Duration, armed bool) { - if b.Delivery == nil || b.Delivery.Batch == nil || windowClosed(b, now) || len(b.Delivery.Batch.Holds) > 0 { - return 0, 0, false // a held batch has no clock: its group's last member closes it (ADR 0418 clause 3) - } - // Rounded up to a whole second, at least one: the server rounds a key's time to live down to whole seconds, so - // a fraction would close the key before the record's time and the look would cut nothing. - least := func(d time.Duration) time.Duration { - if d < time.Second { - return time.Second - } - return (d + time.Second - 1) / time.Second * time.Second - } - w := b.Delivery.Batch - return least(w.ClosesAt.Sub(now)), least(w.AtMost.Sub(now)), true -} - -// armWindow puts a batch's clock on the bus after its record was kept. Where the bus will not take it the window -// closes by a timer of this batch alone, once, as long as is left: never silently never. -func armWindow(ctx context.Context, b inventory.Plan, now time.Time) { - c := windowClockOf() - quiet, most, armed := windowDeadlines(b, now) - if c == nil || !armed { - return - } - if err := c.Arm(ctx, b.ID, quiet, most); err != nil { - fmt.Printf("batches: the window of %s could not be put on the bus (%v); it closes by a timer of its own\n", b.ID, err) - if look := windowLook.Load(); look != nil { - time.AfterFunc(quiet, func() { (*look)(b.ID) }) - } - } -} - -// busWindow is the clock on the controller's bucket. -type busWindow struct { - kv jetstream.KeyValue -} - -// newBusWindow opens the bucket and starts the watcher, which names each batch whose key expired to look until -// the context ends. Only expiry wakes it: a key taken away by Forget or made again by Arm leaves a delete. -func newBusWindow(ctx context.Context, api jetstream.JetStream, look func(batch string)) (*busWindow, error) { - kv, err := api.KeyValue(ctx, broker.MergeWindowBucket) - if err != nil { - return nil, fmt.Errorf("the merge window's bucket: %w", err) - } - w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) - if err != nil { - return nil, fmt.Errorf("watching the merge window: %w", err) - } - go func() { - defer w.Stop() - for { - select { - case <-ctx.Done(): - return - case e, ok := <-w.Updates(): - if !ok { - return - } - if e == nil || e.Operation() != jetstream.KeyValuePurge { - continue // a put, or a delete this controller made - } - if batch, _, found := strings.Cut(e.Key(), "."); found && look != nil { - look(batch) - } - } - } - }() - return &busWindow{kv: kv}, nil -} - -func (w *busWindow) Arm(ctx context.Context, batch string, quiet, most time.Duration) error { - // The most is made once, when the batch opens: it is there already for every merge after the first. - if _, err := w.kv.Create(ctx, batch+".most", []byte("most"), jetstream.KeyTTL(most)); err != nil && - !errors.Is(err, jetstream.ErrKeyExists) { - return err - } - // The quiet key is made again, not changed: a key's time to live is set when it is made. - if err := w.kv.Delete(ctx, batch+".quiet"); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { - return err - } - if _, err := w.kv.Create(ctx, batch+".quiet", []byte("quiet"), jetstream.KeyTTL(quiet)); err != nil { - return err - } - return nil -} - -func (w *busWindow) Forget(ctx context.Context, batch string) { - for _, key := range []string{batch + ".quiet", batch + ".most"} { - _ = w.kv.Delete(ctx, key) - } -} - -// lookingAtBatches counts the looks under way, so a test that ends the window's clock can wait for the last of them before the -// next test replaces what they read. -var lookingAtBatches sync.WaitGroup - -// lookAtBatches is the look an expired window asks for: the plans held, as a merge heard holds them, and the batches -// whose window closed cut (cutBatchesHeld). Waiting for the plans rather than giving up: an expiry is heard once. -func lookAtBatches(ctx context.Context, open *stores) { - lookingAtBatches.Add(1) - defer lookingAtBatches.Done() - release, err := open.inventory.HoldPlans(ctx, true) - if err != nil { - fmt.Printf("batches: a window closed and the plans could not be held for the look: %v\n", err) - return - } - defer release() - if err := cutBatchesHeld(ctx, open, time.Now().UTC()); err != nil { - fmt.Printf("batches: %v\n", err) - } -} - -// startMergeWindow puts the merge window's clock on the bus for as long as the context lives: the controller is -// the lease holder here, the only one that writes it. The watcher looks at the batches when a key expires; the -// batches open now get their keys, and one look is taken, so a window that closed while no controller served is -// cut (ADR 0407 decision 1). No timer polls them. -func startMergeWindow(ctx context.Context, open *stores, js *broker.JetStream) error { - api, err := jetstream.New(js.Conn()) - if err != nil { - return err - } - look := func(string) { lookAtBatches(ctx, open) } - windowLook.Store(&look) - clock, err := newBusWindow(ctx, api, look) - if err != nil { - windowLook.Store(nil) - return err - } - setWindowClock(clock) - go func() { - <-ctx.Done() - setWindowClock(nil) - windowLook.Store(nil) - }() - release, err := open.inventory.HoldPlans(ctx, true) - if err != nil { - return err - } - defer release() - batches, err := open.inventory.Batches(ctx) - if err != nil { - return err - } - now := time.Now().UTC() - for _, b := range batches { - armWindow(ctx, b, now) - } - return cutBatchesHeld(ctx, open, now) -} diff --git a/cmd/mesh-controller/merge_window_bus_integration_test.go b/cmd/mesh-controller/merge_window_bus_integration_test.go deleted file mode 100644 index 5cdae1e8..00000000 --- a/cmd/mesh-controller/merge_window_bus_integration_test.go +++ /dev/null @@ -1,90 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "testing" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/testbus" -) - -func windowOnABus(t *testing.T) (*busWindow, <-chan string) { - t.Helper() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - api, err := jetstream.New(conn) - if err != nil { - t.Fatal(err) - } - heard := make(chan string, 16) - ctx, stop := context.WithCancel(context.Background()) - t.Cleanup(stop) - w, err := newBusWindow(ctx, api, func(batch string) { heard <- batch }) - if err != nil { - t.Fatal(err) - } - return w, heard -} - -func TestTheQuietKeyExpiresAndTheWatcherNamesTheBatch(t *testing.T) { - w, heard := windowOnABus(t) - if err := w.Arm(t.Context(), "plan-1", time.Second, time.Minute); err != nil { - t.Fatal(err) - } - if b, ok := hears(t, heard, 10*time.Second); !ok || b != "plan-1" { - t.Fatalf("heard %q (%v); want plan-1", b, ok) - } -} - -func TestAMergeRestartsTheQuietKeyAndTheMostKeyClosesAllTheSame(t *testing.T) { - w, heard := windowOnABus(t) - ctx := t.Context() - if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil { - t.Fatal(err) - } - time.Sleep(1200 * time.Millisecond) - if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil { // a merge: quiet again, most as it was - t.Fatal(err) - } - if b, ok := hears(t, heard, 1500*time.Millisecond); ok { - t.Fatalf("heard %q before the restarted window was up", b) - } - // Another merge keeps the window open; the most (5 s from the first) closes it at about 5 s. - start := time.Now() - for time.Since(start) < 2500*time.Millisecond { - if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil { - t.Fatal(err) - } - time.Sleep(700 * time.Millisecond) - } - if b, ok := hears(t, heard, 8*time.Second); !ok || b != "plan-2" { - t.Fatalf("the most did not close the batch: heard %q (%v)", b, ok) - } -} - -func TestForgettingABatchWakesNobody(t *testing.T) { - w, heard := windowOnABus(t) - if err := w.Arm(t.Context(), "plan-3", time.Second, time.Minute); err != nil { - t.Fatal(err) - } - w.Forget(t.Context(), "plan-3") - if b, ok := hears(t, heard, 3*time.Second); ok { - t.Fatalf("a forgotten batch was heard expire: %q", b) - } -} diff --git a/cmd/mesh-controller/merge_window_bus_test.go b/cmd/mesh-controller/merge_window_bus_test.go deleted file mode 100644 index 760a360e..00000000 --- a/cmd/mesh-controller/merge_window_bus_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package main - -import ( - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// windowOnABus is the merge window's clock on a real bus, and the batches its watcher heard expire. - -func hears(t *testing.T, heard <-chan string, within time.Duration) (string, bool) { - t.Helper() - select { - case b := <-heard: - return b, true - case <-time.After(within): - return "", false - } -} - -// A batch's quiet key expires when no merge came for the window, and the watcher names the batch (issue 973, -// novox/hq ADR 0407 decision 1). - -// A merge into the batch makes the quiet key again, so the window restarts: it does not expire at the first -// deadline, and the batch's most closes it all the same. - -// A batch cut is forgotten: its keys are taken away by a delete, which is no expiry and wakes nobody. - -// The deadlines a batch's keys are made with: what is left of its window and of its most, at least a second; -// none for a batch whose window is closed or that is no batch. -func TestTheDeadlinesOfABatchsKeys(t *testing.T) { - now := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC) - batch := func(closes, most time.Duration, closed bool) inventory.Plan { - return inventory.Plan{State: inventory.PlanAssembling, Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{ - ClosesAt: now.Add(closes), AtMost: now.Add(most), Closed: closed}}} - } - for _, c := range []struct { - what string - b inventory.Plan - quiet, most time.Duration - armed bool - }{ - {"open", batch(90*time.Second, 10*time.Minute, false), 90 * time.Second, 10 * time.Minute, true}, - {"almost up", batch(100*time.Millisecond, 5*time.Second, false), time.Second, 5 * time.Second, true}, - {"closed by a stop", batch(90*time.Second, 10*time.Minute, true), 0, 0, false}, - {"up", batch(-time.Second, 10*time.Minute, false), 0, 0, false}, - {"cut at once", func() inventory.Plan { - b := batch(90*time.Second, 10*time.Minute, false) - b.Delivery.Batch.CutAtOnce = true - return b - }(), 0, 0, false}, - {"no batch", inventory.Plan{}, 0, 0, false}, - } { - quiet, most, armed := windowDeadlines(c.b, now) - if armed != c.armed || quiet != c.quiet || most != c.most { - t.Errorf("%s: %v, %v, %v; want %v, %v, %v", c.what, quiet, most, armed, c.quiet, c.most, c.armed) - } - } -} - -// A deadline that is not a whole number of seconds is rounded up, never down: the server rounds a key's time to live -// down to whole seconds, so 89.999999 s would expire at 89 s, before the window is closed by the record's times, and -// the look would cut nothing (review of issue 973). -func TestAFractionalDeadlineIsRoundedUpToTheNextWholeSecond(t *testing.T) { - now := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC) - b := inventory.Plan{State: inventory.PlanAssembling, Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{ - ClosesAt: now.Add(89*time.Second + 999999*time.Microsecond), AtMost: now.Add(599*time.Second + time.Nanosecond)}}} - quiet, most, armed := windowDeadlines(b, now) - if !armed || quiet != 90*time.Second || most != 600*time.Second { - t.Errorf("deadlines %v, %v (armed %v); want 1m30s, 10m0s", quiet, most, armed) - } -} diff --git a/cmd/mesh-controller/merged_by_claim_test.go b/cmd/mesh-controller/merged_by_claim_test.go deleted file mode 100644 index 1b3544c7..00000000 --- a/cmd/mesh-controller/merged_by_claim_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package main - -import ( - "context" - "errors" - "testing" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// 1211/AC1 to AC3: the controller keeps a merge announced by a module that claims the delivery seat, whatever its -// name, and takes no word from one that does not (novox/hq ADR 0437 §3). With no bus held, a word that passes the -// claim is held for the bus (ErrTryAgain); a word that does not is refused before the bus is asked. -func TestOnlyAModuleClaimingTheSeatIsTheDeliverysWordOnAMerge(t *testing.T) { - entries := []inventory.Entry{claimingTheSeat("mesh-delivery-gate"), namedOnly("gitea"), namedOnly("mesh-delivery")} - keeper := keepingDeliveryMerges{ - js: func() *broker.JetStream { return nil }, - catalog: func(context.Context) ([]inventory.Entry, error) { return entries, nil }, - } - body := []byte(`{"owner":"novox","repo":"r","number":1,"head_sha":"abc","ask_id":"a","option":"approve"}`) - for emitter, passes := range map[string]bool{ - "mesh-delivery-gate": true, // the claimant, under another name - "gitea": false, // claims no seat - "mesh-delivery": false, // named so, claiming no seat here - } { - err := keeper.KeepDeliveryMerge(context.Background(), emitter, body) - if passes != errors.Is(err, link.ErrTryAgain) || (!passes && !errors.Is(err, link.ErrNotTheDeliverysWord)) { - t.Errorf("%s: %v (passes the claim: %v)", emitter, err, passes) - } - } - entries = []inventory.Entry{claimingTheSeat("mesh-delivery")} - if err := keeper.KeepDeliveryMerge(context.Background(), "mesh-delivery", body); !errors.Is(err, link.ErrTryAgain) { - t.Errorf("today's module, claiming the seat: %v", err) - } -} diff --git a/cmd/mesh-controller/module_health.go b/cmd/mesh-controller/module_health.go index f021b1b8..db18ca72 100644 --- a/cmd/mesh-controller/module_health.go +++ b/cmd/mesh-controller/module_health.go @@ -631,97 +631,6 @@ func accountOf(r inventory.ResourceHealth) string { return r.Target } -// moduleHealthWord is the gate's reading of a module's stated health on a machine (ADR 0240 §4, ADR 0236 -// §2 as amended): good when every long-running resource of it is stated healthy in a statement heard since -// the send; not yet otherwise, saying which. A machine that never stated health is judged as before. -// -// **A wait for a person is its own reading** (novox/hq ADR 0254): when everything unhealthy of the module -// waits for one person's new login (personWait), the reading is healthPerson — not a fault of the build, -// and not something a machine can meet within a bound. -func moduleHealthWord(module, machine string, since time.Time, f gateFacts) (health, string) { - if f.healthErr != nil { - return healthNotYet, "what " + machine + " says of its resources' health cannot be read: " + firstLine(f.healthErr.Error()) - } - h, states := f.health[machine] - if !states { - return healthGood, "" - } - if h.HeardAt.Before(since) { - return healthNotYet, fmt.Sprintf("%s has not said how what %s runs is since it was sent", machine, module) - } - // **A wait for the operator is checked first** (novox/hq ADR 0283): one that does not check out is unhealthy. - resources := checkWaiting(machine, h.Resources, f.waits) - wait, waits := personWait(module, machine, resources) - // **Only a build whose own send put the account in a new group is excused** (issue 318 review): read from - // what the controller sent, never from when the machine says the wait began — that time is the engine's - // memory, reset by its restart and moved by a change of words. A build that adds no account group cannot - // have brought a wait, so a later build sent while the login is still owed is judged as before. - if waits && !f.groupsAdded[module] { - waits = false - } - var found, onWaiting []string - var forOperator []inventory.Wait - for _, r := range resources { - if r.Module != module { - continue - } - // **Any build is excused while its wait for the operator checks out** (ADR 0283 decision 4): a secret not - // given is owed by every build alike, so it is no fault of this one, and the verdict carries it. - if r.State == link.StateWaiting { - forOperator = append(forOperator, r.Waits...) - continue - } - if waits && r.State == link.StateUnhealthy { - continue - } - // **A directory used as found before this send waits for a person** (novox/hq issue 339): the node-engine - // left its owner and mode, and only someone at the machine can hand it over. It is no fault of this - // build, so it does not hold the module's walk — an urgent fix still goes through — and the verdict - // carries the wait. Found by this very send, the send brought it, and it is judged as unhealthy. - if usedAsFound(r) && r.Since.Before(since) { - found = append(found, r.Resource) - continue - } - switch r.State { - case link.StateHealthy: - case link.StateStarting: - return healthNotYet, fmt.Sprintf("its %s %s on %s is still starting", r.Kind, r.Resource, machine) - case link.StateUnhealthy: - if on, held := f.heldOn[module+"@"+machine]; held { - // **Held under a provider that only waits for the operator** (novox/hq issue 405): a wait for a - // person, a pass carrying the wait (ADR 0254, ADR 0283 decision 4) — the walk never waits for the - // operator's secret, whichever module owes it. - if on.waits != "" { - onWaiting = append(onWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which waits for you: %s", - r.Kind, r.Resource, machine, on.on, on.waits)) - continue - } - return healthWaiting, fmt.Sprintf("its %s %s on %s waits on %s, which is unhealthy", r.Kind, - r.Resource, machine, on.on) - } - return healthNotYet, fmt.Sprintf("its %s %s on %s %s", r.Kind, r.Resource, machine, reasonWords(r)) - default: - return healthNotYet, fmt.Sprintf("its %s %s on %s is %s%s", r.Kind, r.Resource, machine, r.State, - reasonAfter(r.Reason)) - } - } - if waits || len(found) > 0 || len(forOperator) > 0 || len(onWaiting) > 0 { - said := onWaiting - if waits { - said = append(said, wait) - } - if len(forOperator) > 0 { - said = append(said, operatorWaitSaid(module, machine, forOperator)) - } - if len(found) > 0 { - said = append(said, fmt.Sprintf("on %s, %s uses %s as found and waits for the operator to hand it over "+ - "(`nox node hand-over %s ` on the control-node)", machine, module, strings.Join(found, ", "), machine)) - } - return healthPerson, strings.Join(said, "; ") - } - return healthGood, "" -} - // usedAsFound is a directory the node-engine states it uses as found (novox/hq issue 339). func usedAsFound(r inventory.ResourceHealth) bool { return r.Kind == link.KindDirectory && r.State == link.StateUnhealthy && strings.HasPrefix(r.Reason, link.ReasonUsedAsFound) diff --git a/cmd/mesh-controller/modules_wire_store_test.go b/cmd/mesh-controller/modules_wire_store_test.go index ea3fcd6f..2e3dfb7d 100644 --- a/cmd/mesh-controller/modules_wire_store_test.go +++ b/cmd/mesh-controller/modules_wire_store_test.go @@ -2,82 +2,4 @@ package main -import ( - "encoding/json" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/link" -) - -// 1145/AC1 on a store: declarationWith itself sets the module of each resource and the build of each module for a -// node recorded as reading them, and for no other. Skips without a store. -func TestAComposedDeclarationCarriesModulesAndBuildsOnlyForANodeThatReadsThem(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - start := time.Now().Add(-time.Hour) - manifest := resticFixture(false) - manifest["module"], manifest["version"] = "restic", "1" - raw, _ := json.Marshal(manifest) - b := link.BuildResult{ID: link.NewBuildID(start), Repository: "novox/mesh-catalog", Path: "modules/restic", - On: "anchor", Module: "restic", Commit: "c1", Manifest: raw, - Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}} - if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil { - t.Fatal(err) - } - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, "restic"); err != nil { - t.Fatal(err) - } - } - anchor, err := inv.NodeByName(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - if err := inv.RecordReadsModules(ctx, anchor.ID, true); err != nil { - t.Fatal(err) - } - wantBuild, err := inv.BuildOf(ctx, "restic", "c1") - if err != nil || wantBuild == "" { - t.Fatalf("the build of restic at c1 is %q (%v)", wantBuild, err) - } - - declare := func(node string) sendable { - t.Helper() - d, err := composeSend(ctx, open, node) - if err != nil { - t.Fatal(err) - } - return d - } - reads := bodyKeys(t, declare("anchor")) - builds, _ := reads["builds"].(map[string]any) - if builds["restic"] != wantBuild { - t.Errorf("1145/AC1: anchor reads modules and was sent builds %v, want restic at %s", reads["builds"], wantBuild) - } - owned := 0 - for _, r := range reads["resources"].([]any) { - r := r.(map[string]any) - if strings.HasPrefix(r["id"].(string), "restic.") { - owned++ - if r["module"] != "restic" { - t.Errorf("1145/AC1: resource %v carries module %v, want restic", r["id"], r["module"]) - } - } - } - if owned == 0 { - t.Fatal("1145/AC1: no resource of restic was composed") - } - - plain := bodyKeys(t, declare("laptop")) - if _, there := plain["builds"]; there { - t.Errorf("1145/AC1: laptop never said it reads modules and was sent builds %v", plain["builds"]) - } - for _, r := range plain["resources"].([]any) { - if _, there := r.(map[string]any)["module"]; there { - t.Fatalf("1145/AC1: laptop was sent a module on %v", r) - } - } -} +import () diff --git a/cmd/mesh-controller/modules_wire_test.go b/cmd/mesh-controller/modules_wire_test.go index 7dd25721..344a7bf1 100644 --- a/cmd/mesh-controller/modules_wire_test.go +++ b/cmd/mesh-controller/modules_wire_test.go @@ -24,22 +24,6 @@ func modulesDeclaration(reads bool) sendable { } } -// 1145/AC1 -func TestAMachineThatReadsModulesIsSentEachResourcesModuleAndEachModulesBuild(t *testing.T) { - keys := bodyKeys(t, modulesDeclaration(true)) - builds, _ := keys["builds"].(map[string]any) - if len(builds) != 1 || builds["web.app"] != "build-7" { - t.Fatalf("1145/AC1: the body carries builds %v", keys["builds"]) - } - want := map[string]any{"web.app.conf": "web.app", "web.app.data": "web.app", "mesh.guard": nil} - for _, r := range keys["resources"].([]any) { - r := r.(map[string]any) - if r["module"] != want[r["id"].(string)] { - t.Errorf("1145/AC1: resource %v carries module %v, want %v", r["id"], r["module"], want[r["id"].(string)]) - } - } -} - // 1145/AC2: the body is byte for byte what it is today. func TestAMachineThatDoesNotReadModulesIsSentTheBodyItWasSentBefore(t *testing.T) { d := modulesDeclaration(false) diff --git a/cmd/mesh-controller/moved.go b/cmd/mesh-controller/moved.go index 6c07c210..e1282fc6 100644 --- a/cmd/mesh-controller/moved.go +++ b/cmd/mesh-controller/moved.go @@ -330,3 +330,29 @@ func approvedAnnounced(ctx context.Context, inv catalogueReader, owner, name str } return fmt.Sprintf("%s/%s#%d merged at %.8s on %s", owner, name, k.Number, k.HeadSHA, why), nil } + +// claimantsOf are the modules among the catalogued entries that claim a seat, each once, in name order. +func claimantsOf(entries []inventory.Entry, seat string) []string { + var out []string + for _, e := range entries { + if e.Manifest.ClaimsSeat(seat) && !slices.Contains(out, e.Manifest.Module) { + out = append(out, e.Manifest.Module) + } + } + sort.Strings(out) + return out +} + +// announcedOf is a kept merge as the forge announced it, and what it moved; empty where the record says none. +func announcedOf(m inventory.BatchedMerge) keptMerge { + var k keptMerge + _ = json.Unmarshal(m.Event, &k) + return k +} + +// keptMerge is a merge as the record keeps it: the forge's announcement, and the modules it moved when it was +// heard. The announcement reads back as a SourceMoved alone, which ignores the rest. +type keptMerge struct { + link.SourceMoved + Moves []string `json:"moves,omitempty"` +} diff --git a/cmd/mesh-controller/new_head_words_test.go b/cmd/mesh-controller/new_head_words_test.go deleted file mode 100644 index 86c8a959..00000000 --- a/cmd/mesh-controller/new_head_words_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package main - -import ( - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/conditions" -) - -// novox/hq ADR 0505 §5: a delivery whose cure is its author's new head reaches the operator in words that name its -// author and say a new head is needed, offer no Stop and never call it the operator's; the line as mesh-delivery-gate -// says it (ready.go readyLines: needs "new-head", author, branch). -func TestADeliveryThatNeedsANewHeadNamesItsAuthorAndOffersNoStop(t *testing.T) { - l := stalledLine{ID: "novox/mesh-controller@499ff5c21861", Number: 432, State: "rejected", For: "2h1m0s", Bound: "2h0m0s", - H2: "none: a new head from its author, or a recheck that passes", Needs: needsNewHead, - Author: "the author of feat/quiet-deliveries (quiet-deliveries on shanks)", Branch: "feat/quiet-deliveries", - Says: "novox/mesh-controller#432 passed its checks at 499ff5c21861 and is not merged: a conflict"} - o := stalledObservations([]stalledLine{l})[0] - for _, words := range []string{o.Headline, o.Explanation, o.Resolved, o.Needs} { - if why, ok := conditions.Plain(words); !ok { - t.Errorf("%q is not plain: %s", words, why) - } - for _, never := range []string{"stop it", "operator's"} { - if strings.Contains(words, never) { - t.Errorf("%q says %q", words, never) - } - } - } - if o.Headline != "Pull request 432 of mesh-controller needs a new head from its author" || - !strings.Contains(o.Explanation, "until the author of feat/quiet-deliveries (quiet-deliveries on shanks) pushes a new head") || - !strings.Contains(o.Explanation, "for 2 hours") || len(o.Actions) != 0 || - o.Needs != "ask the author of feat/quiet-deliveries (quiet-deliveries on shanks) for a new head." { - t.Fatalf("words: %q / %q / %q / %v", o.Headline, o.Explanation, o.Needs, o.Actions) - } - // A branch in code is not said; the author is then "its author". - l.Author = "the author of fix/a1b2c3d4e5f6" - if o := stalledObservations([]stalledLine{l})[0]; !strings.Contains(o.Needs, "ask its author") { - t.Fatalf("a dotted branch was said: %q", o.Needs) - } -} diff --git a/cmd/mesh-controller/new_modules.go b/cmd/mesh-controller/new_modules.go index faee9615..77a26108 100644 --- a/cmd/mesh-controller/new_modules.go +++ b/cmd/mesh-controller/new_modules.go @@ -201,7 +201,6 @@ func newModuleObservations(now time.Time) []conditions.Observation { // looks at every module once now. A bucket not there yet is said: the look at start and a first merge's word still work. func watchNewModules(ctx context.Context, open *stores) { inBackground(func() { - answerHeldMerges(ctx, open.inventory) takeNewModules(ctx, open) }) keys, closeKeys, err := openBuilderKeys() @@ -232,8 +231,6 @@ func wakeNewModule(ctx context.Context, open *stores, module string) { } for _, e := range entries { if e.Manifest.Module == module { - // Known: a plan waiting on its key is woken by the plans' own look. - askToLook() return } } diff --git a/cmd/mesh-controller/non_trunk_merge_test.go b/cmd/mesh-controller/non_trunk_merge_test.go deleted file mode 100644 index 71322750..00000000 --- a/cmd/mesh-controller/non_trunk_merge_test.go +++ /dev/null @@ -1,55 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A merge into a branch other than the trunk starts no delivery (novox/hq ADR 0401 clause 3, issue 889): a -// module recorded with no ref follows the trunk, main, and no other branch. -func TestAMergeIntoANonTrunkBranchStartsNoDelivery(t *testing.T) { - for ref, name := range map[string]string{"": "noref", "main": "mainref"} { - t.Run(name, func(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: "free", Version: "1"}, - inventory.Source{Repository: "novox/free", Seat: "git", Ref: ref, BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - feature := lone(repoMerge("free", "f1", t0)) - feature.Base = "feat/57-config-and-provisioning" - said := captured(t, func() error { return (following{open: open}).hearMerge(t.Context(), feature, t0) }) - if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 0 { - t.Fatalf("a merge into a feature branch: %d walk(s), %d batch(es)", len(ws), len(bs)) - } - if !strings.Contains(said, feature.Base) || !strings.Contains(said, "starts no delivery") { - t.Fatalf("the merge was not said to start no delivery: %q", said) - } - hear(t, open, lone(repoMerge("free", "m1", t0)), t0) - if ws, bs := walks(t, open); len(ws)+len(bs) != 1 { - t.Fatalf("a merge into main: %d walk(s), %d batch(es)", len(ws), len(bs)) - } - }) - } -} - -// A module recorded with an explicit branch keeps following that branch (the lead's settlement of issue 977 Q5). -func TestAModuleRecordedWithABranchStillFollowsIt(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: "pinned", Version: "1"}, - inventory.Source{Repository: "novox/pinned", Seat: "git", Ref: "feat/x", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - m := lone(repoMerge("pinned", "p1", t0)) - m.Base = "feat/x" - hear(t, open, m, t0) - if ws, bs := walks(t, open); len(ws)+len(bs) != 1 { - t.Fatalf("a merge into the branch a module follows: %d walk(s), %d batch(es)", len(ws), len(bs)) - } -} diff --git a/cmd/mesh-controller/one_group_one_plan_test.go b/cmd/mesh-controller/one_group_one_plan_test.go deleted file mode 100644 index f37d07b0..00000000 --- a/cmd/mesh-controller/one_group_one_plan_test.go +++ /dev/null @@ -1,174 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// Acceptance 1 of design 47, the controller's part: a feature across the controller and the catalogue with a new seat -// verb is delivered as one group, with no `after:` line — its merges are one batch, each module is built once and sent -// once, its commits are carried by that one plan, and the plan names the edge that placed each tier. The Approve and -// the one-pass merge are mesh-delivery's. - -// manifestsHeld stubs the forge for a test: the manifest each (repository, path, commit) holds, and none anywhere else. -func manifestsHeld(t *testing.T, held map[[3]string]catalogue.Manifest) { - t.Helper() - was, wasParent := manifestAt, parentOf - t.Cleanup(func() { manifestAt, parentOf = was, wasParent }) - manifestsReadMu.Lock() - clear(manifestsRead) - manifestsReadMu.Unlock() - parentOf = func(context.Context, string, string, string) (string, error) { return "", nil } - manifestAt = func(_ context.Context, _, repo, file, ref string) (catalogue.Manifest, bool, error) { - m, ok := held[[3]string{repo, file, ref}] - return m, ok, nil - } -} - -func TestAFeatureWithANewSeatVerbIsOneBatchAndOnePlanInOrderWithNoAfterLine(t *testing.T) { - for name, controllerFirst := range map[string]bool{"controller_first": true, "catalogue_first": false} { - t.Run(name, func(t *testing.T) { - open := windowed(t) - asked := asksWithPaths(t) - ctx := t.Context() - const head = "feat/one-group-one-plan" - // mesh-delivery holds its seat, so a catalogue plan would wait for its word; this plan carries the controller. - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - controller := fromHead(repoMerge("mesh-controller", "k1", t0), head) - controller.Paths = []string{"module.json", "cmd/mesh-controller/main.go"} - catalog := fromHead(catalogueMerge("cat1aaaaaaaa", "app", t0), head) - manifestsHeld(t, map[[3]string]catalogue.Manifest{ - {"mesh-controller", "module.json", "k1"}: {Module: "mesh-controller", - DefinesSeats: []catalogue.SeatDeclaration{{Name: "seat", Serves: []catalogue.Verb{{Name: "verb"}}}}}, - {"mesh-catalog", "modules/app/module.json", "cat1aaaaaaaa"}: {Module: "app", Invokes: []string{"seat:seat.verb"}}, - }) - states := map[string]string{"novox/mesh-controller": "ready", "novox/mesh-catalog": "ready"} - groupStates(t, head, states) - first, second := controller, catalog - if !controllerFirst { - first, second = catalog, controller - } - // The first member merges; the group waits for the second, however long it takes. - hear(t, open, first, t0) - cutAt(t, open, t0.Add(30*time.Minute)) - if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 { - t.Fatalf("a member of a group alone: %d walk(s), %d batch(es)", len(ws), len(bs)) - } - states["novox/"+second.Repo] = "merged" - second.MergedAt = t0.Add(30 * time.Minute).Format(time.RFC3339Nano) - hear(t, open, second, t0.Add(30*time.Minute)) - cutAt(t, open, t0.Add(30*time.Minute+90*time.Second)) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 0 { - t.Fatalf("after the window: %d walk(s), %d batch(es); want exactly one plan", len(ws), len(bs)) - } - plan := ws[0] - if plan.CommitOf("novox/mesh-controller") != "k1" || plan.CommitOf("novox/mesh-catalog") != "cat1aaaaaaaa" { - t.Fatalf("the plan carries %+v", plan.Commits) - } - if len(plan.Tiers) != 2 || strings.Join(plan.Tiers[0], ",") != "mesh-controller" || strings.Join(plan.Tiers[1], ",") != "app" { - t.Fatalf("the plan's tiers are %v; want the controller, then the caller of its verb", plan.Tiers) - } - if placed := plan.Modules["app"].Placed; !strings.Contains(placed, "verb before caller: seat.verb") || - !strings.Contains(placed, "after mesh-controller") { - t.Fatalf("the plan does not name the edge that placed the caller: %q", placed) - } - if plan.Modules["mesh-controller"].Placed != "" { - t.Fatalf("the first tier is placed by %q", plan.Modules["mesh-controller"].Placed) - } - if plan.Delivery != nil && (plan.Delivery.Awaits != "" || plan.Waiting()) { - t.Fatalf("a plan carrying the controller waits for %q's word", plan.Delivery.Awaits) - } - // The cut takes the first tier from the controller's key at its commit and asks no build; the take-in itself is a - // later look's, which this test does not run, so nothing is recorded as asked. - if c := plan.Modules["mesh-controller"]; !c.Key || c.State != "asked" || c.KeyAt != "k1" || len(*asked) != 0 { - t.Fatalf("the first tier is not taken from its key once: %+v, %d ask(s) %v", c, len(*asked), *asked) - } - // Every merge is answered by that one plan. - for _, repository := range []string{"novox/mesh-controller", "novox/mesh-catalog"} { - merges, err := open.inventory.MergesOf(ctx, plan.ID) - if err != nil || len(merges) != 2 { - t.Fatalf("the plan answers %d merge(s) of %s: %v", len(merges), repository, err) - } - } - }) - } -} - -// Issue 1049: a move delivered half before its second merge. The controller adds a module the catalogue removes, in one -// group: the merges are one batch and one plan in either order, no build of the module is asked (it is registered from -// the builder's key, ADR 0501 §6), and the catalogue's removal does not forget it. -func TestAMoveOfAModuleAcrossTwoRepositoriesIsOnePlanAndBuildsItOnceFromTheNewOne(t *testing.T) { - for name, controllerFirst := range map[string]bool{"controller_first": true, "catalogue_first": false} { - t.Run(name, func(t *testing.T) { - open := windowed(t) - asked := asksWithPaths(t) - ctx := t.Context() - const head = "feat/whole-modules" - // The module runs on a machine from the catalogue's repository, so its removal there cannot forget it. - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "route-proxy", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/route-proxy", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.Assign(ctx, "anchor", "route-proxy"); err != nil { - t.Fatal(err) - } - manifestsHeld(t, nil) - controller := fromHead(repoMerge("mesh-controller", "k1", t0), head) - controller.Paths = []string{"modules/route-proxy/module.json", "modules/route-proxy/main.go"} - controller.ModuleDirs, controller.ModuleDirsSaid = []string{"modules/route-proxy"}, true - catalog := fromHead(catalogueMerge("cat1aaaaaaaa", "route-proxy", t0), head) - catalog.Removed = []string{"modules/route-proxy/module.json"} - catalog.ModuleDirs, catalog.ModuleDirsSaid = nil, true - states := map[string]string{"novox/mesh-controller": "ready", "novox/mesh-catalog": "ready"} - groupStates(t, head, states) - first, second := controller, catalog - if !controllerFirst { - first, second = catalog, controller - } - hear(t, open, first, t0) - states["novox/"+second.Repo] = "merged" - second.MergedAt = t0.Add(20 * time.Second).Format(time.RFC3339Nano) - hear(t, open, second, t0.Add(20*time.Second)) - cutAt(t, open, t0.Add(20*time.Second+90*time.Second)) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 0 { - t.Fatalf("a move: %d walk(s), %d batch(es); want exactly one plan", len(ws), len(bs)) - } - // **No build of it is asked from either repository** (ADR 0501 §6, issue 1851): the move is registered from the - // builder's done key of route-proxy in the controller's repository, on the operator's Approve of the group. - for _, a := range *asked { - if a[1] == "modules/route-proxy" { - t.Errorf("a build of route-proxy was asked: %v", a) - } - } - entries, err := open.inventory.Catalogued(ctx) - if err != nil { - t.Fatal(err) - } - kept := false - for _, e := range entries { - kept = kept || e.Manifest.Module == "route-proxy" - } - if !kept { - t.Fatal("the catalogue's removal forgot route-proxy, which a machine runs") - } - }) - } -} diff --git a/cmd/mesh-controller/operator_wait_integration_test.go b/cmd/mesh-controller/operator_wait_integration_test.go deleted file mode 100644 index 98a468ef..00000000 --- a/cmd/mesh-controller/operator_wait_integration_test.go +++ /dev/null @@ -1,133 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -func TestTheNeedsOperatorConditionNamesTheAct(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}} - if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 1}, time.Now()); err != nil { - t.Fatal(err) - } - if got, _ := needsOperatorOpen(t, k); got != nil { - t.Fatal("raised on one statement") - } - if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 2}, time.Now()); err != nil { - t.Fatal(err) - } - got, open := needsOperatorOpen(t, k) - if got == nil { - t.Fatalf("not raised on two statements: %+v", open) - } - for _, c := range open { - if c.Kind == kindModuleUnhealthy { - t.Fatalf("raised as a fault too: %+v", c) - } - } - if got.Kind != kindNeedsOperator || got.Resolver != conditions.ResolverOperator || got.Severity != conditions.Warning { - t.Fatalf("the condition: %+v", got) - } - if !strings.Contains(got.Needs, "type the password of the source games at workstation's desk prompt") || - !strings.Contains(got.Explanation, "hidden prompt opens at the desk") { - t.Fatalf("its needs do not name the act: %q", got.Needs) - } - if strings.Contains(strings.ToLower(got.Explanation), "nothing for you") || !strings.Contains(got.Explanation, "nothing was undone") { - t.Fatalf("its explanation: %q", got.Explanation) - } - if !strings.Contains(got.Summary, "smb-password-games") || !strings.Contains(got.Summary, "nox secret ask workstation mounts smb-password-games") { - t.Fatalf("its summary does not name the secret and the line: %q", got.Summary) - } - // Long open is still a warning: only the operator can end it. - if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 3}, time.Now().Add(48*time.Hour)); err != nil { - t.Fatal(err) - } - if got, _ := needsOperatorOpen(t, k); got == nil || got.Severity == conditions.Urgent { - t.Fatalf("after two days: %+v", got) - } - // Given: the next statement does not say it, and it clears. - if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil { - t.Fatal(err) - } - if got, _ := needsOperatorOpen(t, k); got != nil { - t.Fatal("not cleared once given") - } -} - -func TestASettingsWaitAsksForTheApproval(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(usernameWait)}} - for i := 1; i <= 2; i++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil { - t.Fatal(err) - } - } - got, _ := needsOperatorOpen(t, k) - if got == nil || !strings.Contains(got.Needs, "approve smb-users of mounts on workstation when it is proposed to you") { - t.Fatalf("the condition: %+v", got) - } -} - -func TestAWaitThatFailsItsCheckRaisesUnhealthy(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - checked := checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait)}, - factsGiven(map[string]time.Time{"smb-password-games": time.Now()})) - rs := map[string][]inventory.ResourceHealth{"mounts": checked} - for i := 1; i <= 2; i++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil { - t.Fatal(err) - } - } - got, open := needsOperatorOpen(t, k) - if got != nil { - t.Fatalf("a wait for a secret given raised needs-operator: %+v", got) - } - unhealthy := false - for _, c := range open { - unhealthy = unhealthy || c.Key == moduleUnhealthyKey("mounts", "workstation") - } - if !unhealthy { - t.Fatalf("not raised as unhealthy: %+v", open) - } -} - -func TestANeedsOperatorThatBecameUnhealthySaysSo(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - waiting := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}} - for i := 1; i <= 2; i++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", waiting, map[string]int{"mounts": i}, time.Now()); err != nil { - t.Fatal(err) - } - } - broken := waitingResource() - broken.State, broken.Reason, broken.Waits = link.StateUnhealthy, "the source games refused its login", nil - for i := 3; i <= 4; i++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {broken}}, - map[string]int{"mounts": i}, time.Now()); err != nil { - t.Fatal(err) - } - } - got, open := needsOperatorOpen(t, k) - if got != nil { - t.Fatal("needs-operator still open after it became a fault") - } - found := false - for _, c := range open { - found = found || c.Key == moduleUnhealthyKey("mounts", "workstation") - } - if !found { - t.Fatalf("the fault is not raised: %+v", open) - } -} diff --git a/cmd/mesh-controller/operator_wait_test.go b/cmd/mesh-controller/operator_wait_test.go deleted file mode 100644 index 15879fa7..00000000 --- a/cmd/mesh-controller/operator_wait_test.go +++ /dev/null @@ -1,147 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A part that waits for the operator's secret or setting (novox/hq ADR 0283, issue 386). - -var mountsManifest = catalogue.Manifest{Module: "mounts", Version: "1", - Settings: map[string]catalogue.SettingDeclaration{"smb-users": {}, "sources": {}}, - OwnSecrets: catalogue.OwnSecrets{ - "smb-password-*": {Path: "/s/smb-password-*.secret", IssuedBy: catalogue.IssuedOutside}, - "broker": {Path: "/s/broker"}, - "made": {Path: "/s/made", Taken: catalogue.TakenAtStart}, - "licence": {Path: "/s/licence", IssuedBy: catalogue.IssuedOutside}, - }} - -var passwordWait = inventory.Wait{Part: "the source games", Secret: "smb-password-games", - What: "the password of the source games"} - -var usernameWait = inventory.Wait{Part: "the source games", Setting: "smb-users", What: "the username of the source games"} - -func waitingResource(waits ...inventory.Wait) inventory.ResourceHealth { - return inventory.ResourceHealth{Module: "mounts", Resource: "mounts.watch", Kind: "process", - Target: "mesh-mounts-watch.service", State: link.StateWaiting, Check: "tool", - Reason: "the source games waits for its password", Waits: waits} -} - -func factsGiven(given map[string]time.Time) operatorWaitFacts { - return operatorWaitFacts{manifests: map[string]catalogue.Manifest{"mounts": mountsManifest}, - given: map[string]map[string]time.Time{"mounts@workstation": given}} -} - -// Rule 3: a wait is excused only when what it names is the module's, issued outside the mesh, and not given there. -func TestAWaitIsExcusedOnlyWhenItChecksOut(t *testing.T) { - at := time.Date(2026, 10, 10, 15, 8, 0, 0, time.UTC) - for _, c := range []struct { - name string - w inventory.Wait - f operatorWaitFacts - says string // "" when excused - }{ - {"a member of an outside family, not given", passwordWait, factsGiven(nil), ""}, - {"an outside secret by name, not given", inventory.Wait{Part: "p", Secret: "licence", What: "w"}, factsGiven(nil), ""}, - {"a declared setting", usernameWait, factsGiven(nil), ""}, - {"a member given", passwordWait, factsGiven(map[string]time.Time{"smb-password-games": at}), "was given at 2026-10-10 15:08"}, - {"a secret not declared", inventory.Wait{Part: "p", Secret: "smb-credentials", What: "w"}, factsGiven(nil), "does not declare"}, - {"a secret the mesh makes", inventory.Wait{Part: "p", Secret: "made", What: "w"}, factsGiven(nil), "mesh makes itself"}, - {"the bus account", inventory.Wait{Part: "p", Secret: "broker", What: "w"}, factsGiven(nil), "mesh makes itself"}, - {"a setting not declared", inventory.Wait{Part: "p", Setting: "logins", What: "w"}, factsGiven(nil), "does not declare"}, - {"both", inventory.Wait{Part: "p", Secret: "licence", Setting: "smb-users", What: "w"}, factsGiven(nil), "one secret, one setting or one provision"}, - {"neither", inventory.Wait{Part: "p", What: "w"}, factsGiven(nil), "one secret, one setting or one provision"}, - {"no manifest known", passwordWait, operatorWaitFacts{}, "no manifest"}, - {"what was given cannot be read", passwordWait, - operatorWaitFacts{manifests: map[string]catalogue.Manifest{"mounts": mountsManifest}}, "could not be read"}, - } { - err := checkWait("mounts", "workstation", c.w, c.f) - switch { - case c.says == "" && err != nil: - t.Errorf("%s: refused: %v", c.name, err) - case c.says != "" && (err == nil || !strings.Contains(err.Error(), c.says)): - t.Errorf("%s: %v; want a refusal saying %q", c.name, err, c.says) - } - } -} - -// A waiting resource whose wait does not check out, or that names nothing, is judged unhealthy, saying why. -func TestAWaitThatFailsItsCheckIsUnhealthy(t *testing.T) { - given := factsGiven(map[string]time.Time{"smb-password-games": time.Now()}) - got := checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait)}, given) - if got[0].State != link.StateUnhealthy || !strings.Contains(got[0].Reason, "was given") { - t.Fatalf("a wait for a secret given: %+v", got[0]) - } - got = checkWaiting("workstation", []inventory.ResourceHealth{waitingResource()}, factsGiven(nil)) - if got[0].State != link.StateUnhealthy || !strings.Contains(got[0].Reason, "names nothing") { - t.Fatalf("a wait naming nothing: %+v", got[0]) - } - got = checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait, usernameWait)}, factsGiven(nil)) - if got[0].State != link.StateWaiting { - t.Fatalf("two waits that check out: %+v", got[0]) - } -} - -// Rule 4: the gate passes a module whose only parts not healthy wait for the operator, carrying the wait; anything -// else beside it is judged as before; and any build is excused, not only one that added something. -func TestTheGatePassesAWaitForTheOperatorCarriedAlong(t *testing.T) { - now := time.Now() - since := now.Add(-time.Minute) - healthy := inventory.ResourceHealth{Module: "mounts", Resource: "mounts.apply", Kind: "process", - Target: "mesh-mounts-apply.service", State: link.StateHealthy} - f := gateFacts{now: now, waits: factsGiven(nil), groupsAdded: map[string]bool{"mounts": false}, - health: map[string]inventory.NodeHealth{"workstation": {Node: "workstation", HeardAt: now, - Resources: []inventory.ResourceHealth{healthy, waitingResource(passwordWait)}}}} - h, why := moduleHealthWord("mounts", "workstation", since, f) - if h != healthPerson || !strings.Contains(why, "waits for the operator: the password of the source games") || - !strings.Contains(why, "nox secret ask workstation mounts smb-password-games") { - t.Fatalf("an excused wait reads %v %q; want a wait for a person naming the act", h, why) - } - // A second resource unhealthy beside it: not yet, as before. - down := healthy - down.State, down.Reason = link.StateUnhealthy, "down" - f.health["workstation"] = inventory.NodeHealth{Node: "workstation", HeardAt: now, - Resources: []inventory.ResourceHealth{down, waitingResource(passwordWait)}} - if h, why := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet { - t.Fatalf("a resource down beside the wait reads %v %q", h, why) - } - // The password given and the module still saying it waits: not excused. - f.waits = factsGiven(map[string]time.Time{"smb-password-games": now}) - f.health["workstation"] = inventory.NodeHealth{Node: "workstation", HeardAt: now, - Resources: []inventory.ResourceHealth{healthy, waitingResource(passwordWait)}} - if h, why := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet || !strings.Contains(why, "was given") { - t.Fatalf("a wait for a secret given reads %v %q", h, why) - } - // Facts never read (no manifest): never excused. - f.waits = operatorWaitFacts{} - if h, _ := moduleHealthWord("mounts", "workstation", since, f); h != healthNotYet { - t.Fatalf("a wait nothing could check reads %v", h) - } -} - -func needsOperatorOpen(t *testing.T, k *conditions.Keeper) (*conditions.Condition, []conditions.Condition) { - t.Helper() - open, err := k.Open(t.Context()) - if err != nil { - t.Fatal(err) - } - for i, c := range open { - if c.Key == needsOperatorKey("mounts", "workstation") { - return &open[i], open - } - } - return nil, open -} - -// Rule 5: two statements of an excused wait raise needs-operator, the operator's, a warning however long, naming the -// act; a statement without it clears it. - -// A wait that fails its check is the module's own fault: unhealthy, with why, and no needs-operator. - -// A module that waited and is then broken says so when the wait clears, and the other way round. diff --git a/cmd/mesh-controller/over_budget.go b/cmd/mesh-controller/over_budget.go deleted file mode 100644 index 03c578ff..00000000 --- a/cmd/mesh-controller/over_budget.go +++ /dev/null @@ -1,125 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "time" - - "github.com/nats-io/nats.go" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A delivery over its budget is loud (novox/hq ADR 0282 decision 7, issue 382): the self-check reads -// mesh-delivery's `times` and raises the warning `delivery..over-budget` when the newest delivery of a -// class whose delivery time is known took longer than its class's budget — three minutes for a leaf module (ADR -// 0304 decision 4), ten for a core module — naming the delivery and its longest phase. It clears when the next delivery of that class -// lands within its budget. Measurement only: the condition says, it never holds or acts on a delivery. - -// probeBudgetsID is the probe that reads the delivery times. -const probeBudgetsID = "D16" - -// kindOverBudget is what a delivery over its class's budget raises. -const kindOverBudget = "over-budget" - -// deliveryBudgets are the budgets by class (ADR 0282 decision 1, the leaf's as ADR 0304 decision 4 changed it); -// the probe raises nothing for another class. -var deliveryBudgets = map[string]time.Duration{inventory.ClassLeaf: 3 * time.Minute, inventory.ClassCore: 10 * time.Minute} - -// timesAnswer is what mesh-delivery's `times` answers, as far as the probe reads it. -type timesAnswer struct { - Classes []timesClass `json:"classes"` -} - -// timesClass is one class's line of `times`. -type timesClass struct { - Class string `json:"class"` - Latest *timesLatest `json:"latest,omitempty"` -} - -// timesLatest is the newest delivery of a class whose delivery time is known. -type timesLatest struct { - ID string `json:"id"` - TookMS int64 `json:"took_ms"` - Longest string `json:"longest,omitempty"` - Landed string `json:"landed,omitempty"` -} - -// deliveryTimes is what the delivery's owner says of its delivery times; nothing when no holder is on record or -// none answers (D3 says that one). -func deliveryTimes(ctx context.Context, conn *nats.Conn, held bool) (*timesAnswer, error) { - if !held { - return nil, nil - } - raw, err := askDeliveryOwner(ctx, conn, "times", map[string]any{}) - if errors.Is(err, link.ErrNothingServes) { - return nil, nil - } - if err != nil { - return nil, err - } - var a timesAnswer - if err := json.Unmarshal(raw, &a); err != nil { - return nil, fmt.Errorf("%s.times answered something unreadable: %w", catalogue.DeliverySeat, err) - } - return &a, nil -} - -// overBudgetObservations are the conditions of the classes whose newest delivery took longer than its budget: -// strictly longer, so a delivery of exactly its budget is within it. -func overBudgetObservations(a *timesAnswer) []conditions.Observation { - if a == nil { - return nil - } - var out []conditions.Observation - for _, c := range a.Classes { - budget, ok := deliveryBudgets[c.Class] - if !ok || c.Latest == nil { - continue - } - took := time.Duration(c.Latest.TookMS) * time.Millisecond - if took <= budget { - continue - } - longest := c.Latest.Longest - if longest == "" { - longest = "not known" - } - out = append(out, conditions.Observation{Scope: conditions.ScopeDelivery, ID: c.Class, Kind: kindOverBudget, - Severity: conditions.Warning, - Summary: fmt.Sprintf("the %s delivery %s took %s from its merge to running everywhere, over its budget of %s; "+ - "its longest phase: %s — `mesh-delivery.times`", c.Class, c.Latest.ID, humanDuration(took), - humanDuration(budget), longest), - Said: fmt.Sprintf("%s took %s (budget %s), longest phase %s", c.Latest.ID, took.Round(time.Second), budget, longest), - Headline: fmt.Sprintf("A %s delivery took %s, over its %s budget", c.Class, humanDuration(took), - humanDuration(budget)), - Explanation: fmt.Sprintf("The delivery %s took %s from its merge until every machine ran it; a %s module is "+ - "held to %s (ADR 0282). Most of the time went to %s. Nothing was held or changed because of this: it is "+ - "a measurement.", c.Latest.ID, humanDuration(took), c.Class, humanDuration(budget), longest), - Resolved: fmt.Sprintf("the next %s delivery lands within %s", c.Class, humanDuration(budget)), - }) - } - return out -} - -// probeBudgets is D16: the newest delivery of each class lands within its class's budget. -func probeBudgets(ctx context.Context, d *doctor) ([]conditions.Observation, error) { - entries, err := d.open.inventory.Catalogued(ctx) - if err != nil { - return nil, err - } - var conn *nats.Conn - if d.js != nil { - conn = d.js.Conn() - } - a, err := deliveryTimes(ctx, conn, deliverySeatHeld(entries)) - if err != nil { - return nil, err - } - return overBudgetObservations(a), nil -} diff --git a/cmd/mesh-controller/over_budget_integration_test.go b/cmd/mesh-controller/over_budget_integration_test.go deleted file mode 100644 index c16b24ad..00000000 --- a/cmd/mesh-controller/over_budget_integration_test.go +++ /dev/null @@ -1,50 +0,0 @@ -//go:build integration - -package main - -import ( - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestAWalksClassAndWindowAreReadAtItsCut(t *testing.T) { - entries := []inventory.Entry{ - {Manifest: catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}}, - {Manifest: catalogue.Manifest{Module: "gitea"}}, - } - walk := func(modules ...string) inventory.Plan { - p := inventory.Plan{Modules: map[string]*inventory.PlanModule{}} - for _, m := range modules { - p.Modules[m] = &inventory.PlanModule{} - } - return p - } - for want, w := range map[string]inventory.Plan{ - inventory.ClassLeaf: walk("gitea"), inventory.ClassCore: walk("gitea", "mesh-controller"), - } { - if got := classOf(w, entries); got != want { - t.Errorf("%v: %s, want %s", w.Modules, got, want) - } - } - if got := classOf(walk("dnsmasq"), entries); got != inventory.ClassCore { - t.Errorf("the resolver's holder is core: %s", got) - } - now := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC) - batch := inventory.Plan{Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{ - ClosesAt: now.Add(-20 * time.Second), AtMost: now.Add(5 * time.Minute)}}} - times := walkTimesAtCut(batch, walk("gitea"), entries, now) - if times.Cut == nil || !times.Cut.Equal(now) || times.WindowClosed == nil || !times.WindowClosed.Equal(now.Add(-20*time.Second)) || - times.Class != inventory.ClassLeaf { - t.Fatalf("times at the cut: %+v", times) - } - batch.Delivery.Batch.AtMost = now.Add(-time.Minute) - if times := walkTimesAtCut(batch, walk("gitea"), entries, now); !times.WindowClosed.Equal(now.Add(-time.Minute)) { - t.Fatalf("a window closed at its maximum: %v", times.WindowClosed) - } - if times := walkTimesAtCut(inventory.Plan{Delivery: &inventory.PlanDelivery{Alone: true}}, walk("gitea"), entries, now); times.WindowClosed != nil { - t.Fatalf("a merge walked alone had no window: %v", times.WindowClosed) - } -} diff --git a/cmd/mesh-controller/over_budget_test.go b/cmd/mesh-controller/over_budget_test.go deleted file mode 100644 index 9ba16ee4..00000000 --- a/cmd/mesh-controller/over_budget_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A leaf delivery of 3 minutes 10 seconds raises delivery.leaf.over-budget naming its longest phase; one of -// exactly three minutes, a core one of nine and a class without a budget raise nothing (ADR 0282 decision 7, -// the leaf's budget as ADR 0304 decision 4 changed it). -func TestADeliveryOverItsBudgetIsLoud(t *testing.T) { - a := ×Answer{Classes: []timesClass{ - {Class: inventory.ClassLeaf, Latest: ×Latest{ID: "novox/mesh-catalog@abc", TookMS: (3*time.Minute + 10*time.Second).Milliseconds(), - Longest: "judgement 2m40s"}}, - {Class: inventory.ClassCore, Latest: ×Latest{ID: "novox/mesh-controller@def", TookMS: (9 * time.Minute).Milliseconds(), - Longest: "build 1m"}}, - {Class: "unclassed", Latest: ×Latest{ID: "x@y", TookMS: time.Hour.Milliseconds()}}, - }} - obs := overBudgetObservations(a) - if len(obs) != 1 { - t.Fatalf("one class over its budget, got %d: %+v", len(obs), obs) - } - o := obs[0] - if o.Key() != "delivery.leaf.over-budget" || !strings.Contains(o.Summary, "judgement 2m40s") || - !strings.Contains(o.Summary, "novox/mesh-catalog@abc") { - t.Fatalf("the condition names its delivery and longest phase: %s — %s", o.Key(), o.Summary) - } - // The next leaf delivery within its budget clears it: the probe raises nothing for the class. - a.Classes[0].Latest = ×Latest{ID: "novox/mesh-catalog@ghi", TookMS: (3 * time.Minute).Milliseconds()} - if obs := overBudgetObservations(a); len(obs) != 0 { - t.Fatalf("a delivery of exactly its budget is within it: %+v", obs) - } - a.Classes[1].Latest.TookMS = (10*time.Minute + time.Second).Milliseconds() - if obs := overBudgetObservations(a); len(obs) != 1 || obs[0].Key() != "delivery.core.over-budget" { - t.Fatalf("a core delivery over ten minutes: %+v", obs) - } - if obs := overBudgetObservations(nil); obs != nil { - t.Fatal("no answer raises nothing") - } - // No delivery of a class with a known time yet: nothing said of it. - if obs := overBudgetObservations(×Answer{Classes: []timesClass{{Class: inventory.ClassLeaf}}}); len(obs) != 0 { - t.Fatalf("a class with no delivery: %+v", obs) - } -} - -// The probe's question is one the controller's grant names: a question the bus refuses checks nothing. -func TestTheControllerMayAskForTheDeliveryTimes(t *testing.T) { - found := false - for _, v := range broker.VerbsTheControllerAsksTheDeliveryOwner { - found = found || (v.Seat == catalogue.DeliverySeat && v.Verb == "times") - } - if !found { - t.Fatal("the grant does not name mesh-delivery.times") - } -} - -// A walk's class is core when it walks a module of the controller's own path or one holding the mesh's resolver, -// leaf otherwise; its window closed at its batch's window, or its maximum, never after its cut. - -// What each machine of the rest was sent is kept only for the machines the send reached. -func TestTheRestIsKeptForTheMachinesTheSendReached(t *testing.T) { - got := restOf([]string{"ace", "g14"}, []string{"ace", "shanks"}, map[string]inventory.SentDeclaration{"ace": {Digest: "d1"}}) - if len(got) != 1 || got["ace"].Digest != "d1" { - t.Fatalf("rest: %+v", got) - } - if restOf([]string{"g14"}, []string{"ace"}, nil) != nil { - t.Fatal("no machine reached: nothing kept") - } -} diff --git a/cmd/mesh-controller/own_grants.go b/cmd/mesh-controller/own_grants.go index 8964a078..b4fe9a5b 100644 --- a/cmd/mesh-controller/own_grants.go +++ b/cmd/mesh-controller/own_grants.go @@ -60,9 +60,9 @@ func startWithOwnGrants(ctx context.Context, c ownGrantsCarrier, then func()) { // ownGrantsOf is the carrier of the serving controller: the grants step's reading and its grants-only send. func ownGrantsOf(open *stores) ownGrantsCarrier { return ownGrantsCarrier{ - behind: func(ctx context.Context) (string, bool, error) { return brokerBehindOf(ctx, open, nil) }, + behind: func(ctx context.Context) (string, bool, error) { return brokerBehind(ctx, open, nil) }, send: func(ctx context.Context, holder string) error { - _, err := sendRollout(keepingEveryBuild(withScope(ctx, sendScope{}), holder), open, []string{holder}) + _, err := sendToEach(ctx, open, []string{holder}) return err }, } diff --git a/cmd/mesh-controller/own_grants_test.go b/cmd/mesh-controller/own_grants_test.go index d7a8ea4f..1caaf51c 100644 --- a/cmd/mesh-controller/own_grants_test.go +++ b/cmd/mesh-controller/own_grants_test.go @@ -2,7 +2,6 @@ package main import ( "context" - "errors" "strings" "testing" ) @@ -20,54 +19,3 @@ func TestAStartingControllerSendsItsGrantsBeforeItWrites(t *testing.T) { t.Fatalf("the start did %v", order) } } - -// 1800 (3), (4): no send when the list is current or there is no bus with a list; one that cannot be read or sent is -// said once, and the controller goes on. -func TestAStartingControllerSendsNothingWhenTheListIsCurrentAndGoesOnWhenRefused(t *testing.T) { - sends := 0 - send := func(context.Context, string) error { sends++; return nil } - for _, c := range []ownGrantsCarrier{ - {behind: func(context.Context) (string, bool, error) { return "novox", false, nil }, send: send}, - {behind: func(context.Context) (string, bool, error) { return "", true, nil }, send: send}, - } { - if said := c.carry(t.Context()); said != "" || sends != 0 { - t.Fatalf("a current list sent %d and said %q", sends, said) - } - } - refused := ownGrantsCarrier{behind: func(context.Context) (string, bool, error) { return "novox", true, nil }, - send: func(context.Context, string) error { return errNotGrantsOnly }} - went := false - startWithOwnGrants(t.Context(), refused, func() { went = true }) - if !went { - t.Fatal("a refused send stopped the controller") - } - if said := refused.carry(t.Context()); !strings.Contains(said, "could not be sent to novox") || - !strings.Contains(said, errNotGrantsOnly.Error()) { - t.Fatalf("a refused send is said as %q", said) - } - unread := ownGrantsCarrier{behind: func(context.Context) (string, bool, error) { return "", false, errors.New("no store") }, - send: send} - if said := unread.carry(t.Context()); !strings.Contains(said, "cannot be read") || sends != 0 { - t.Fatalf("an unread list: sent %d, said %q", sends, said) - } -} - -// 1800 (2): the start's send is the grants step's: composed with every build of the bus's machine kept at the one it -// runs, so the declaration differs from its last only by the user list (and is refused unsent otherwise, -// errNotGrantsOnly, release.go), whatever a running plan registered since. -func TestAStartingControllersSendKeepsEveryBuildOfTheBusMachine(t *testing.T) { - was := sendRollout - t.Cleanup(func() { sendRollout = was }) - var kept bool - var named []string - sendRollout = func(ctx context.Context, _ *stores, names []string) ([]string, error) { - kept, named = everyBuildKept(ctx, "novox"), names - return names, nil - } - if err := ownGrantsOf(nil).send(t.Context(), "novox"); err != nil { - t.Fatal(err) - } - if !kept || len(named) != 1 || named[0] != "novox" { - t.Fatalf("the start's send keeps every build: %v, names %v", kept, named) - } -} diff --git a/cmd/mesh-controller/own_path_by_claim_test.go b/cmd/mesh-controller/own_path_by_claim_test.go deleted file mode 100644 index 0b02f7ac..00000000 --- a/cmd/mesh-controller/own_path_by_claim_test.go +++ /dev/null @@ -1,93 +0,0 @@ -package main - -import ( - "context" - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// The controller finds the delivery's owner by the claim of the seat mesh-delivery, never by the module's name -// (novox/hq ADR 0437 §3, issue 1210). The gate is a module named mesh-delivery-gate; this is only a fixture name. - -func claimingTheSeat(module string) inventory.Entry { - return inventory.Entry{Manifest: catalogue.Manifest{Module: module, - Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}} -} - -func namedOnly(module string) inventory.Entry { - return inventory.Entry{Manifest: catalogue.Manifest{Module: module}, On: []string{"anchor"}} -} - -// 1210/AC1: a module claiming the seat is on the controller's own path whatever its name, awaiting the delivery's -// word as today's module does; a module named mesh-delivery that claims no seat is not on it. -func TestTheModuleClaimingTheSeatIsOnTheControllersOwnPath(t *testing.T) { - gate := []inventory.Entry{claimingTheSeat("mesh-delivery-gate"), namedOnly("mesh-delivery")} - own := ownPathOf(gate) - if _, on := own["mesh-delivery-gate"]; !on { - t.Error("the module claiming the seat is not on the controller's own path") - } - if _, on := own["mesh-delivery"]; on { - t.Error("a module named mesh-delivery that claims no seat is on the controller's own path") - } - if d := awaitsFor(gate, []string{"gitea", "plex"}); d == nil || d.Awaits != catalogue.DeliverySeat { - t.Errorf("an ordinary walk awaits %+v with the gate holding the seat", d) - } - if d := awaitsFor(gate, []string{"mesh-delivery-gate", "gitea"}); d != nil { - t.Errorf("a walk moving the gate awaits %+v; the gate cannot gate itself", d) - } - if d := awaitsFor(gate, []string{"mesh-delivery", "gitea"}); d == nil { - t.Error("a walk moving a module named mesh-delivery that claims no seat starts at once") - } - if !ownWalk(walkOf("novox/one", "notes", "mesh-delivery-gate"), own) || ownWalk(walkOf("novox/one", "notes", "mesh-delivery"), own) { - t.Error("a walk is on the own path by the claim of the module it plans") - } - if ownPath([]inventory.Entry{claimingTheSeat("mesh-delivery-gate")}) != true || ownPath([]inventory.Entry{namedOnly("mesh-delivery")}) { - t.Error("a merge touches the own path by the claim of the module it touches") - } - if classOfModules([]string{"mesh-delivery-gate"}, gate) != inventory.ClassCore { - t.Error("the module claiming the seat is not core") - } -} - -// 1210/AC2: the merge gate's rule of the module claiming the seat is one reading; a module claiming no delivery -// seat gets its ordinary rule. -func TestTheModuleClaimingTheSeatGetsTheDeliverysGateRule(t *testing.T) { - withTheGatesTimes(t) - claimant := claimingTheSeat("mesh-delivery-gate").Manifest - r := gateRuleOf("mesh-delivery-gate", claimant) - if r.passes != 1 || r.within != 0 || !strings.HasPrefix(r.said, "mesh-delivery's rule: one reading (novox/hq ADR 0351)") { - t.Errorf("the gate's rule is %+v", r) - } - r = gateRuleOf("mesh-delivery", catalogue.Manifest{Module: "mesh-delivery"}) - if !strings.HasPrefix(r.said, "a leaf module's rule") { - t.Errorf("a module named mesh-delivery claiming no seat gets %q", r.said) - } -} - -// 1210/AC3: with both today's module and the gate claiming the seat (R3d's group), both are on the own path and -// both get the delivery's rule. -func TestBothClaimantsAreOnTheOwnPathAndGetTheRule(t *testing.T) { - withTheGatesTimes(t) - both := []inventory.Entry{claimingTheSeat("mesh-delivery"), claimingTheSeat("mesh-delivery-gate")} - own := ownPathOf(both) - for _, m := range []string{"mesh-delivery", "mesh-delivery-gate"} { - if _, on := own[m]; !on { - t.Errorf("%s is not on the own path", m) - } - if r := gateRuleOf(m, claimingTheSeat(m).Manifest); r.passes != 1 || !strings.HasPrefix(r.said, "mesh-delivery's rule") { - t.Errorf("%s gets %+v", m, r) - } - if d := awaitsFor(both, []string{m, "gitea"}); d != nil { - t.Errorf("a walk moving %s awaits %+v", m, d) - } - } -} - -// theDeliveryClaims is a catalogue whose one module claiming the seat is today's mesh-delivery, for the tests that -// announce a merge from it. -func theDeliveryClaims(context.Context) ([]inventory.Entry, error) { - return []inventory.Entry{claimingTheSeat("mesh-delivery")}, nil -} diff --git a/cmd/mesh-controller/pending_integration_test.go b/cmd/mesh-controller/pending_integration_test.go deleted file mode 100644 index 306ae903..00000000 --- a/cmd/mesh-controller/pending_integration_test.go +++ /dev/null @@ -1,606 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "errors" - "slices" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -func failedBuild(t *testing.T, open *stores, id string) { - t.Helper() - if err := (builds{open.inventory, open}).Built(t.Context(), outcome(id, "modules/sensors", "3da80a4b00aa", - "", "boom")); err != nil { - t.Fatal(err) - } -} - -func aCatalogueMesh(t *testing.T) *stores { - t.Helper() - open := aMesh(t) - if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: "networkmanager", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - return open -} - -// mergeAdding keeps the build request a merge adding a module asked before R4e (issue 300): nothing asks one since (ADR -// 0501 §6), and the pending assignments still read the ones kept. These tests are of what an assignment does with one. -func mergeAdding(t *testing.T, open *stores, dir, commit string) { - t.Helper() - if err := open.inventory.RecordBuildRequest(t.Context(), inventory.BuildRequest{ID: "b-" + dir, - Repository: "novox/mesh-catalog", Seat: "git", Path: dir, Ref: "main", Commit: commit, For: "merge", - At: time.Now().UTC()}); err != nil { - t.Fatal(err) - } -} - -func TestAnAssignmentWaitsForTheBuildInFlight(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - - requests, err := open.inventory.RequestsNamed(ctx, "sensors") - if err != nil || len(requests) != 1 || requests[0].For != "merge" || requests[0].Commit != "3da80a4b00aa" { - t.Fatalf("the merge's build request: %+v %v", requests, err) - } - - said, err := assign(ctx, open, "laptop", "sensors") - if err != nil { - t.Fatalf("an assignment while its build runs was refused: %v", err) - } - t.Logf("assign laptop sensors, while its build runs:\n%s", said) - for _, want := range []string{"being built from novox/mesh-catalog@3da80a4b (modules/sensors)", - "build b-modules/sensors", "kept as pending", "the controller assigns sensors to laptop", "`unassign laptop sensors`"} { - if !strings.Contains(said, want) { - t.Fatalf("the answer does not say %q:\n%s", want, said) - } - } - if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("a module not registered was assigned") - } - pending := pendingOf(t, open) - if len(pending) != 1 || pending[0].State != inventory.PendingWaiting || pending[0].Build != "b-modules/sensors" { - t.Fatalf("pending: %+v", pending) - } - if again, err := assign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(again, "already waits") { - t.Fatalf("a second assignment: %q %v", again, err) - } - - asked, err := theThreeQuestions(ctx, open) - if err != nil { - t.Fatal(err) - } - if asked.well() { - t.Fatal("status called the mesh well while an assignment waits") - } - body, err := statusAsJSON(asked) - if err != nil { - t.Fatal(err) - } - var doc meshStatus - if err := json.Unmarshal(body, &doc); err != nil { - t.Fatal(err) - } - if len(doc.Pending) != 1 || doc.Pending[0].State != "waiting" || doc.Pending[0].Module != "sensors" { - t.Fatalf("status says pending %+v", doc.Pending) - } - - if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", - "3da80a4b00aa", "sensors", "")); err != nil { - t.Fatal(err) - } - if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("the build registered sensors and the pending assignment was not made") - } - pending = pendingOf(t, open) - if len(pending) != 1 || pending[0].State != inventory.PendingApplied || - !strings.HasSuffix(pending[0].Note, "`push laptop` sends it") { - t.Fatalf("pending after the build: %+v", pending) - } -} - -func TestAStatusReadSettlesNothing(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) - for range 2 { - if _, err := theThreeQuestions(ctx, open); err != nil { - t.Fatal(err) - } - } - if slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingWaiting { - t.Fatalf("a status read settled a pending assignment: %+v", pendingOf(t, open)) - } - said := settlePending(ctx, open, time.Now()) - if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingApplied { - t.Fatalf("the tick did not make it: %v %+v", said, pendingOf(t, open)) - } -} - -func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", - "3da80a4b00aa", "", "go build: undefined: sensorsRead")); err != nil { - t.Fatal(err) - } - pending := pendingOf(t, open) - if len(pending) != 1 || pending[0].State != inventory.PendingExpired { - t.Fatalf("pending after a failed build: %+v", pending) - } - // Its words since R4g (whyNotBuilt): the build, when it was heard, and that it failed (issue 1938: kept from an older - // run until the inventory changed beneath it). - for _, want := range []string{"expired", "modules/sensors in novox/mesh-catalog; its last build, b-modules/sensors at", ", failed: go build", - "undefined: sensorsRead", "nothing was assigned", "build \"true\""} { - if !strings.Contains(pending[0].Note, want) { - t.Fatalf("the expiry does not say %q: %s", want, pending[0].Note) - } - } - if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("a failed build's module was assigned") - } - - settlePending(ctx, open, time.Now()) - key := pendingObservation(pending[0]).Key() - c, found, err := conditionsFrom.Get(ctx, key) - if err != nil || !found || c.Kind != kindPendingEnded { - t.Fatalf("no condition %s for the assignment not made: %+v %v %v", key, c, found, err) - } - if asked, err := theThreeQuestions(ctx, open); err != nil || asked.well() || len(asked.conditions) == 0 { - t.Fatalf("status does not hold the open condition: %v", err) - } - - _, err = assign(ctx, open, "laptop", "sensors") - if !errors.Is(err, inventory.ErrNoSuchModule) { - t.Fatalf("a module whose build failed: %v", err) - } - for _, want := range []string{"sensors is known and not registered", "its last build, b-modules/sensors", - "failed: go build: undefined: sensorsRead", "registered from the builder's done key", "novox/mesh-catalog (path modules/sensors)"} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("the refusal does not say %q:\n%v", want, err) - } - } - - said, err := unassign(ctx, open, "laptop", "sensors") - if err != nil || !strings.Contains(said, "taken back") { - t.Fatalf("unassign of an expired pending assignment: %q %v", said, err) - } - settlePending(ctx, open, time.Now()) - if _, found, _ := conditionsFrom.Get(ctx, key); found { - t.Fatal("the condition stayed open after the assignment was taken back") - } - // Nothing of it keeps status from being well any more: no open pending assignment, no open condition of it. - asked, err := theThreeQuestions(ctx, open) - if err != nil || pendingOpen(asked.pending) { - t.Fatalf("status still counts the pending assignment: %+v %v", asked.pending, err) - } - for _, c := range asked.conditions { - if c.Kind == kindPendingEnded { - t.Fatalf("status still holds the condition: %+v", c) - } - } -} - -func TestTheConditionClearsWhenTheModuleIsAssignedLater(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", - "3da80a4b00aa", "", "boom")); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - key := pendingObservation(pendingOf(t, open)[0]).Key() - register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - if _, found, _ := conditionsFrom.Get(ctx, key); found { - t.Fatal("the condition stayed open after the module was assigned") - } -} - -func TestAFailedAskIsNotABuildInFlight(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-waited", Repository: "novox/mesh-catalog", - Seat: "git", Path: "modules/gauges", Ref: "main", For: "build"}); err != nil { - t.Fatal(err) - } - if err := inv.MarkNotAsked(ctx, "build-waited", "cannot submit a build: no responders"); err != nil { - t.Fatal(err) - } - _, err := assign(ctx, open, "laptop", "gauges") - if err == nil || !strings.Contains(err.Error(), "build build-waited, asked by build, was not handed over") { - t.Fatalf("a build not handed over: %v", err) - } - // Heard first, the outcome stands: marking it afterwards changes nothing. - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-waited", Repository: "novox/mesh-catalog", Ref: "main", - Path: "modules/gauges", On: "anchor", Failed: "the real failure"}); err != nil { - t.Fatal(err) - } - _, err = assign(ctx, open, "laptop", "gauges") - if err == nil || !strings.Contains(err.Error(), "failed: the real failure") { - t.Fatalf("an outcome heard after a failed wait: %v", err) - } -} - -func TestAPlansBuildRequestIsThePlans(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa", - Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - // A walk asks no build, so it leaves no build request (askModule takes the module from its key): the plan's app is - // taken from the key at the commit the plan carries. - requests, err := open.inventory.RequestsNamed(ctx, "app") - if err != nil || len(requests) != 0 { - t.Fatalf("a request was recorded for the plan: %+v %v", requests, err) - } - recent, err := open.inventory.RecentPlans(ctx, 5) - if err != nil || len(recent) != 1 { - t.Fatalf("the plans: %+v %v", recent, err) - } - if app := recent[0].Modules["app"]; app == nil || !app.Key || app.KeyAt != "c1aaaaaaaa" { - t.Fatalf("the plan's app is not taken from its key at the commit it carries: %+v", app) - } -} - -func TestABuildBeingRegisteredIsInFlight(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-done", Repository: "novox/mesh-catalog", - Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge"}); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-done", Repository: "novox/mesh-catalog", Ref: "main", - Path: "modules/sensors", Module: "sensors", On: "anchor"}); err != nil { - t.Fatal(err) - } - said, err := assign(ctx, open, "laptop", "sensors") - if err != nil || !strings.Contains(said, "built and being registered") || !strings.Contains(said, "kept as pending") { - t.Fatalf("a build being registered: %q %v", said, err) - } - settlePending(ctx, open, time.Now().Add(registerGrace+time.Minute)) - p := pendingOf(t, open) - if len(p) != 1 || p[0].State != inventory.PendingExpired || !strings.Contains(p[0].Note, "recorded and not registered") { - t.Fatalf("past the grace: %+v", p) - } -} - -func TestAnUnknownModuleIsRefusedWithTheClosestNames(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "sensord", Version: "1"}) - asked := asksWithPaths(t) - _, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") - if !errors.Is(err, inventory.ErrNoSuchModule) { - t.Fatalf("an unknown module: %v", err) - } - for _, want := range []string{"no module of that name: sensors", "the catalogue holds no module of that name", - "no build request the controller kept (the last 30 days)", "the closest names it holds: sensord", - "asks for nothing: the controller asks no build"} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("the refusal does not say %q:\n%v", want, err) - } - } - if strings.Contains(err.Error(), "merge") { - t.Fatalf("the refusal claims a merge it never looked at: %v", err) - } - if len(*asked) != 0 || len(pendingOf(t, open)) != 0 { - t.Fatalf("an unknown module asked %v, pending %+v", *asked, pendingOf(t, open)) - } -} - -func TestAnActWithAModuleNotRegisteredIsRefusedWhole(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - register(t, open, catalogue.Manifest{Module: "known", Version: "1"}) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - _, err := assign(ctx, open, "laptop", "known", "sensors") - if err == nil || !strings.Contains(err.Error(), "nothing was assigned") || !strings.Contains(err.Error(), "being built") { - t.Fatalf("an act naming a module in flight: %v", err) - } - if slices.Contains(assignedTo(t, open, "laptop"), "known") || len(pendingOf(t, open)) != 0 { - t.Fatal("an act refused whole was made in part") - } -} - -func TestUnassignWithdrawsAPendingAssignment(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - said, err := unassign(ctx, open, "laptop", "sensors") - if err != nil || !strings.Contains(said, "withdrawn") { - t.Fatalf("unassign of a pending assignment: %q %v", said, err) - } - if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", - "3da80a4b00aa", "sensors", "")); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("a withdrawn assignment was made") - } -} - -func TestAWithdrawalNeverOverridesAClaim(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - p := pendingOf(t, open)[0] - if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok { - t.Fatalf("claim: %v %v", ok, err) - } - _, err := unassign(ctx, open, "laptop", "sensors") - if err == nil || !strings.Contains(err.Error(), "is being assigned sensors now") { - t.Fatalf("unassign of a claimed pending assignment: %v", err) - } - if got := pendingOf(t, open)[0]; got.State != inventory.PendingApplying { - t.Fatalf("the claim was overridden: %+v", got) - } - // Pending(zero) is the waiting ones alone (review point 7). - if waiting, err := inv.Pending(ctx, time.Time{}); err != nil || len(waiting) != 0 { - t.Fatalf("Pending(zero) read a claimed row: %+v %v", waiting, err) - } - - // Left by a controller that stopped: back to waiting, since sensors is not assigned. - settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) - if got := pendingOf(t, open)[0]; got.State != inventory.PendingWaiting { - t.Fatalf("a stale claim was not released: %+v", got) - } - - // Withdrawn, then the build registers it: applyPending finds nothing to claim. - if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) - if line := applyPending(ctx, open, p, "b-modules/sensors"); line != "" { - t.Fatalf("a withdrawn pending assignment was made: %s", line) - } - if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("a withdrawn pending assignment was assigned") - } - - // A stale claim whose module was assigned is applied. - q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"}) - if err != nil { - t.Fatal(err) - } - if ok, _ := inv.ClaimPending(ctx, q.ID); !ok { - t.Fatal("claim") - } - if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) - rows, _ := inv.PendingFor(ctx, "anchor", "sensors") - if len(rows) != 1 || rows[0].State != inventory.PendingApplied { - t.Fatalf("a stale claim of an assigned module: %+v", rows) - } -} - -func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - for _, r := range []inventory.BuildRequest{ - {ID: "build-lost", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/lost", Ref: "main", - For: "merge", At: time.Now().Add(-buildRequestBound / 2)}, - {ID: "build-heard", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/heard", Ref: "main", - For: "merge", At: time.Now()}, - } { - if err := inv.RecordBuildRequest(ctx, r); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"lost", "heard"} { - if _, err := assign(ctx, open, "laptop", m); err != nil { - t.Fatal(err) - } - } - register(t, open, catalogue.Manifest{Module: "heard", Version: "1"}) - - settlePending(ctx, open, time.Now().Add(buildRequestBound)) - if !slices.Contains(assignedTo(t, open, "laptop"), "heard") { - t.Fatal("a module registered meanwhile was not assigned at the next tick") - } - byModule := map[string]inventory.PendingAssignment{} - for _, p := range pendingOf(t, open) { - byModule[p.Module] = p - } - if p := byModule["lost"]; p.State != inventory.PendingExpired || !strings.Contains(p.Note, "no outcome of build build-lost") { - t.Fatalf("a build that said nothing: %+v", p) - } - if p := byModule["heard"]; p.State != inventory.PendingApplied { - t.Fatalf("a module registered meanwhile: %+v", p) - } - - // Ended waits are not deleted, for the chain removes no key (ADR 0408 section 4): after KeptFor they are no longer listed, open - // ones never are left out, nor one whose condition is still open (second review, bug B). - if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil { - t.Fatal(err) - } - later := time.Now().Add(inventory.KeptFor + time.Hour) - settlePending(ctx, open, later) - rows, err := inv.Pending(ctx, later.Add(-inventory.KeptFor)) - if err != nil { - t.Fatal(err) - } - for _, r := range rows { - if !r.Open() && (r.Raised == nil || r.Cleared != nil) { - t.Fatalf("an ended pending assignment with no open condition is listed past %s: %+v", inventory.KeptFor, r) - } - } - for _, r := range rows { - if r.Module == "heard" { - t.Fatalf("an applied wait is listed past %s: %+v", inventory.KeptFor, r) - } - } -} - -func TestARaisedRowIsKeptUntilItsConditionClears(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - failedBuild(t, open, "b-modules/sensors") - settlePending(ctx, open, time.Now()) - row := pendingOf(t, open)[0] - - settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) - if rowOf(t, open, row.ID).Raised == nil || !conditionOpen(t, row) { - t.Fatal("a raised row was pruned, or its condition closed, while the condition was open") - } - if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) - // No key of the chain is removed (ADR 0408 §4): a cleared wait stays in the value of its key, saying it was cleared. - if known, err := inv.PendingKnown(ctx, []string{row.ID}); err != nil || !known[row.ID] || rowOf(t, open, row.ID).Cleared == nil { - t.Fatalf("a cleared row is not kept, saying so: %v", err) - } - - // On anchor, then anchor removed: the row goes, and its condition with it at the next tick. - if _, err := assign(ctx, open, "anchor", "sensors"); err == nil { - t.Fatal("known and not built was not refused") - } - q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"}) - if err != nil { - t.Fatal(err) - } - if _, err := inv.SettlePending(ctx, q.ID, inventory.PendingWaiting, inventory.PendingExpired, "boom"); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - q.Node = "anchor" - if !conditionOpen(t, q) { - t.Fatal("not raised") - } - if _, err := inv.RemoveNodeForTest(ctx, "anchor"); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - if conditionOpen(t, q) { - t.Fatal("a removed machine's condition stayed open") - } -} - -func TestAStaleClaimIsSettledFromWhatTheMeshHolds(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - claimed := func(node string) inventory.PendingAssignment { - p, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: "sensors", Build: "b-x"}) - if err != nil { - t.Fatal(err) - } - if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok { - t.Fatalf("claim: %v %v", ok, err) - } - return p - } - notAssigned, assigned := claimed("laptop"), claimed("anchor") - register(t, open, catalogue.Manifest{Module: "unrelated", Version: "1"}) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "sensors", Version: "1"}, inventory.Source{}); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil { - t.Fatal(err) - } - - settlePending(ctx, open, time.Now()) - if rowOf(t, open, notAssigned.ID).State != inventory.PendingApplying || rowOf(t, open, assigned.ID).State != inventory.PendingApplying { - t.Fatal("a fresh claim was settled") - } - settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) - if got := rowOf(t, open, assigned.ID); got.State != inventory.PendingApplied { - t.Fatalf("a stale claim of an assigned module: %+v", got) - } - // Released to waiting, and in the same tick made, since sensors is registered now. - if got := rowOf(t, open, notAssigned.ID); got.State != inventory.PendingApplied || - !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatalf("a stale claim of a module not assigned: %+v", got) - } -} - -// A10 (ADR 0501 §6; issue 1851): an assignment of a module the indexer holds and the catalogue does not yet waits for -// the builder's done key of it, asking nothing, and is made when that key registers the module. -func TestAnAssignmentWaitsForTheKeyAndAsksNothing(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - noBuildAsked(t) - b := aBuilderOnTheBus(t) - fp := fpOf('a') - manifest := b.indexed(t, "sensors", "1", "novox/mesh-catalog", "modules/sensors", "3da80a4b00aa0000", fp) - b.built(t, "sensors", "1", fp, manifest, sdkdelivery.BuildBuilding) - said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") - if err != nil || !strings.Contains(said, "registered from the builder's done key") || !strings.Contains(said, "kept as pending") { - t.Fatalf("A10: the assignment of a module whose key is coming: %q %v", said, err) - } - if p := pendingOf(t, open); len(p) != 1 || p[0].Build != keyPendingPrefix+"sensors" { - t.Fatalf("A10: pending %+v", p) - } - settlePending(ctx, open, time.Now().Add(time.Hour)) - if p := pendingOf(t, open); p[0].State != inventory.PendingWaiting { - t.Fatalf("A10: a pending assignment waiting for a building key expired: %+v", p) - } - b.built(t, "sensors", "1", fp, manifest, sdkdelivery.BuildDone) - takeNewModules(ctx, open, "sensors") - if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatalf("A10: the key registered sensors and the pending assignment was not made: %+v", pendingOf(t, open)) - } -} diff --git a/cmd/mesh-controller/person_wait_integration_test.go b/cmd/mesh-controller/person_wait_integration_test.go deleted file mode 100644 index 72bb5dd6..00000000 --- a/cmd/mesh-controller/person_wait_integration_test.go +++ /dev/null @@ -1,165 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - k, _ := withConditionsInMemory(t) - at := h0 - say := func(rs ...link.ResourceHealth) { - t.Helper() - at = at.Add(time.Second) - for i := range rs { - rs[i].Since = at - } - if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, - Resources: rs}, at); err != nil { - t.Fatal(err) - } - } - asLink := func(r inventory.ResourceHealth) link.ResourceHealth { - return link.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target, State: r.State, - Reason: r.Reason, Account: r.Account} - } - account, unit := asLink(relogin("lights", "operator")), asLink(userUnit("lights", "operator")) - openNow := func() []conditions.Condition { - t.Helper() - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - return list - } - say(account, unit) - say(account, unit) - list := openNow() - if len(list) != 1 || list[0].Key != "module.lights.laptop.relogin-needed" { - t.Fatalf("raised %+v; want lights' relogin-needed alone", list) - } - c := list[0] - if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverOperator || - !strings.HasPrefix(c.Summary, "relogin needed on laptop:") { - t.Fatalf("the condition is %s %s %q", c.Severity, c.Resolver, c.Summary) - } - // After the new login the account is healthy, and the unit, still failing, is the module's own fault. - account.State, account.Reason = link.StateHealthy, "" - say(account, unit) - list = openNow() - if len(list) != 1 || list[0].Key != "module.lights.laptop.unhealthy" { - t.Fatalf("after the login: %+v; want lights' own unhealthy condition alone", list) - } - unit.State, unit.Reason = link.StateHealthy, "" - say(account, unit) - if list = openNow(); len(list) != 0 { - t.Fatalf("after the unit runs: %+v; want nothing open", list) - } -} - -func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { - return map[string]bool{"laptop": true}, nil - } - backlogFacts := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f, err := backlogFacts(ctx, open, component) - f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(), - Resources: []inventory.ResourceHealth{ - {Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy}, - {Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, - Reason: "down"}}}} - return f, err - } - gateEvery, gateBound = 0, 300*time.Millisecond - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - advancePlans(ctx, b.open) - if p := b.release(t); p.State != inventory.PlanRolling { - break - } - time.Sleep(20 * time.Millisecond) - } - p := b.release(t) - if p.State != inventory.PlanFailed { - t.Fatalf("the walk is %s: %s; want it failed on late", p.State, p.Note) - } - if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed || - !strings.Contains(v.Why, "on its own") { - t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err) - } - if failed, _ := inv.GateFailed(ctx, "build-late-c2"); !failed { - t.Fatal("late's build is not marked failed at its gate") - } - current, _ := inv.CurrentBuilds(ctx) - if current["app"].Commit != "c2" || current["late"].Commit != "c1" { - t.Fatalf("registered: app %s, late %s; want app kept at c2 and late put back to c1", current["app"].Commit, - current["late"].Commit) - } - if !strings.Contains(p.Note, "passed on their own and kept: app") { - t.Errorf("the walk does not say what kept its pass: %s", p.Note) - } -} - -func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - store := conditions.NewInMemory() - told := &conditions.Told{} - k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, Teller: told}) - t.Cleanup(func() { k.Close(context.Background()) }) - at := h0 - say := func(rs ...link.ResourceHealth) { - t.Helper() - at = at.Add(time.Second) - for i := range rs { - rs[i].Since = at - } - if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, - Resources: rs}, at); err != nil { - t.Fatal(err) - } - } - unit := link.ResourceHealth{Module: "lights", Resource: "lights.daemon", Kind: link.KindUnit, Target: "lights.service", - Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"} - account := link.ResourceHealth{Module: "lights", Resource: "lights.operator", Kind: link.KindAccount, Target: "operator", - Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group lights"} - say(unit) - say(unit) // lights not working - say(account, unit) - say(account, unit) // now it waits for a login - var resolved []string - for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline) && len(resolved) == 0; { - time.Sleep(20 * time.Millisecond) - for _, e := range told.Said() { - if e.Change == conditions.ChangeCleared { - resolved = append(resolved, e.Condition.Key+": "+e.Condition.Resolved) - // The clearing line is held to the plain rule too (ADR 0253). - w := conditions.Words{Headline: e.Condition.Headline, Explanation: e.Condition.Explanation, - Resolved: e.Condition.Resolved, Needs: e.Condition.Needs} - if why, ok := conditions.PlainWords(w, "laptop"); !ok { - t.Errorf("%s: its clearing words are not plain: %s", e.Condition.Key, why) - } - } - } - } - if len(resolved) != 1 || !strings.Contains(resolved[0], "module.lights.laptop.unhealthy: lights on laptop now waits only for a new login") { - t.Fatalf("cleared %v; want the not-working condition cleared as now waiting for a login", resolved) - } -} diff --git a/cmd/mesh-controller/person_wait_test.go b/cmd/mesh-controller/person_wait_test.go index eeb3a77d..7a3d3073 100644 --- a/cmd/mesh-controller/person_wait_test.go +++ b/cmd/mesh-controller/person_wait_test.go @@ -1,12 +1,6 @@ package main import ( - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -25,146 +19,3 @@ func userUnit(module, account string) inventory.ResourceHealth { Target: module + "-daemon.service", Account: account, State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"} } - -// Only what depends on the new login alone is a wait; anything else unhealthy of the module is judged as -// before, so that a fault is never excused. -func TestOnlyWhatDependsOnTheNewLoginIsAWait(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - notInGroup := relogin("lights", "operator") - notInGroup.Reason = "not in the group lights: the apply has not put operator there; its outcome says why" - systemUnit := userUnit("lights", "operator") - systemUnit.Account = "" - otherAccount := userUnit("lights", "guest") - container := inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container", Target: "lights", - State: link.StateUnhealthy, Reason: "down"} - otherModule := userUnit("sound", "operator") - starting := userUnit("lights", "operator") - starting.State = link.StateStarting - byTarget := relogin("lights", "operator") - byTarget.Account = "" // an account said by its target alone is still that account - - for _, c := range []struct { - name string - rs []inventory.ResourceHealth - waits bool - }{ - {"the account alone", []inventory.ResourceHealth{relogin("lights", "operator")}, true}, - {"the account and its unit in that account's manager", []inventory.ResourceHealth{relogin("lights", "operator"), - userUnit("lights", "operator")}, true}, - {"an account named by its target", []inventory.ResourceHealth{byTarget, userUnit("lights", "operator")}, true}, - {"another module's unit is not this module's", []inventory.ResourceHealth{relogin("lights", "operator"), otherModule}, true}, - {"a unit still starting is no fault", []inventory.ResourceHealth{relogin("lights", "operator"), starting}, true}, - {"a unit and no account", []inventory.ResourceHealth{userUnit("lights", "operator")}, false}, - {"an account not in its group", []inventory.ResourceHealth{notInGroup, userUnit("lights", "operator")}, false}, - {"a unit whose manager is not said", []inventory.ResourceHealth{relogin("lights", "operator"), systemUnit}, false}, - {"a unit in another account's manager", []inventory.ResourceHealth{relogin("lights", "operator"), otherAccount}, false}, - {"a container beside the wait", []inventory.ResourceHealth{relogin("lights", "operator"), container}, false}, - } { - said, waits := personWait("lights", "laptop", c.rs) - if waits != c.waits { - t.Errorf("%s: waits %v; want %v", c.name, waits, c.waits) - continue - } - if waits && !strings.HasPrefix(said, "relogin needed on laptop: lights waits for a new login of operator") { - t.Errorf("%s: said %q", c.name, said) - } - } -} - -// The gate reads a wait for a person as its own reading, never a fault, and passes with it; the same module -// with a container down beside it is not yet healthy, as before. -func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - now := time.Now() - since := now.Add(-time.Minute) - account := relogin("lights", "operator") - f := gateFacts{now: now, health: map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: now, - Resources: []inventory.ResourceHealth{account, userUnit("lights", "operator")}}}, - groupsAdded: map[string]bool{"lights": true}} - if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthPerson || !strings.Contains(why, "relogin needed on laptop") { - t.Fatalf("a wait for a new login reads %v %q; want a wait for a person", h, why) - } - // **Only a move that put the account in a new group is excused** (issue 318 review): a later build that - // adds no group did not bring the wait, and is judged as before. - f.groupsAdded["lights"] = false - if h, why := moduleHealthWord("lights", "laptop", since, f); h != healthNotYet { - t.Fatalf("a wait the move did not bring reads %v %q; want not yet", h, why) - } - f.groupsAdded["lights"] = true - h := f.health["laptop"] - h.Resources = append(h.Resources, inventory.ResourceHealth{Module: "lights", Resource: "lights.web", Kind: "container", - Target: "lights", State: link.StateUnhealthy, Reason: "down"}) - f.health["laptop"] = h - if got, why := moduleHealthWord("lights", "laptop", since, f); got != healthNotYet { - t.Fatalf("a container down beside the wait reads %v %q; want not yet", got, why) - } -} - -// The module's condition says the wait in one sentence for a person, and clears on the first statement that -// no longer says it — said as the module's own fault when its unit still fails after the new login. - -// A module that is healthy on its own keeps its pass when another module of the same send fails at the -// bound: it is neither put back nor left without a verdict for every other walk to wait on. - -// What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers -// no button, since nothing but their own new login can do it. -func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - o := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{ - relogin("openrazer", "operator"), userUnit("openrazer", "operator")}) - plainExample(t, o, "openrazer waits for a new login on g14", - "Needs you: log out of g14 completely and log in again, or restart it. Openrazer put your account in a "+ - "group it needs. You logged in before that, so openrazer cannot run until you log in again. Its "+ - "update is in place and nothing was undone.") - if len(o.Actions) != 0 || o.Needs == "" { - t.Errorf("relogin: needs %q, actions %+v; want needs and no button", o.Needs, o.Actions) - } - // **Not "your account" when it is not the operator's** (issue 318 review). - other := reloginObservation("openrazer", "g14", "relogin needed on g14: …", "operator", []inventory.ResourceHealth{ - relogin("openrazer", "guest"), userUnit("openrazer", "guest")}) - plainExample(t, other, "openrazer waits for a new login on g14", - "Needs you: have the account it names log out of g14 completely and log in again, or restart g14. Openrazer "+ - "put an account on g14 in a group it needs. That account logged in before that, so openrazer cannot run "+ - "until it logs in again. Its update is in place and nothing was undone.") - if unknown := reloginObservation("openrazer", "g14", "…", "", []inventory.ResourceHealth{relogin("openrazer", "operator")}); strings.Contains(unknown.Explanation, "your account") { - t.Errorf("an operator not known is still told it is their account: %q", unknown.Explanation) - } - // The kind's own wording, for a condition raised without words, names the module whole, dots and all, - // and never claims the account is the operator's. - w := plainWordings[kindReloginNeeded](conditions.Observation{Scope: conditions.ScopeModule, ID: "razer.lights.g14", Machine: "g14"}) - if w.Headline != "razer.lights waits for a new login on g14" || w.Needs == "" || len(w.Actions) != 0 || - strings.Contains(w.Explanation, "your account") { - t.Errorf("the kind's wording: %+v", w) - } -} - -// When a module not working turns into one waiting for a new login, and back, the clearing line says what it -// became, never that it works again (issue 318 review). - -// Which moves add an account group, read from the builds' manifests (issue 318 review). -func TestOnlyAMoveThatAddsAnAccountGroupCanBringAWait(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - user := func(groups ...any) catalogue.Manifest { - return catalogue.Manifest{Module: "lights", Resources: []map[string]any{{"id": "operator", "type": "user", - "name": "operator", "groups": groups}}} - } - none := catalogue.Manifest{Module: "lights"} - for _, c := range []struct { - name string - from catalogue.Manifest - had bool - to catalogue.Manifest - addsSome bool - }{ - {"a group added", none, true, user("lights"), true}, - {"the same group kept", user("lights"), true, user("lights"), false}, - {"a second group added", user("lights"), true, user("lights", "video"), true}, - {"a group taken away", user("lights", "video"), true, user("lights"), false}, - {"new to the machine, with a group", none, false, user("lights"), true}, - {"new to the machine, no group", none, false, none, false}, - } { - if got := addsAccountGroups(c.from, c.had, c.to); got != c.addsSome { - t.Errorf("%s: adds %v; want %v", c.name, got, c.addsSome) - } - } -} diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 184e5d69..3c7cf997 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -218,18 +218,6 @@ var plainWordings = map[string]func(conditions.Observation) words{ "One more module or secret may stop anything new from reaching %s.", m, m), Resolved: "Resolved: the instructions for " + m + " have room again"} }), - kindCoreDowngrade: worded(func(o conditions.Observation) words { - m := machineOr(o, "a machine") - component := idPart(o, 0) - if o.Scope != conditions.ScopeCore || !guarded(component) { - component = "core component" - } - return words{Headline: m + " keeps a newer " + component + " than the mesh holds", - Needs: fmt.Sprintf("push %s to take the older build, or a newer merge or rebuild moves the record forward.", m), - Explanation: fmt.Sprintf("The mesh's record of its own %s is older than the build %s was last sent, so "+ - "the mesh's sends keep the newer one there rather than put it back.", component, m), - Resolved: "Resolved: " + m + " runs the " + component + " the mesh holds"} - }), kindBindingKept: worded(func(o conditions.Observation) words { m := machineOr(o, "a machine") return words{Headline: "A module's data source is held on " + m, @@ -435,33 +423,6 @@ var plainWordings = map[string]func(conditions.Observation) words{ Explanation: "Its walk across the machines has not moved for longer than usual. Nothing is lost.", Resolved: "Resolved: the delivery moves again"} }), - kindBatchNotCut: worded(func(o conditions.Observation) words { - return words{Headline: "Merged changes are not being delivered", - Explanation: "Merges collected for one delivery should have been planned and were not. Nothing is lost.", - Resolved: "Resolved: the merged changes are being delivered"} - }), - kindBatchBehindWalk: worded(func(o conditions.Observation) words { - return words{Headline: "Merged changes wait behind a slow delivery", - Explanation: "Merges collected for the next delivery wait for the delivery before them, which is taking " + - "longer than it should. Nothing is lost.", - Resolved: "Resolved: the merged changes no longer wait"} - }), - kindWalkWaiting: worded(func(o conditions.Observation) words { - return words{Headline: "A delivery is waiting to start", - Explanation: "A merged change is built, and mesh-delivery (the module that decides when a delivery goes " + - "out) has not let it start yet.", - Resolved: "Resolved: the delivery is no longer waiting"} - }), - kindPlanFailed: worded(func(o conditions.Observation) words { - return words{Headline: "A delivery failed", - Explanation: "A delivery did not finish. Nothing is tried again until you say so.", - Resolved: "Resolved: the delivery is no longer failed"} - }), - kindMovesHeld: worded(func(o conditions.Observation) words { - return words{Headline: "An update waits for you", - Explanation: "A new version of a module would restart a container or touch its data, so it waits for you.", - Resolved: "Resolved: the update is no longer waiting"} - }), kindImageMissing: worded(func(o conditions.Observation) words { return words{Headline: "A module is not updated", Explanation: "The new version of a module names a container image that the mesh's store does not hold, so it " + @@ -474,30 +435,6 @@ var plainWordings = map[string]func(conditions.Observation) words{ Explanation: "A build was asked for and never answered. It is asked again with the next delivery.", Resolved: "Resolved: the build has its result"} }), - kindRolledBack: worded(func(o conditions.Observation) words { - m := machineOr(o, "a machine") - module := idPart(o, 0) - if o.Scope != conditions.ScopeBuild && o.Scope != conditions.ScopeCore { - module = "a module" - } - if o.Scope == conditions.ScopeCore { - return words{Headline: "A core update was put back on " + m, - Explanation: fmt.Sprintf("A new build of the mesh's own %s did not become healthy on %s, so the build "+ - "before was put back and runs.", module, m), - Resolved: "Resolved: the core on " + m + " is healthy"} - } - return words{Headline: conditions.Capital(module) + " update put back on " + m, - Explanation: fmt.Sprintf("The new build of %s did not become healthy on %s, so the mesh put back the one "+ - "before, which runs. The change is not delivered until it is fixed.", module, m), - Resolved: "Resolved: " + module + " on " + m + " is settled"} - }), - kindRollbackFailed: worded(func(o conditions.Observation) words { - m := machineOr(o, "a machine") - return words{Headline: "Putting back an update failed on " + m, - Needs: fmt.Sprintf("check what runs on %s; the details say which build.", m), - Explanation: fmt.Sprintf("A build that failed on %s could not be put back to the one before.", m), - Resolved: "Resolved: " + m + " runs a good build again"} - }), "release-held": worded(func(o conditions.Observation) words { return releaseHeldWords(nil, o.Also) }), @@ -630,26 +567,12 @@ var plainWordings = map[string]func(conditions.Observation) words{ Explanation: "The mesh's message system is being upgraded; some things pause until it is done.", Resolved: "The bus upgrade is done"} }), - kindBusStepWaiting: worded(func(o conditions.Observation) words { - return words{Headline: "Sends wait for a bus upgrade", - Needs: "start the bus upgrade " + FromMeshMCPServer, - Explanation: "A new version of the mesh's message system is built, and only a person installs it. Until " + - "then nothing else is sent to the machine that runs it.", - Resolved: "Resolved: the bus upgrade started, and sends go on"} - }), kindBusUpgradeFailed: worded(func(o conditions.Observation) words { return words{Headline: "The bus upgrade failed", Needs: "decide whether to put the bus back to the version before; the details say how.", Explanation: "The upgrade of the mesh's message system did not end healthy in time.", Resolved: "Resolved: the bus is healthy after its upgrade"} }), - kindBackupHeld: worded(func(o conditions.Observation) words { - return words{Headline: "A delivery waits for a backup that failed", - Needs: "fix what stopped the backup, then release the delivery.", - Explanation: "Before a delivery moves a module that keeps data, that data is backed up. A backup failed or " + - "did not answer in ten minutes, so nothing of the delivery is built or sent.", - Resolved: "Resolved: the backup is done and the delivery goes on"} - }), kindStuck: worded(func(o conditions.Observation) words { return words{Headline: "Something in a delivery has stopped moving", Explanation: "The watchman noticed a part of the delivery that did not move when it should have: a build " + @@ -657,12 +580,6 @@ var plainWordings = map[string]func(conditions.Observation) words{ "said again until it moves.", Resolved: "Resolved: it moves again"} }), - kindSendKeyRefused: worded(func(o conditions.Observation) words { - return words{Headline: "A delivery cannot watch a machine for its report", - Explanation: "The bus refused the note that makes the mesh notice a machine that never reports on a send. " + - "The delivery goes on, and the note is written again at its next look.", - Resolved: "Resolved: the delivery watches the machine for its report again"} - }), kindConsumerBehind: worded(func(o conditions.Observation) words { return words{Headline: "Messages pile up for a listener", Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " + @@ -939,186 +856,6 @@ func reloginNeeds(node string) string { return fmt.Sprintf("log out of %s completely and log in again, or restart it.", node) } -// stalledWords are the plain words of a delivery held past its bound, as mesh-delivery says it. -func stalledWords(l stalledLine, o conditions.Observation) (headline, explanation, resolved, needs string, - actions []conditions.Action) { - repository, _, _ := strings.Cut(l.ID, "@") - name := repoName(repository) - if l.State == "unannounced" { - // A pull request the forge never announced (novox/hq issue 347): no delivery exists, so there is nothing to - // stop, release or close; a new commit on its branch is announced and checked. - long := "for too long" - if d, err := time.ParseDuration(l.For); err == nil { - long = "for " + humanDuration(d) - } - if o.Resolver == conditions.ResolverOperator { - needs = "push a new commit to its branch; the forge announces it and the mesh checks it." - } - pull := "A pull request of " + name - if l.Number > 0 { - pull = fmt.Sprintf("Pull request %s #%d", name, l.Number) - } - return fmt.Sprintf("%s has had no merge check %s", pull, long), - fmt.Sprintf("%s has been open %s on a branch that requires the merge check, and the mesh was never asked "+ - "to check it: the forge never announced it. It cannot merge until it is checked.", pull, long), - fmt.Sprintf("%s has a merge check now, or is closed", pull), needs, nil - } - if l.State == "unanswered" { - return unansweredWords(l, o) - } - if l.Needs == needsNewHead { - return newHeadWords(l, name) - } - held := l.State - if held == "" { - held = "held" - } - long := "too long" - if d, err := time.ParseDuration(l.For); err == nil { - long = "for " + humanDuration(d) - } - if o.Resolver == conditions.ResolverOperator { - // Asked of the operator, approved on a channel that proves who answered (novox/hq ADR 0259); the - // router says where each can be answered, so the words do not. - release := conditions.Action{Label: "Release", Verb: "mesh-delivery.release", Level: conditions.LevelApprove, - Arguments: map[string]string{"id": l.ID, "why": ""}} - stop := conditions.Action{Label: "Stop", Verb: "mesh-delivery.stop", Level: conditions.LevelApprove, - Arguments: map[string]string{"id": l.ID, "why": ""}} - switch held { - case "held": - needs, actions = "release it, or stop it, "+FromMeshMCPServer, []conditions.Action{release, stop} - case "ready", "checked": - needs = "merge its pull request, or close it." - case "stopped": - // Already stopped: mesh-delivery.stop refuses it, so neither offer nor ask it (novox/hq issue 1581). - // What ends it is the stalled line's own word, such as a later merge that delivers its modules. - needs = stoppedNeeds(l) - default: - needs, actions = "stop it "+FromMeshMCPServer, []conditions.Action{stop} - } - } - return fmt.Sprintf("Delivery of %s %s %s", name, held, long), - fmt.Sprintf("A delivery of %s has been %s %s, past its limit.", name, held, long), - fmt.Sprintf("Delivery of %s is no longer %s", name, held), needs, actions -} - -// newHeadWords are the words of a delivery whose cure is its author's new head (novox/hq ADR 0505 §5): it names the -// author and says a new head is needed; it offers no Stop and never calls it the operator's, because the operator -// hears of it only once the author's own watch has failed, after two hours with no new head. -func newHeadWords(l stalledLine, name string) (headline, explanation, resolved, needs string, actions []conditions.Action) { - pull := "A pull request of " + name - if l.Number > 0 { - pull = fmt.Sprintf("Pull request %d of %s", l.Number, name) - } - author := l.Author - if _, plain := conditions.Plain(author); !plain { - author = "its author" // a branch named in code (a dotted name, a number) is not said - } - long := "for a long time" - if d, err := time.ParseDuration(l.For); err == nil { - long = "for " + humanDuration(d) - } - return pull + " needs a new head from its author", - fmt.Sprintf("%s cannot merge until %s pushes a new head: its check failed after its Approve, and no new head "+ - "came %s.", pull, author, long), - pull + " has a new head, or is closed", - "ask " + author + " for a new head.", nil -} - -// stoppedNeeds is what ends a delivery that is already stopped, from the stalled line's healer words ("none: a -// merge that delivers its modules ends it — the operator's"): the part that says what ends it. -func stoppedNeeds(l stalledLine) string { - ends := strings.TrimSpace(strings.TrimPrefix(l.H2, "none:")) - ends, _, _ = strings.Cut(ends, " — ") - if ends == "" { - ends = "a later merge that delivers its modules ends it" - } - return "nothing to stop, it is stopped already: " + ends + "." -} - -// unansweredWords are the plain words of a pull request's head whose merge check was asked and has not answered -// within its bound (novox/hq issue 438): mesh-delivery holds no delivery of it, so there is nothing to stop, release -// or close, and no action is offered. The operator's acts are a new commit, which asks the check again, or a look -// at the build queue, where a check that never ran may still wait. -// -// A head whose delivery is held in ready or rejected is said the same way (novox/hq issue 441), and mesh-delivery -// names that state as data (Held). Its check answered once, so the build queue is the wrong place to look: the step -// is a recheck of the delivery, named by its id, which asks the check again (novox/hq issue 451). It is advised, not -// offered as an action, since the controller is not granted the recheck. -func unansweredWords(l stalledLine, o conditions.Observation) (headline, explanation, resolved, needs string, - actions []conditions.Action) { - repository, _, _ := strings.Cut(l.ID, "@") - pull := "A pull request of " + repoName(repository) - if l.Number > 0 { - pull = fmt.Sprintf("Pull request %s #%d", repoName(repository), l.Number) - } - long, limit := "for too long", "" - if d, err := time.ParseDuration(l.For); err == nil { - long = "for " + humanDuration(d) - } - if d, err := time.ParseDuration(l.Bound); err == nil { - limit = ", past its limit of " + humanDuration(d) - } - check := "merge check" - if said := uncheckedWaitWords(l); len(said) > 0 { - check = "merge check (" + strings.Join(said, ", ") + ")" - } - which := "Its " + check - switch l.Held { - case "ready", "rejected": - which = "Its delivery is " + l.Held + ", but on the forge its " + check - if o.Resolver == conditions.ResolverOperator { - needs = "recheck its delivery, which asks its check again — mesh-delivery.recheck with id " + l.ID + - " and why — or push a new commit to its branch, " + FromMeshMCPServer - } - default: - if o.Resolver == conditions.ResolverOperator { - needs = "push a new commit to its branch, which asks the check again, or see whether its check still " + - "waits in the build queue: mesh-controller.queue." - } - } - return fmt.Sprintf("%s's merge check has not answered %s", pull, long), - fmt.Sprintf("%s is open on a branch that requires the merge check. %s was asked and has not answered %s%s. "+ - "It cannot merge until its check answers.", pull, which, long, limit), - fmt.Sprintf("%s has an answer from its merge check, or is closed", pull), needs, nil -} - -// uncheckedWaitWords are the merge checks an unanswered line waits on, as the operator reads them: from the checks -// given as data, each time in the controller's local zone, as every other time in a message — "mesh/check -// pending since 02:11" — never the UTC time inside mesh-delivery's finished words, which cannot be said again in -// another zone (novox/hq issue 443). A line from a mesh-delivery that gives no such data is said by its words. -func uncheckedWaitWords(l stalledLine) []string { - if len(l.Checks) == 0 { - return l.Waiting - } - out := make([]string, 0, len(l.Checks)) - for _, c := range l.Checks { - switch c.State { - case "never-set": - out = append(out, c.Context+" never set") - case "pending": - out = append(out, c.Context+" pending"+sinceWords(c.Since)) - default: - out = append(out, c.Context+" not answered") - } - } - return out -} - -// sinceWords is " since 02:11" in the controller's local zone, with its day when that is not today ("since 23:05 on -// 9 Oct"), so the time stays absolute; a time not given, or not readable, is said so and never made up. -func sinceWords(since string) string { - at, err := time.Parse(time.RFC3339, since) - if err != nil { - return ", since a time the forge did not say" - } - local, today := at.In(wordsZone()), wordsNow().In(wordsZone()) - if local.YearDay() == today.YearDay() && local.Year() == today.Year() { - return " since " + local.Format("15:04") - } - return " since " + local.Format("15:04 on 2 Jan") -} - // causeWords is a hand-act's cause as a person says it. func causeWords(cause string) string { return strings.NewReplacer(".", " ", "_", " ").Replace(cause) diff --git a/cmd/mesh-controller/plain_words_test.go b/cmd/mesh-controller/plain_words_test.go index 653eaf1c..a4092a29 100644 --- a/cmd/mesh-controller/plain_words_test.go +++ b/cmd/mesh-controller/plain_words_test.go @@ -1,7 +1,6 @@ package main import ( - "encoding/json" "regexp" "strings" "testing" @@ -45,51 +44,6 @@ func plainExample(t *testing.T, o conditions.Observation, headline, explanation said, labels, o.Resolved) } -// **A delivery waiting**: the popup read "the walk of novox/mesh-catalog a6385479 has waited 56m0s for -// mesh-delivery's word to start: `mesh-delivery.show` for the delivery that landed as a6385479 says why; -// `plans go plan-1791454185265004861 --why …` starts it by hand". Under four hours it needs nothing — no -// popup; past them, the operator starts or stops it from the notification. -func TestADeliveryWaitingNeedsNothingUntilItsBoundThenOffersStartAndStop(t *testing.T) { - now := time.Date(2026, 10, 8, 11, 36, 0, 0, time.UTC) - f := calm(now) - f.waits = []waitFacts{{id: "plan-1791454185265004861", repository: "novox/mesh-catalog", commit: "a6385479c0ffee", - awaits: "mesh-delivery", since: now.Add(-56 * time.Minute), modules: []string{"openrazer"}}} - got := watchWaits(f) - if len(got) != 1 { - t.Fatalf("%+v", got) - } - plainExample(t, got[0], "openrazer delivery waiting to start", - "Nothing for you to do. The change to openrazer is merged and built, and has waited 56 minutes for "+ - "mesh-delivery (the module that decides when a delivery goes out) to let it start. It becomes a question "+ - "for you if it still waits after 4 hours.") - if !strings.Contains(got[0].Summary, "plans go plan-1791454185265004861") { - t.Errorf("the summary lost the way on for whoever looks closer: %q", got[0].Summary) - } - - // Past four hours it is urgent, and asks the operator to start or stop it (novox/hq ADR 0259): the plan's - // own verbs, approved, which the controller performs on the warrant. The router says where to answer. - f.waits[0].since = now.Add(-5 * time.Hour) - got = watchWaits(f) - plainExample(t, got[0], "openrazer delivery waiting to start", - "Needs you: start it, or stop it, from the mesh MCP server; this notification cannot do it. The change to openrazer is merged and built, and mesh-delivery (the "+ - "module that decides when a delivery goes out) has not let it start for 5 hours, so mesh-delivery may "+ - "be stuck.", "Start", "Stop") - for i, want := range []string{"go", "stop"} { - a := got[0].Actions[i] - if a.Verb != "mesh-controller.plans" || a.Arguments[want] != "plan-1791454185265004861" || - a.Level != conditions.LevelApprove || a.Arguments["cause"] != conditions.CauseOperatorAnswer { - t.Errorf("%s: %+v", a.Label, a) - } - } - - // Many modules are counted, not listed in the headline. - f.waits[0].modules = []string{"a", "b", "c", "d"} - got = watchWaits(f) - if got[0].Headline != "mesh-catalog delivery waiting to start" || !strings.Contains(got[0].Explanation, "4 modules of mesh-catalog") { - t.Errorf("%+v", got[0]) - } -} - // **A module unhealthy**: "openrazer on g14 is not healthy: its unit openrazer-daemon.service failed in the // account's own service manager (exit-code)". Restarting is not an acknowledgement: it is asked of the // operator at the approve level (novox/hq ADR 0259), so a desk click never performs it (ADR 0258). @@ -160,21 +114,6 @@ func TestAHealerWantedNeedsNothingFromTheOperator(t *testing.T) { "wanted for it. Nothing is broken now.") } -// **A delivery held past its bound**, as mesh-delivery says it: "the delivery novox/hq@055550802096 has been -// held for 36h2m6s, past its bound of 24h0m0s (it waits for the operator): healer H2 may none: …". -func TestADeliveryHeldAsksForReleaseOrStopInWords(t *testing.T) { - got := stalledObservations([]stalledLine{{ID: "novox/hq@055550802096", State: "held", For: "36h2m6s", - Bound: "24h0m0s", H2: "none: the state is the operator's", Says: "it waits for the operator"}}) - plainExample(t, got[0], "Delivery of hq held for 36 hours", - "Needs you: release it, or stop it, from the mesh MCP server; this notification cannot do it. A delivery of hq has been held for 36 hours, past its limit.", - "Release", "Stop") - for i, verb := range []string{"mesh-delivery.release", "mesh-delivery.stop"} { - if a := got[0].Actions[i]; a.Verb != verb || a.Arguments["id"] != "novox/hq@055550802096" || a.Level != conditions.LevelApprove { - t.Errorf("%+v", a) - } - } -} - // **Every kind the controller raises has plain words**, and its words are plain for a subject of every // shape it is raised with. func TestEveryWordingIsPlain(t *testing.T) { @@ -242,27 +181,6 @@ func TestDataLossOffersNoSilence(t *testing.T) { } } -// **Updates held after a failed release** (2026-10-08): the popup read "Needs you: release them, or leave -// them held." — naming neither what waits nor where it is released. It names the modules and machines, and -// the mesh MCP server. -func TestUpdatesHeldNameWhatWaitsAndWhereItIsReleased(t *testing.T) { - saved := backlogNow - t.Cleanup(func() { backlogNow = saved }) - backlogNow.held = "release-1791457717307061152 failed (failed its gate on g14); what waits is released again by a person" - backlogNow.waiting = map[string][]inventory.CarriedMove{ - "shanks": {{Module: "openrazer"}}, - "g14": {{Module: "openrazer"}, {Module: "sensors"}}, - } - got := backlogObservation() - if len(got) != 1 { - t.Fatalf("raised %d", len(got)) - } - plainExample(t, got[0], "Updates of openrazer and sensors wait for your release", - "Needs you: release them from the mesh MCP server; this notification cannot do it. Updates of openrazer and "+ - "sensors on g14 and shanks wait for a person to release them, because the last walk failed. "+ - "They are not delivered until then, and stay held if you leave them.", "Release") -} - // **What held them is said in the glossary's words** (2026-10-08 review): the backlog is held after any // walk failed, not a release, and a "check" is a pull request's status. So the words say the walk failed, // and never that a release failed or a check did — with the modules and machines named or not. @@ -286,236 +204,3 @@ func TestUpdatesHeldSayTheWalkFailed(t *testing.T) { // mcpVerb is a need that opens with a verb only the mesh MCP server performs (ADR 0258 §1): release, stop, // start, restart, and a restore. var mcpVerb = regexp.MustCompile(`^(release|stop|start|restart|restore)\b`) - -// **A need no notification can answer says where it is answered** (ADR 0258 §1): every wording whose need -// opens with a verb the mesh MCP server performs, and offers no action, ends with FromMeshMCPServer — the -// kinds worded here for every subject shape, and those worded where they are raised. A new kind that misses -// it fails here; release-held did (2026-10-08). -func TestANeedNoNotificationAnswersNamesTheMeshMCPServer(t *testing.T) { - check := func(what string, w words) { - t.Helper() - if w.Needs == "" || len(w.Actions) > 0 || !mcpVerb.MatchString(w.Needs) { - return - } - if !strings.HasSuffix(w.Needs, FromMeshMCPServer) { - t.Errorf("%s needs %q without %q", what, w.Needs, FromMeshMCPServer) - } - } - subjects := []conditions.Observation{ - {Scope: conditions.ScopeMachine, ID: "ace", Machine: "ace"}, - {Scope: conditions.ScopeMachine, ID: "ace.immich.library", Machine: "ace"}, - {Scope: conditions.ScopeModule, ID: "openrazer.g14", Machine: "g14"}, - {Scope: conditions.ScopeDelivery, ID: "novox/hq@055550802096"}, - {Scope: conditions.ScopeCore, ID: "controller.anchor", Machine: "anchor"}, - {Scope: conditions.ScopeMesh, ID: "release", Also: []string{"g14"}}, - } - for kind, fn := range plainWordings { - for _, s := range subjects { - for _, sev := range []conditions.Severity{conditions.Warning, conditions.Urgent} { - s.Kind, s.Severity, s.Resolver = kind, sev, conditions.ResolverOperator - check(kind+" about "+s.ID, fn(s)) - } - } - } - check("a walk waiting", words{Needs: waitingNeeds(conditions.Urgent)}) - check("a module's failed service", words{Needs: moduleNeeds("g14", - []inventory.ResourceHealth{{Kind: link.KindUnit, Target: "openrazer-daemon.service"}})}) - for _, state := range []string{"held", "ready", "failing"} { - _, _, _, needs, actions := stalledWords(stalledLine{ID: "novox/hq@055550802096", State: state, For: "36h"}, - conditions.Observation{Resolver: conditions.ResolverOperator}) - check("a delivery "+state, words{Needs: needs, Actions: actions}) - } - check("updates held", releaseHeldWords([]string{"openrazer"}, []string{"g14"})) -} - -// **A pull request the forge never announced says what to do about it** (novox/hq issue 347): mesh-delivery says one -// as a stalled line in state `unannounced` — no delivery exists, so there is nothing to stop, release or close — -// and the operator's one act is a new commit on its branch, which the forge announces. -func TestAnUnannouncedPullRequestSaysToPushANewCommit(t *testing.T) { - l := stalledLine{ID: "novox/hq@bfe82315f42c", Number: 243, State: "unannounced", For: "11m0s", Bound: "10m0s", - H2: "none: the forge never announced it, so there is no delivery to close — the operator's", - Says: "novox/hq#243 is open on main, which requires the merge check, and its head has had no merge check"} - obs := stalledObservations([]stalledLine{l}) - if len(obs) != 1 || obs[0].Resolver != conditions.ResolverOperator { - t.Fatalf("an unannounced pull request is not the operator's: %+v", obs) - } - o := obs[0] - if strings.Contains(o.Needs, "stop") || strings.Contains(o.Needs, "release") || !strings.Contains(o.Needs, "commit") { - t.Fatalf("it says to %q", o.Needs) - } - if !strings.Contains(o.Headline, "no merge check") || !strings.Contains(o.Headline, "#243") || - !strings.Contains(o.Explanation, "never") { - t.Fatalf("it reads %q / %q", o.Headline, o.Explanation) - } -} - -// **A pull request whose merge check never answered says which check, since when, and what to do** (novox/hq issue -// 438): mesh-delivery says one as a stalled line in state `unanswered`, the line below exactly as its test makes it. -// No delivery of the head is held, so there is nothing to stop, release or close: no action is offered, and the -// operator's acts are a new commit, which asks the check again, or a look at the build queue. -func TestAnUnansweredMergeCheckSaysWhichCheckAndWhatToDo(t *testing.T) { - var l stalledLine - if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+ - `"waiting":["mesh/merge-gate pending since 2026-10-11T00:11:39Z","mesh/repo-check never set"],"for":"1h1m0s",`+ - `"bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's",`+ - `"says":"novox/mesh-controller#212 is open on main, which requires the merge check, and its head's check `+ - `started and never answered: mesh/merge-gate pending since 2026-10-11T00:11:39Z, mesh/repo-check never set. `+ - `No delivery of it is held here, so nothing asks it again. A new commit on its branch announces it and asks `+ - `its check"}`), &l); err != nil { - t.Fatal(err) - } - obs := stalledObservations([]stalledLine{l}) - if len(obs) != 1 || obs[0].Resolver != conditions.ResolverOperator { - t.Fatalf("an unanswered merge check is not the operator's: %+v", obs) - } - o := obs[0] - t.Logf("headline: %s\nexplanation: %s\nneeds: %s\nresolved: %s", o.Headline, o.Explanation, o.Needs, o.Resolved) - if len(o.Actions) != 0 { - t.Fatalf("it offers an action a head with no delivery cannot take: %+v", o.Actions) - } - if strings.Contains(o.Needs, "stop") || strings.Contains(o.Needs, "release") || !strings.Contains(o.Needs, "commit") || - !strings.Contains(o.Needs, "in the build queue: mesh-controller.queue") || strings.Contains(o.Needs, "build seat") { - t.Fatalf("it says to %q", o.Needs) - } - if o.Headline != "Pull request mesh-controller #212's merge check has not answered for 61 minutes" { - t.Fatalf("its headline reads %q", o.Headline) - } - for _, want := range []string{"mesh/merge-gate pending since 2026-10-11T00:11:39Z", "mesh/repo-check never set", - "past its limit of 60 minutes"} { - if !strings.Contains(o.Explanation, want) { - t.Fatalf("its explanation does not say %q: %q", want, o.Explanation) - } - } - if strings.Contains(o.Explanation, "never asked") || strings.Contains(o.Explanation, "never announced") { - t.Fatalf("it says the mesh was never asked, of a head whose check started: %q", o.Explanation) - } - - // A line from a mesh-delivery that names no checks still says the check did not answer, and offers nothing. - var bare stalledLine - if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+ - `"for":"1h1m0s","bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's"}`), - &bare); err != nil { - t.Fatal(err) - } - o = stalledObservations([]stalledLine{bare})[0] - if len(o.Actions) != 0 || !strings.Contains(o.Explanation, "has not answered") || strings.Contains(o.Headline, "Delivery of") { - t.Fatalf("a line naming no checks reads %q / %q, actions %+v", o.Headline, o.Explanation, o.Actions) - } -} - -// **An unanswered merge check's time reaches the operator in their own time, never as raw UTC** (novox/hq issue -// 443): mesh-delivery says each check waited on as data — its context, its state and since when, in RFC 3339 — beside -// the finished words it gave before, and the controller words it in its local zone, as every other time in a message. -// The line is the one mesh-delivery says, as in the report of issue 443, with the checks it now carries. -func TestAnUnansweredMergeChecksTimeIsSaidInLocalTime(t *testing.T) { - // The operator's zone given through the seam, never by writing time.Local: other tests' goroutines read it, and - // the build seat runs the suite under the race detector. - wasZone, wasNow := wordsZone, wordsNow - wordsZone = func() *time.Location { return time.FixedZone("CEST", 2*60*60) } - wordsNow = func() time.Time { return time.Date(2026, 10, 11, 1, 12, 39, 0, time.UTC) } - t.Cleanup(func() { wordsZone, wordsNow = wasZone, wasNow }) - - var l stalledLine - if err := json.Unmarshal([]byte(`{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",`+ - `"waiting":["mesh/merge-gate pending since 2026-10-11T00:11:39Z","mesh/repo-check never set"],`+ - `"checks":[{"context":"mesh/merge-gate","state":"pending","since":"2026-10-11T00:11:39Z"},`+ - `{"context":"mesh/repo-check","state":"never-set"}],"for":"1h1m0s",`+ - `"bound":"1h0m0s","h2":"none: no delivery of it is held here, so there is none to close — the operator's",`+ - `"says":"novox/mesh-controller#212 is open on main, which requires the merge check, and its head's check `+ - `started and never answered: mesh/merge-gate pending since 2026-10-11T00:11:39Z, mesh/repo-check never set. `+ - `No delivery of it is held here, so nothing asks it again. A new commit on its branch announces it and asks `+ - `its check"}`), &l); err != nil { - t.Fatal(err) - } - o := stalledObservations([]stalledLine{l})[0] - t.Logf("explanation: %s", o.Explanation) - for _, raw := range []string{"2026-10-11T00:11:39Z", "00:11", "UTC"} { - if strings.Contains(o.Explanation, raw) || strings.Contains(o.Headline, raw) { - t.Fatalf("the operator reads %q: %q / %q", raw, o.Headline, o.Explanation) - } - } - for _, want := range []string{"mesh/merge-gate pending since 02:11", "mesh/repo-check never set"} { - if !strings.Contains(o.Explanation, want) { - t.Fatalf("its explanation does not say %q: %q", want, o.Explanation) - } - } - - // A check pending since another day than today says which day, so the time stays absolute. - l.Checks[0].Since = "2026-10-09T21:05:00Z" - if o = stalledObservations([]stalledLine{l})[0]; !strings.Contains(o.Explanation, "mesh/merge-gate pending since 23:05 on 9 Oct") { - t.Fatalf("a check pending since an earlier day reads %q", o.Explanation) - } - - // A pending check whose time the forge did not say is said so, with no time made up. - l.Checks[0].Since = "" - if o = stalledObservations([]stalledLine{l})[0]; !strings.Contains(o.Explanation, "mesh/merge-gate pending, since a time the forge did not say") { - t.Fatalf("a pending check without its time reads %q", o.Explanation) - } -} - -// **An unanswered merge check on a ready or rejected delivery is told to recheck it** (novox/hq issue 451): -// mesh-delivery says such a line too (issue 441) and names the delivery's state as data, `held`. That check already -// answered once, so the build queue is the wrong place to look; a recheck of the delivery, named by its id, asks the -// check again. A line with no delivery held keeps the advice of issue 438, and so does a line from a mesh-delivery -// that does not say `held` yet: the controller never guesses the state from the words. -func TestAnUnansweredMergeCheckOnAReadyOrRejectedDeliveryOffersARecheck(t *testing.T) { - line := func(held string) stalledLine { - var l stalledLine - raw := `{"id":"novox/mesh-controller@c11222026a3b","number":212,"state":"unanswered",` + - `"waiting":["mesh/merge-gate pending since 2026-10-11T00:11:39Z"],"for":"1h1m0s","bound":"1h0m0s",` + - `"h2":"none: its delivery is ` + held + `, a state the operator moves — the operator's",` + - `"says":"novox/mesh-controller#212 is open on main, which requires the merge check, and its delivery is ` + - held + ` here, but on the forge its check has not answered: mesh/merge-gate pending since ` + - `2026-10-11T00:11:39Z. The pull request waits on that status and nothing asks it again. A recheck of the ` + - `delivery asks its check again, as does a new commit on its branch"` - if held != "" { - raw += `,"held":"` + held + `"` - } - if err := json.Unmarshal([]byte(raw+`}`), &l); err != nil { - t.Fatal(err) - } - return l - } - for _, held := range []string{"ready", "rejected"} { - o := stalledObservations([]stalledLine{line(held)})[0] - t.Logf("%s: %s\nneeds: %s", held, o.Explanation, o.Needs) - if !strings.Contains(o.Needs, "mesh-delivery.recheck") || !strings.Contains(o.Needs, "novox/mesh-controller@c11222026a3b") { - t.Fatalf("a %s delivery's unanswered check is not offered its recheck: %q", held, o.Needs) - } - if strings.Contains(o.Needs, "build queue") || strings.Contains(o.Needs, "mesh-controller.queue") { - t.Fatalf("a %s delivery's unanswered check, which answered once, is sent to the build queue: %q", held, o.Needs) - } - if !strings.Contains(o.Explanation, "Its delivery is "+held) { - t.Fatalf("a %s delivery's explanation does not say its state: %q", held, o.Explanation) - } - if len(o.Actions) != 0 { - t.Fatalf("it offers an action the controller is not granted: %+v", o.Actions) - } - } - // No delivery held, or a mesh-delivery that does not say: the advice of issue 438, and no recheck. - o := stalledObservations([]stalledLine{line("")})[0] - if !strings.Contains(o.Needs, "in the build queue: mesh-controller.queue") || strings.Contains(o.Needs, "recheck") { - t.Fatalf("a line with no delivery held says to %q", o.Needs) - } -} - -// **A delivery already stopped is not asked to be stopped** (novox/hq issue 1581): mesh-delivery.stop refuses a -// stopped delivery, so the condition offers no Stop and says what ends it, in the stalled line's own words. Any -// other held state keeps its Stop. -func TestAStoppedDeliveryIsNotAskedToBeStopped(t *testing.T) { - l := stalledLine{ID: "novox/mesh-catalog@2f6086120c46", State: "stopped", For: "12h19m36s", - H2: "none: a merge that delivers its modules ends it — the operator's", - Says: "a merged change never reached the machines"} - _, _, _, needs, actions := stalledWords(l, conditions.Observation{Resolver: conditions.ResolverOperator}) - if len(actions) != 0 { - t.Fatalf("a stopped delivery offers %v, want no action", actions) - } - if strings.Contains(needs, "stop it") || !strings.Contains(needs, "a merge that delivers its modules ends it") { - t.Fatalf("needs = %q, want what ends it and no ask to stop", needs) - } - l.State = "failing" - _, _, _, needs, actions = stalledWords(l, conditions.Observation{Resolver: conditions.ResolverOperator}) - if len(actions) != 1 || actions[0].Label != "Stop" || !strings.HasPrefix(needs, "stop it") { - t.Fatalf("another held state changed: needs %q, actions %v", needs, actions) - } -} diff --git a/cmd/mesh-controller/plan_announce_copy_test.go b/cmd/mesh-controller/plan_announce_copy_test.go deleted file mode 100644 index 66c0a3d0..00000000 --- a/cmd/mesh-controller/plan_announce_copy_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "reflect" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// heldPublish is a bus that holds a plan-moved publish until it is let go, and keeps the body. -type heldPublish struct { - link.Bus - let chan struct{} - body chan []byte -} - -func (h *heldPublish) PublishSeatEvent(_ context.Context, _, _ string, body []byte) error { - <-h.let - h.body <- body - return nil -} - -func aWalkWithSteps() inventory.Plan { - built := time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) - return inventory.Plan{ID: "plan-1", Repository: "novox/one", Commit: "abc", State: "walking", Tier: 1, - Created: built, Updated: built, Revision: 4, - Tiers: [][]string{{"alpha"}, {"beta", "gamma"}}, - Modules: map[string]*inventory.PlanModule{ - "alpha": {State: "built", BuiltAt: &built, Commit: "abc", First: []string{"ace"}}, - "beta": {State: "asked", Why: "waiting"}, - "gamma": {}, - }, - Times: &inventory.PlanTimes{Started: &built}, WalkSaid: inventory.WalkStarted} -} - -func TestAnAnnouncementIsThePlanAsItWasWhenSaidWhatTheWalkDoesNext(t *testing.T) { // 1260/AC1 - p := aWalkWithSteps() - bus := &heldPublish{let: make(chan struct{}), body: make(chan []byte, 1)} - sayPlanMoved(t.Context(), bus, nil, p) - - // The walk goes on, writing its steps while the announcement is being made. - for i := 0; i < 2000; i++ { - step := p.Modules["beta"] - step.State, step.Why = "built", "changed after the call" - step.First = append(step.First[:0], "novox") - p.Modules["delta"] = &inventory.PlanModule{State: "asked"} - p.Tiers[1][0] = "changed" - p.Times.Started = nil - delete(p.Modules, "delta") - } - close(bus.let) - - var said inventory.Plan - select { - case body := <-bus.body: - if err := json.Unmarshal(body, &said); err != nil { - t.Fatal(err) - } - case <-time.After(10 * time.Second): - t.Fatal("nothing was announced") - } - if said.Modules["beta"].State != "asked" || said.Modules["beta"].Why != "waiting" || said.Tiers[1][0] != "beta" || - len(said.Modules) != 3 || said.Times == nil || said.Times.Started == nil { - t.Errorf("the announcement carries the walk as it went on, not as it was: %+v", said) - } -} - -func TestAnAnnouncementOfAnUnchangedPlanIsTheBodyItWasBeforeTheCopy(t *testing.T) { // 1260/AC2 - p := aWalkWithSteps() - bus := &heldPublish{let: make(chan struct{}), body: make(chan []byte, 1)} - close(bus.let) - sayPlanMoved(t.Context(), bus, nil, p) - var got []byte - select { - case got = <-bus.body: - case <-time.After(10 * time.Second): - t.Fatal("nothing was announced") - } - // What it was: the plan whole, its phases worked out, and what the walk says of itself. - q := p - q.Phases = q.WalkPhases(time.Now(), nil) - want, err := json.Marshal(planMovedBody{Plan: q, walkSaid: sayWalk(t.Context(), nil, q)}) - if err != nil { - t.Fatal(err) - } - var a, b map[string]any - if json.Unmarshal(got, &a) != nil || json.Unmarshal(want, &b) != nil { - t.Fatal("a body is not JSON") - } - delete(a, "phases") // worked out from the time of the call - delete(b, "phases") - if !reflect.DeepEqual(a, b) { - t.Errorf("the body changed:\n got %s\nwant %s", got, want) - } - if a["walk"] != "plan-1" { - t.Errorf("the walk's own words are not in the body: %s", got) - } -} diff --git a/cmd/mesh-controller/plan_backups.go b/cmd/mesh-controller/plan_backups.go deleted file mode 100644 index 024b2fdc..00000000 --- a/cmd/mesh-controller/plan_backups.go +++ /dev/null @@ -1,453 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "slices" - "sort" - "strings" - "sync" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// **A backup before the send** (novox/hq ADR 0404 decisions 2 and 3, design 47, issue 1065). Once a delivery plan -// has started and before it asks its first tier, the controller asks the node-backup holder of every machine that -// runs a moved module with an item protected by a backup to back that module up now, and asks nothing of a builder -// and sends nothing until every one of them answered done. The plan keeps the record (inventory.PlanBackups). - -// backupsWithin is how long a backup is given from its ask to say it is done, and backupsPollEvery how soon the plan -// is looked at again while it waits: variables so a test waits for neither. -var ( - backupsWithin = 10 * time.Minute - backupsPollEvery = 10 * time.Second -) - -// night is one module's last night as a holder's `backed-up` says it (lastNight). -type night struct { - OK bool `json:"ok"` - At time.Time `json:"at"` - Error string `json:"error,omitempty"` -} - -// askBackupNow asks a machine's backup holder to back a module up now; the holder answers when it has started. A -// variable so a test needs no bus. An error is the holder's own words when it refused, or why it could not be asked. -var askBackupNow = func(ctx context.Context, machine, module string) error { - return onTheBus(func(conn *nats.Conn) error { - answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "now", machine, - map[string]any{"module": module}, 30*time.Second) - if err != nil { - return &backupNotAsked{fmt.Errorf("%s's backup holder was not asked: %w", machine, err)} - } - if answer.Error != "" { - return fmt.Errorf("%s", answer.Error) - } - return nil - }) -} - -// backupNotAsked is an ask that could not be made, as opposed to one the holder refused: the step is tried again. -type backupNotAsked struct{ error } - -func (e *backupNotAsked) Unwrap() error { return e.error } - -// readLastNights reads a machine's holder's `backed-up` by module: how each module's last backup went. A variable -// so a test needs no bus. -var readLastNights = func(ctx context.Context, machine string) (map[string]*night, error) { - var out map[string]*night - err := onTheBus(func(conn *nats.Conn) error { - answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "backed-up", machine, map[string]any{}, 10*time.Second) - if err != nil { - return err - } - if answer.Error != "" { - return fmt.Errorf("%s", answer.Error) - } - var read []struct { - Module string `json:"module"` - LastNight *night `json:"lastNight"` - } - if err := json.Unmarshal(answer.Result, &read); err != nil { - return fmt.Errorf("its answer is not readable: %w", err) - } - out = map[string]*night{} - for _, m := range read { - out[m.Module] = m.LastNight - } - return nil - }) - return out, err -} - -// backupFacts are what the step needs of the mesh: the modules with an item protected by a backup, where each -// runs, and which machines hold node-backup. -type backupFacts struct { - backedUp map[string][]string // module → machines running it - holder map[string]bool -} - -// backupFactsFrom reads the facts from the catalogue's entries and the seat holdings. A module is backed up when -// one of its own items is (catalogue.DataItem.BackedUp: never a cache, `backup: none` or redundancy alone). A -// module whose policy records (ADR 0377) is decided per machine after the build, so every machine running it is -// asked: a superset costs time and never leaves a send without its restore point. -func backupFactsFrom(entries []inventory.Entry, held []catalogue.Held) backupFacts { - f := backupFacts{backedUp: map[string][]string{}, holder: map[string]bool{}} - for _, e := range entries { - for _, it := range e.Manifest.DataItems() { - if it.BackedUp() { - f.backedUp[e.Manifest.Module] = slices.Clone(e.On) - break - } - } - } - for _, h := range held { - if s, known := catalogue.SeatNamed(h.Claim); known && s.Name == catalogue.BackupSeat { - f.holder[h.Node] = true - } - } - return f -} - -// loadBackupFacts reads the facts from the inventory. The holders are read as `seats` reads them, from the world -// every machine's composition makes (theRestOfTheMesh): the seat_holding table keeps one holder of a node seat, and -// node-backup is held on every machine. -func loadBackupFacts(ctx context.Context, inv *inventory.Inventory) (backupFacts, error) { - return loadBackupFactsFrom(inv.Catalogued, func() ([]catalogue.Held, error) { - shelf, err := inv.Catalogue(ctx) - if err != nil { - return nil, err - } - world, err := restOfTheMeshOf(ctx, inv, shelf, "") - if err != nil { - return nil, err - } - return world.Held, nil - }, ctx) -} - -// loadBackupFactsFrom is loadBackupFacts given where the entries and the holdings come from. A source that fails -// fails the step, to be tried again: a failure never reads as a machine with no holder, which would send without a -// backup. -func loadBackupFactsFrom(entries func(context.Context) ([]inventory.Entry, error), - held func() ([]catalogue.Held, error), ctx context.Context) (backupFacts, error) { - es, err := entries(ctx) - if err != nil { - return backupFacts{}, fmt.Errorf("the backups of the plan could not be worked out: %w", err) - } - h, err := held() - if err != nil { - return backupFacts{}, fmt.Errorf("the backups of the plan could not be worked out: who holds %s: %w", - catalogue.BackupSeat, err) - } - return backupFactsFrom(es, h), nil -} - -// stepBackups takes one step of a plan's backups and says whether the plan may go on to ask its first tier. It -// asks what is not asked, reads how the asked ones went, and records it all in p.Backups; it waits (false, nil) -// while any is pending, and stays stopped (false, nil) once one failed, which part B holds and says. A plan whose -// moved modules have no backed-up item keeps no record and goes on at once. A holder that cannot be asked is tried again -// within the bound and then failed; an error is left for what the step itself cannot do. -func stepBackups(ctx context.Context, p *inventory.Plan, facts backupFacts, now time.Time) (bool, error) { - proceed, err := stepBackupsOnce(ctx, p, facts, now) - if err == nil { - noteBackups(p) - } - return proceed, err -} - -func stepBackupsOnce(ctx context.Context, p *inventory.Plan, facts backupFacts, now time.Time) (bool, error) { - if p.Backups == nil { - b := wantedBackups(*p, facts) - if b == nil { - return true, nil - } - p.Backups = b - } - b := p.Backups - switch b.State { - case inventory.BackupDone: - return true, nil - case inventory.BackupFailed: - return false, nil - } - nights := map[string]map[string]*night{} - for i := range b.Asked { - a := &b.Asked[i] - if a.State != inventory.BackupPending { - continue - } - if a.AskedAt == nil { - asked := now - if a.TriedAt == nil { - a.TriedAt = &asked - } - if err := askBackupNow(ctx, a.Machine, a.Module); err != nil { - var unasked *backupNotAsked - if errors.As(err, &unasked) { - // A holder off the bus is tried again within the bound, by the look and never by the - // "tried again" error path; past it the backup is failed and the plan held. - a.Words = "could not be asked: " + err.Error() - if now.Sub(*a.TriedAt) >= backupsWithin { - a.State = inventory.BackupFailed - } - continue - } - a.State, a.Words = inventory.BackupFailed, err.Error() - a.AskedAt, a.TriedAt = &asked, nil - continue - } - a.AskedAt, a.TriedAt, a.Words = &asked, nil, "" - continue - } - if _, read := nights[a.Machine]; !read { - n, err := readLastNights(ctx, a.Machine) - if err != nil { - n = nil // not read now: the bound below still ends the wait - } - nights[a.Machine] = n - } - if n := nights[a.Machine][a.Module]; n != nil && n.At.After(*a.AskedAt) { - if n.OK { - a.State, a.Words = inventory.BackupDone, "" - } else { - a.State, a.Words = inventory.BackupFailed, n.Error - if a.Words == "" { - a.Words = "the backup failed and did not say why" - } - } - continue - } - if now.Sub(*a.AskedAt) >= backupsWithin { - a.State = inventory.BackupFailed - a.Words = "did not answer in " + wordsOfBound(backupsWithin) - } - } - b.State = backupsStateOf(b.Asked) - switch b.State { - case inventory.BackupDone: - return true, nil - case inventory.BackupPending: - lookAgainForBackups(p.ID) - } - return false, nil -} - -// wantedBackups is the record of backups a plan has to take, or nil for none: one entry per machine with a holder -// running a moved module that has a backed-up item, and the machines with none. -func wantedBackups(p inventory.Plan, facts backupFacts) *inventory.PlanBackups { - var asked []inventory.PlanBackup - noHolder := map[string]bool{} - for _, tier := range p.Tiers { - for _, module := range tier { - for _, machine := range facts.backedUp[module] { - if !facts.holder[machine] { - noHolder[machine] = true - continue - } - asked = append(asked, inventory.PlanBackup{Machine: machine, Module: module, State: inventory.BackupPending}) - } - } - } - if len(asked) == 0 && len(noHolder) == 0 { - return nil - } - sort.SliceStable(asked, func(i, j int) bool { - if asked[i].Machine != asked[j].Machine { - return asked[i].Machine < asked[j].Machine - } - return asked[i].Module < asked[j].Module - }) - b := &inventory.PlanBackups{Asked: asked} - for m := range noHolder { - b.NoHolder = append(b.NoHolder, m) - } - sort.Strings(b.NoHolder) - b.State = backupsStateOf(asked) - return b -} - -// backupsStateOf is the whole of the backups' states: failed if any failed, done if all are (or none are asked). -func backupsStateOf(asked []inventory.PlanBackup) string { - state := inventory.BackupDone - for _, a := range asked { - switch a.State { - case inventory.BackupFailed: - return inventory.BackupFailed - case inventory.BackupPending: - state = inventory.BackupPending - } - } - return state -} - -func wordsOfBound(d time.Duration) string { - if d == 10*time.Minute { - return "ten minutes" - } - if d%time.Minute == 0 { - return fmt.Sprintf("%d minutes", int(d/time.Minute)) - } - return d.String() -} - -var ( - backupLookMu sync.Mutex - backupLooks = map[string]bool{} -) - -// lookAgainForBackups has the serving controller look at the plans again soon, once per waiting plan: a backup ending -// is no event the controller hears, so the wait is looked at by a timer, never by blocking the hold on the plans. -func lookAgainForBackups(plan string) { - backupLookMu.Lock() - if backupLooks[plan] { - backupLookMu.Unlock() - return - } - backupLooks[plan] = true - backupLookMu.Unlock() - time.AfterFunc(backupsPollEvery, func() { - backupLookMu.Lock() - delete(backupLooks, plan) - backupLookMu.Unlock() - askToLook() - }) -} - -// backupsFirst is the step advanceOnce takes before it asks a walk's first tier: whether the plan may go on. -func backupsFirst(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) (bool, error) { - facts := backupFacts{} - if p.Backups == nil { - f, err := loadBackupFacts(ctx, inv) - if err != nil { - return false, err - } - facts = f - } - return stepBackups(ctx, p, facts, time.Now().UTC()) -} - -// backupsNote begins the note of a plan that waits on or is held by its backups. -const backupsNote = "backups: " - -// noteBackups says in the plan's note what its backups wait on, or, held, which backup failed and in what words and -// the verb that asks again; a note of its own that the backups ended is cleared. Not the "tried again" note of a -// failed step: a held plan is not retried (novox/hq ADR 0404 decision 4). -func noteBackups(p *inventory.Plan) { - if p.Backups == nil { - return - } - var pending, failed []string - for _, a := range p.Backups.Asked { - switch a.State { - case inventory.BackupPending: - pending = append(pending, a.Module+" on "+a.Machine) - case inventory.BackupFailed: - failed = append(failed, fmt.Sprintf("%s on %s: %s", a.Module, a.Machine, a.Words)) - } - } - switch p.Backups.State { - case inventory.BackupPending: - p.Note = backupsNote + "waiting for the backup of " + strings.Join(pending, ", ") - case inventory.BackupFailed: - p.Note = backupsNote + "held — the backup of " + strings.Join(failed, "; ") + "; nothing is asked or sent. " + - "`delivery-backups-again` asks it again" - default: - if strings.HasPrefix(p.Note, backupsNote) { - p.Note = "" - } - } -} - -// waitingOnBackups says a plan has not asked its first tier because its backups are pending or failed: no tier -// late, as S3 and the lines of `plans` read it. -func waitingOnBackups(p inventory.Plan) bool { - return p.Open() && p.Release == nil && p.Tier == 0 && !askedAny(p) && p.Backups != nil && - p.Backups.State != inventory.BackupDone -} - -// askBackupsAgain clears the failed backups of a held plan, so the next step asks those again and no other. It -// says which. Refused for a plan with no failed backup. -func askBackupsAgain(p *inventory.Plan, by, why string) ([]inventory.PlanBackup, error) { - if !p.Open() || p.Backups == nil || p.Backups.State != inventory.BackupFailed { - return nil, fmt.Errorf("%s has no failed backup: there is nothing to ask again", p.ID) - } - var again []inventory.PlanBackup - for i := range p.Backups.Asked { - a := &p.Backups.Asked[i] - if a.State != inventory.BackupFailed { - continue - } - again = append(again, *a) - a.State, a.AskedAt, a.TriedAt, a.Words = inventory.BackupPending, nil, nil, "" - } - p.Backups.State = backupsStateOf(p.Backups.Asked) - p.Note = backupsNote + "asked again by " + by + ": " + why - return again, nil -} - -// backupsAgain is askBackupsAgain for the plan with this id, kept: the verb `delivery-backups-again`. -func backupsAgain(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, []inventory.PlanBackup, error) { - release, err := inv.HoldPlans(ctx, true) - if err != nil { - return inventory.Plan{}, nil, err - } - defer release() - p, err := inv.PlanByID(ctx, id) - if err != nil { - return inventory.Plan{}, nil, err - } - again, err := askBackupsAgain(&p, by, why) - if err != nil { - return p, nil, err - } - if err := inv.SavePlan(ctx, &p); err != nil { - return p, nil, err - } - return p, again, nil -} - -// kindBackupHeld is a delivery plan held for a backup that failed or stayed silent; backupHeldProbe its row. -const ( - kindBackupHeld = "backup-held" - backupHeldProbe = "DK" -) - -// observeBackupHolds is one condition per plan held for its backups, with the backups' own words as its evidence. -func observeBackupHolds(plans []inventory.Plan) []conditions.Observation { - var out []conditions.Observation - for _, p := range plans { - if !p.Open() || p.Backups == nil || p.Backups.State != inventory.BackupFailed { - continue - } - var said, machines []string - for _, a := range p.Backups.Asked { - if a.State == inventory.BackupFailed { - said = append(said, fmt.Sprintf("%s on %s: %s", a.Module, a.Machine, a.Words)) - machines = append(machines, a.Machine) - } - } - out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.ID, Token: "backup-held", - Kind: kindBackupHeld, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, - Machine: machines[0], Also: machines[1:], - Summary: fmt.Sprintf("the delivery %s is held: %d of its backups failed or did not answer, so nothing is "+ - "asked or sent", p.ID, len(said)), - Said: strings.Join(said, "; ")}) - } - return out -} - -// probeBackupHeld reads the open plans for the ones held for their backups. -func probeBackupHeld(ctx context.Context, d *doctor) ([]conditions.Observation, error) { - plans, err := d.open.inventory.OpenPlans(ctx) - if err != nil { - return nil, err - } - return observeBackupHolds(plans), nil -} diff --git a/cmd/mesh-controller/plan_backups_hold_test.go b/cmd/mesh-controller/plan_backups_hold_test.go deleted file mode 100644 index 6294e204..00000000 --- a/cmd/mesh-controller/plan_backups_hold_test.go +++ /dev/null @@ -1,154 +0,0 @@ -package main - -import ( - "context" - "fmt" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A backup that failed or stayed silent holds the plan, is said, and a verb asks it again (novox/hq ADR 0404 -// decision 4, issue 1066). - -// aHeldPlan is a plan whose backup on ace failed and whose backup on g14 is done. -func aHeldPlan(t *testing.T) (*inventory.Plan, backupFacts) { - t.Helper() - entries := []inventory.Entry{aModuleWith(t, "mail", []string{"ace", "g14"}, anItem("a", ""))} - facts := backupFactsFrom(entries, holders("ace", "g14")) - seat := &aFakeBackupSeat{nights: map[string]*night{}} - seat.install(t) - p := aPlanOf("mail") - stepBackups(t.Context(), p, facts, backupT0) - seat.nights["ace/mail"] = &night{OK: false, At: backupT0.Add(time.Minute), Error: "no space left on device"} - seat.nights["g14/mail"] = &night{OK: true, At: backupT0.Add(time.Minute)} - if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(2*time.Minute)); proceed || err != nil { - t.Fatalf("proceed=%v err=%v", proceed, err) - } - return p, facts -} - -// The held plan says which machine, which module and the backup's own words in its note, and is not called tried again. -func TestAHeldPlanSaysTheMachineTheModuleAndTheWordsInItsNote(t *testing.T) { - p, _ := aHeldPlan(t) - for _, want := range []string{"ace", "mail", "no space left on device", "delivery-backups-again"} { - if !strings.Contains(p.Note, want) { - t.Fatalf("note %q lacks %q", p.Note, want) - } - } - if strings.Contains(p.Note, "tried again") || p.State != inventory.PlanBuilding { - t.Fatalf("a held plan is not failed and not retried: %q %s", p.Note, p.State) - } - if !waitingOnBackups(*p) { - t.Fatal("a plan held by its backups is no tier late (S3 leaves it out)") - } -} - -// delivery-backups-again clears the failed records, and the next step asks those backups again and only those. -func TestBackupsAgainClearsTheFailedRecordsAndTheNextStepAsksThemAgain(t *testing.T) { - p, facts := aHeldPlan(t) - again, err := askBackupsAgain(p, "mesh-delivery", "a person released it") - if err != nil || len(again) != 1 || again[0].Machine != "ace" || again[0].Module != "mail" { - t.Fatalf("again=%+v err=%v", again, err) - } - if p.Backups.State != inventory.BackupPending { - t.Fatalf("state %q", p.Backups.State) - } - seat := &aFakeBackupSeat{nights: map[string]*night{}} - seat.install(t) - now := backupT0.Add(10 * time.Minute) - if proceed, err := stepBackups(t.Context(), p, facts, now); proceed || err != nil { - t.Fatalf("proceed=%v err=%v", proceed, err) - } - if strings.Join(seat.asked, " ") != "ace/mail" { - t.Fatalf("asked %v; only the failed backup is asked again", seat.asked) - } - seat.nights["ace/mail"] = &night{OK: true, At: now.Add(time.Minute)} - if proceed, _ := stepBackups(t.Context(), p, facts, now.Add(2*time.Minute)); !proceed { - t.Fatal("the plan must go on once the backup it asked again is done") - } - if strings.Contains(p.Note, "delivery-backups-again") { - t.Fatalf("the hold's note stays after it ended: %q", p.Note) - } -} - -// Refused for a plan with no failed backup. -func TestBackupsAgainIsRefusedForAPlanWithNoFailedBackup(t *testing.T) { - if _, err := askBackupsAgain(aPlanOf("mail"), "x", "y"); err == nil { - t.Fatal("a plan with no backups was asked again") - } - p := aPlanOf("mail") - p.Backups = &inventory.PlanBackups{State: inventory.BackupPending, - Asked: []inventory.PlanBackup{{Machine: "ace", Module: "mail", State: inventory.BackupPending}}} - if _, err := askBackupsAgain(p, "x", "y"); err == nil { - t.Fatal("a plan whose backups are pending was asked again") - } -} - -// The verb's argument vector: the plan and a reason are required. -func TestTheVerbDeliveryBackupsAgainTakesAPlanAndAWhy(t *testing.T) { - got, err := argvFor("delivery-backups-again", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen"}) - want := []string{"delivery", "backups-again", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen"} - if err != nil || strings.Join(got, "|") != strings.Join(want, "|") { - t.Fatalf("got %v %v, want %v", got, err, want) - } - if _, err := argvFor("delivery-backups-again", map[string]any{"plan": "plan-1"}); err == nil { - t.Fatal("no reason was accepted") - } -} - -// One condition per held plan, the backup one, with the words in its evidence; none for a plan only waiting. -func TestAHeldPlanRaisesOneConditionWithTheBackupsWords(t *testing.T) { - held, _ := aHeldPlan(t) - waiting := aPlanOf("mail") - waiting.ID = "plan-w" - waiting.Backups = &inventory.PlanBackups{State: inventory.BackupPending, - Asked: []inventory.PlanBackup{{Machine: "ace", Module: "mail", State: inventory.BackupPending}}} - got := observeBackupHolds([]inventory.Plan{*held, *waiting}) - if len(got) != 1 { - t.Fatalf("observations %+v", got) - } - o := got[0] - if o.Kind != kindBackupHeld || o.Scope != conditions.ScopePlan || o.ID != held.ID || - !strings.Contains(o.Said, "no space left on device") || !strings.Contains(o.Said, "mail") || !strings.Contains(o.Said, "ace") { - t.Fatalf("%+v", o) - } -} - -// A holder off the bus cannot be asked: the plan waits within the bound, retried by the look and not by the error -// path, and at the bound the backup is failed with the reason, so the hold and the condition take over. -func TestABackupHolderThatCannotBeAskedIsFailedAtTheBoundWithTheReason(t *testing.T) { - seat := &aFakeBackupSeat{} - seat.install(t) - was := askBackupNow - t.Cleanup(func() { askBackupNow = was }) - calls := 0 - askBackupNow = func(_ context.Context, machine, module string) error { - calls++ - return &backupNotAsked{fmt.Errorf("%s's backup holder was not asked: no responders", machine)} - } - entries := []inventory.Entry{aModuleWith(t, "mail", []string{"ace"}, anItem("a", ""))} - facts := backupFactsFrom(entries, holders("ace")) - p := aPlanOf("mail") - if proceed, err := stepBackups(t.Context(), p, facts, backupT0); proceed || err != nil || p.Backups.State != inventory.BackupPending { - t.Fatalf("proceed=%v err=%v %+v", proceed, err, p.Backups) - } - if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(backupsWithin/2)); proceed || err != nil || - p.Backups.State != inventory.BackupPending || calls != 2 || p.Backups.Asked[0].AskedAt != nil { - t.Fatalf("within the bound: proceed=%v err=%v calls=%d %+v", proceed, err, calls, p.Backups) - } - if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(backupsWithin)); proceed || err != nil { - t.Fatalf("proceed=%v err=%v", proceed, err) - } - a := p.Backups.Asked[0] - if p.Backups.State != inventory.BackupFailed || a.State != inventory.BackupFailed || - !strings.Contains(a.Words, "could not be asked: ") || !strings.Contains(a.Words, "no responders") { - t.Fatalf("%+v", p.Backups) - } - if strings.Contains(p.Note, "tried again") || !strings.Contains(p.Note, "held") { - t.Fatalf("note %q", p.Note) - } -} diff --git a/cmd/mesh-controller/plan_backups_test.go b/cmd/mesh-controller/plan_backups_test.go deleted file mode 100644 index 644f1950..00000000 --- a/cmd/mesh-controller/plan_backups_test.go +++ /dev/null @@ -1,276 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// The backups before a delivery plan asks anything (novox/hq ADR 0404 decisions 2 and 3, issue 1065). - -// aModuleWith is a catalogue entry for a module keeping the given data items, on the given machines. -func aModuleWith(t *testing.T, name string, on []string, items ...string) inventory.Entry { - t.Helper() - raw := `{"module":"` + name + `","version":"1"` - if len(items) > 0 { - raw += `,"data":{"own":[` + strings.Join(items, ",") + `]}` - } - raw += `}` - var m catalogue.Manifest - if err := json.Unmarshal([]byte(raw), &m); err != nil { - t.Fatal(err) - } - return inventory.Entry{Manifest: m, On: on} -} - -func anItem(id, extra string) string { - if extra != "" { - extra = "," + extra - } - return `{"id":"` + id + `","path":"/x/` + id + `","class":"valuable","why":"w"` + extra + `}` -} - -func holders(nodes ...string) []catalogue.Held { - var out []catalogue.Held - for _, n := range nodes { - out = append(out, catalogue.Held{Claim: catalogue.BackupSeat, Node: n, Module: "restic"}) - } - return out -} - -// aFakeBackupSeat stands in for the machines' node-backup holders. -type aFakeBackupSeat struct { - asked []string // "machine/module", in order - nights map[string]*night - refuse map[string]string -} - -func (f *aFakeBackupSeat) install(t *testing.T) { - t.Helper() - was, wasRead := askBackupNow, readLastNights - t.Cleanup(func() { askBackupNow, readLastNights = was, wasRead }) - askBackupNow = func(_ context.Context, machine, module string) error { - f.asked = append(f.asked, machine+"/"+module) - if why := f.refuse[machine+"/"+module]; why != "" { - return fmt.Errorf("%s", why) - } - return nil - } - readLastNights = func(_ context.Context, machine string) (map[string]*night, error) { - out := map[string]*night{} - for k, n := range f.nights { - if m, mod, _ := strings.Cut(k, "/"); m == machine { - out[mod] = n - } - } - return out, nil - } -} - -func aPlanOf(modules ...string) *inventory.Plan { - p := &inventory.Plan{ID: "plan-b", State: inventory.PlanBuilding, Tiers: [][]string{modules}, - Modules: map[string]*inventory.PlanModule{}} - for _, m := range modules { - p.Modules[m] = &inventory.PlanModule{} - } - return p -} - -var backupT0 = time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC) - -// Only a module with an item protected by a backup is asked, and on the machines that run it. -func TestOnlyModulesWithAnItemProtectedByABackupAreAsked(t *testing.T) { - seat := &aFakeBackupSeat{} - seat.install(t) - entries := []inventory.Entry{ - aModuleWith(t, "copied", []string{"ace", "g14"}, anItem("a", "")), - aModuleWith(t, "dumped", []string{"ace"}, anItem("b", `"backup":{"dump":"pg_dump","into":"d"}`), anItem("d", "")), - aModuleWith(t, "mirrored", []string{"ace"}, anItem("c", `"redundancy":"a mirror"`)), - aModuleWith(t, "unbacked", []string{"ace"}, anItem("e", `"backup":"none"`)), - aModuleWith(t, "cached", []string{"ace"}, `{"id":"f","path":"/x/f","class":"cache"}`), - aModuleWith(t, "dataless", []string{"ace"}), - } - p := aPlanOf("copied", "dumped", "mirrored", "unbacked", "cached", "dataless") - facts := backupFactsFrom(entries, holders("ace", "g14")) - proceed, err := stepBackups(t.Context(), p, facts, backupT0) - if err != nil || proceed { - t.Fatalf("proceed=%v err=%v; the plan must wait for the backups it asked", proceed, err) - } - want := "ace/copied ace/dumped g14/copied" - if got := strings.Join(seat.asked, " "); got != want { - t.Fatalf("asked %q, want %q", got, want) - } - if p.Backups == nil || p.Backups.State != inventory.BackupPending || len(p.Backups.Asked) != 3 { - t.Fatalf("the plan's record: %+v", p.Backups) - } -} - -// A machine with no backup holder is asked nothing, the plan says so, and it does not hold the plan. -func TestANodeWithNoBackupHolderAsksNothingAndThePlanSaysSo(t *testing.T) { - seat := &aFakeBackupSeat{} - seat.install(t) - entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace", "bare"}, anItem("a", ""))} - p := aPlanOf("copied") - proceed, err := stepBackups(t.Context(), p, backupFactsFrom(entries, holders("ace")), backupT0) - if err != nil || proceed { - t.Fatalf("proceed=%v err=%v", proceed, err) - } - if strings.Join(seat.asked, " ") != "ace/copied" { - t.Fatalf("asked %v", seat.asked) - } - if p.Backups == nil || len(p.Backups.NoHolder) != 1 || p.Backups.NoHolder[0] != "bare" { - t.Fatalf("the plan must say bare has no holder: %+v", p.Backups) - } - // Only the no-holder machine: nothing to wait for, the plan goes on, and still says it. - only := []inventory.Entry{aModuleWith(t, "copied", []string{"bare"}, anItem("a", ""))} - q := aPlanOf("copied") - proceed, err = stepBackups(t.Context(), q, backupFactsFrom(only, holders("ace")), backupT0) - if err != nil || !proceed || q.Backups == nil || q.Backups.NoHolder[0] != "bare" || q.Backups.State != inventory.BackupDone { - t.Fatalf("proceed=%v err=%v record=%+v", proceed, err, q.Backups) - } -} - -// Nothing of the plan goes on until every backup answered done; a backup is not asked twice. -func TestNothingIsAskedOfTheBuildOrSentUntilEveryBackupAnsweredDone(t *testing.T) { - seat := &aFakeBackupSeat{nights: map[string]*night{}} - seat.install(t) - entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace", "g14"}, anItem("a", ""))} - facts := backupFactsFrom(entries, holders("ace", "g14")) - p := aPlanOf("copied") - if proceed, _ := stepBackups(t.Context(), p, facts, backupT0); proceed { - t.Fatal("went on before any backup answered") - } - // ace is done, g14 still running (its last night is from before the ask). - seat.nights["ace/copied"] = &night{OK: true, At: backupT0.Add(time.Minute)} - seat.nights["g14/copied"] = &night{OK: true, At: backupT0.Add(-time.Hour)} - if proceed, _ := stepBackups(t.Context(), p, facts, backupT0.Add(2*time.Minute)); proceed { - t.Fatal("went on while g14's backup had not answered") - } - seat.nights["g14/copied"] = &night{OK: true, At: backupT0.Add(3 * time.Minute)} - proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(4*time.Minute)) - if err != nil || !proceed || p.Backups.State != inventory.BackupDone { - t.Fatalf("proceed=%v err=%v record=%+v", proceed, err, p.Backups) - } - if len(seat.asked) != 2 { - t.Fatalf("asked %v; each backup is asked once", seat.asked) - } -} - -// A failed backup, and a silent one past the bound, are recorded with the backup's own words and the plan stops. -func TestAFailedOrSilentBackupIsRecordedAndThePlanStopsAsking(t *testing.T) { - seat := &aFakeBackupSeat{nights: map[string]*night{}} - seat.install(t) - entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace", "g14"}, anItem("a", ""))} - facts := backupFactsFrom(entries, holders("ace", "g14")) - p := aPlanOf("copied") - stepBackups(t.Context(), p, facts, backupT0) - seat.nights["ace/copied"] = &night{OK: false, At: backupT0.Add(time.Minute), Error: "no space left on device"} - if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(backupsWithin+time.Minute)); proceed || err != nil { - t.Fatalf("proceed=%v err=%v", proceed, err) - } - got := map[string]inventory.PlanBackup{} - for _, b := range p.Backups.Asked { - got[b.Machine] = b - } - if got["ace"].State != inventory.BackupFailed || got["ace"].Words != "no space left on device" { - t.Fatalf("ace: %+v", got["ace"]) - } - if got["g14"].State != inventory.BackupFailed || !strings.Contains(got["g14"].Words, "did not answer in ten minutes") { - t.Fatalf("g14: %+v", got["g14"]) - } - if p.Backups.State != inventory.BackupFailed { - t.Fatalf("state %q", p.Backups.State) - } - // It stays stopped, asks nothing more, and says no retry in the plan's note. - n := len(seat.asked) - if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(time.Hour)); proceed || err != nil || len(seat.asked) != n { - t.Fatalf("a failed backup must stay stopped: proceed=%v err=%v asked=%v", proceed, err, seat.asked) - } -} - -// A holder that refuses the ask says why, in the record. -func TestAHolderThatRefusesTheAskIsRecordedFailedWithItsWords(t *testing.T) { - seat := &aFakeBackupSeat{refuse: map[string]string{"ace/copied": "copied declares nothing to back up"}} - seat.install(t) - entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace"}, anItem("a", ""))} - p := aPlanOf("copied") - if proceed, err := stepBackups(t.Context(), p, backupFactsFrom(entries, holders("ace")), backupT0); proceed || err != nil { - t.Fatalf("proceed=%v err=%v", proceed, err) - } - if b := p.Backups.Asked[0]; b.State != inventory.BackupFailed || b.Words != "copied declares nothing to back up" { - t.Fatalf("%+v", b) - } -} - -// A plan moving no backed-up item asks nothing and keeps no record (design 47: costs nothing). -func TestAPlanThatMovesNoBackedUpItemAsksNothing(t *testing.T) { - seat := &aFakeBackupSeat{} - seat.install(t) - entries := []inventory.Entry{ - aModuleWith(t, "dataless", []string{"ace"}), - aModuleWith(t, "mirrored", []string{"ace"}, anItem("c", `"redundancy":"a mirror"`)), - } - p := aPlanOf("dataless", "mirrored") - proceed, err := stepBackups(t.Context(), p, backupFactsFrom(entries, holders("ace")), backupT0) - if err != nil || !proceed || p.Backups != nil || len(seat.asked) != 0 { - t.Fatalf("proceed=%v err=%v record=%+v asked=%v", proceed, err, p.Backups, seat.asked) - } -} - -// The holders of a node seat are every machine's, and a source that fails is no machine without a holder. -func TestTheBackupHoldersAreEveryMachinesAndAFailedReadIsNotNoHolder(t *testing.T) { - entries := func(context.Context) ([]inventory.Entry, error) { - return []inventory.Entry{aModuleWith(t, "mail", []string{"ace", "g14"}, anItem("a", ""))}, nil - } - f, err := loadBackupFactsFrom(entries, func() ([]catalogue.Held, error) { return holders("ace", "g14"), nil }, t.Context()) - if err != nil || !f.holder["ace"] || !f.holder["g14"] { - t.Fatalf("%+v %v", f, err) - } - if _, err := loadBackupFactsFrom(entries, func() ([]catalogue.Held, error) { return nil, fmt.Errorf("store down") }, - t.Context()); err == nil || !strings.Contains(err.Error(), "store down") { - t.Fatalf("a failed read of the holders must fail the step: %v", err) - } -} - -// Issue 1086: the cut of a plan asked its first tier itself, so gitea was sent to novox with no backup asked. -func giteaOnNovox(t *testing.T) inventory.Entry { - return aModuleWith(t, "gitea", []string{"novox"}, - `{"id":"data","path":"/var/lib/gitea/data","class":"valuable","why":"repositories"}`) -} - -func TestACutPlanAsksTheBackupOfGiteaOnNovoxBeforeItAsksItsFirstTier(t *testing.T) { - seat := &aFakeBackupSeat{} - seat.install(t) - facts := backupFactsFrom([]inventory.Entry{giteaOnNovox(t)}, holders("novox", "ace")) - p := aPlanOf("gitea", "mesh-delivery") - tierAsked := false - err := askFirstTierAtCut(p, - func(p *inventory.Plan) (bool, error) { return stepBackups(t.Context(), p, facts, backupT0) }, - func() error { tierAsked = true; return nil }) - if err != nil || tierAsked { - t.Fatalf("the first tier was asked before the backup: asked=%v err=%v", tierAsked, err) - } - if strings.Join(seat.asked, " ") != "novox/gitea" || p.Backups == nil || p.Backups.State != inventory.BackupPending { - t.Fatalf("asked %v, record %+v", seat.asked, p.Backups) - } - if !strings.HasPrefix(p.Note, backupsNote) { - t.Fatalf("note %q", p.Note) - } -} - -func TestACutPlanWhoseBackupsCouldNotBeWorkedOutAsksNothingAndSaysSo(t *testing.T) { - p := aPlanOf("gitea") - tierAsked := false - err := askFirstTierAtCut(p, - func(*inventory.Plan) (bool, error) { return false, fmt.Errorf("the catalogue could not be read") }, - func() error { tierAsked = true; return nil }) - if err != nil || tierAsked || !strings.Contains(p.Note, "could not be worked out") { - t.Fatalf("asked=%v err=%v note %q", tierAsked, err, p.Note) - } -} diff --git a/cmd/mesh-controller/plan_clock_integration_test.go b/cmd/mesh-controller/plan_clock_integration_test.go deleted file mode 100644 index 51e968d8..00000000 --- a/cmd/mesh-controller/plan_clock_integration_test.go +++ /dev/null @@ -1,56 +0,0 @@ -//go:build integration - -package main - -import ( - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -func TestAPlanStandingStillIsNotSavedAgain(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := asksRecorded(t) - // app's key is not built, so the plan stands on its tier as it did while the build ran; with the stand-in builder's - // key done at once, one step would carry it to done. - holdKeys(t, "app") - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa", - Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - advanceHeld(ctx, open) - recent, err := inv.RecentPlans(ctx, 1) - if err != nil || len(recent) != 1 || len(*asked) != 0 { - t.Fatalf("no plan building: %v %v, asked %v", recent, err, *asked) - } - before := recent[0] - if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" || - !before.Modules["app"].Key || before.Modules["app"].KeyAt != "c1aaaaaaaa" { - t.Fatalf("the plan is not waiting on its build: %+v", before) - } - - for range 3 { - advanceHeld(ctx, open) - } - after, err := inv.PlanByID(ctx, before.ID) - if err != nil { - t.Fatal(err) - } - if after.Revision != before.Revision || !after.Updated.Equal(before.Updated) { - t.Fatalf("a plan that did not move was saved again: revision %d → %d, saved %s → %s", - before.Revision, after.Revision, before.Updated, after.Updated) - } - if len(*asked) != 0 { - t.Fatalf("advancing took the key in: %v", *asked) - } -} diff --git a/cmd/mesh-controller/plan_clock_test.go b/cmd/mesh-controller/plan_clock_test.go deleted file mode 100644 index 7de204d1..00000000 --- a/cmd/mesh-controller/plan_clock_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq issue 296: a plan saved a few seconds ago that has stood in its tier for forty minutes says -// forty minutes, and LATE — the line counted from the last save, which every advance made. -func TestAPlansLineCountsFromItsTierNotFromItsLastSave(t *testing.T) { - now := time.Date(2026, 10, 7, 20, 30, 0, 0, time.UTC) - asked := now.Add(-40 * time.Minute) - p := inventory.Plan{ID: "plan-t", Repository: "novox/mesh-catalog", Commit: "3ad22356", State: inventory.PlanBuilding, - Created: now.Add(-40 * time.Minute), Updated: now.Add(-3 * time.Second), TierEntered: now.Add(-40 * time.Minute), - Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked}}} - - line := planLine(p, now) - if !strings.Contains(line, "building for 40m0s") || !strings.Contains(line, "LATE") { - t.Fatalf("a plan in its tier for 40m, saved 3s ago, reads %q", line) - } - st := planStatuses([]inventory.Plan{p}, now, nil)[0] - if !st.Late || !st.Since.Equal(p.TierEntered) { - t.Errorf("status --json says %+v", st) - } - if _, late := openPlans([]inventory.Plan{p}, now, nil); late != 1 { - t.Errorf("status counted %d late", late) - } - - // LATE is the stalled condition said on the line: the same start and the same bound, measured or not. - bounds := boundsOf([]inventory.Duration{{Subject: p.Repository, Took: 20 * time.Minute}}) - if got := bounds.of(p.Repository); got != time.Hour { - t.Fatalf("a repository whose tiers took 20m is given %s", got) - } - if got := bounds.of("novox/other"); got != tierAtLeast { - t.Fatalf("a repository with nothing measured is given %s", got) - } - for _, in := range []time.Duration{40 * time.Minute, 70 * time.Minute} { - q := p - q.TierEntered = now.Add(-in) - bound := bounds.of(q.Repository) - late := strings.Contains(planLineWith(q, now, bound), "LATE") - stalled := len(watchPlans(&signalFacts{now: now, plans: []planFacts{{id: q.ID, repository: q.Repository, - commit: q.Commit, tiers: 1, entered: inTierSince(q), bound: bound}}})) == 1 - if late != stalled || late != (in > bound) { - t.Errorf("in its tier %s with a bound of %s: LATE %v, stalled %v", in, bound, late, stalled) - } - } - - // A plan read without its tier's stamp counts from its last save, the only time it has. - old := p - old.TierEntered = time.Time{} - if line := planLine(old, now); !strings.Contains(line, "for 3s") || strings.Contains(line, "LATE") { - t.Errorf("a plan without its tier's stamp reads %q", line) - } - - // A walk that waited for its delivery's word counts its wait from when it was opened, and its tier - // from the word. - waiting := p - waiting.Delivery = &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat} - waiting.Modules = map[string]*inventory.PlanModule{} - if line := planLine(waiting, now); !strings.Contains(line, "for 40m0s") || strings.Contains(line, "LATE") { - t.Errorf("a walk waiting 40m reads %q", line) - } - if st := planStatuses([]inventory.Plan{waiting}, now, nil)[0]; st.Late { - t.Errorf("a waiting walk is called late: %+v", st) - } - let := now.Add(-5 * time.Minute) - gone := p - gone.Delivery = &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat, Go: &let} - if line := planLine(gone, now); !strings.Contains(line, "building for 5m0s") || strings.Contains(line, "LATE") { - t.Errorf("a walk let go 5m ago, opened 40m ago, reads %q", line) - } -} - -// novox/hq issue 296: a plan whose tier stands still is not written again by every advance — its revision, -// its last save and its `plan-moved` stay where its last change left them. - -// novox/hq issue 920: a building plan whose current tier is all built waits on its note (a send refused at the -// gate), and its line, `status` and the stalled condition say that, not "building for ... LATE". -func TestAPlanLineSaysTheRefusalOnceItsTierIsBuilt(t *testing.T) { - now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC) - built := now.Add(-time.Hour) - refusal := "tier 1: a build waits there for its gate: novox (mesh-delivery-indexer d68d7ef3→a5aa69a4) — tried again" - mk := func(second string) inventory.Plan { - return inventory.Plan{ID: "plan-r", Repository: "novox/mesh-sdk", Commit: "b2f5ebb2", State: inventory.PlanBuilding, - Tier: 1, Created: now.Add(-2 * time.Hour), Updated: now, TierEntered: now.Add(-time.Hour), Note: refusal, - Tiers: [][]string{{"a"}, {"b", "c"}}, - Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &built}, - "b": {State: "built", BuiltAt: &built}, "c": {State: second}}} - } - - // 920/AC1 - p := mk("built") - line := planLine(p, now) - if !strings.Contains(line, "a build waits there for its gate: novox") || strings.Contains(line, "building for") { - t.Errorf("920/AC1: a plan with its tier built and a refusal reads %q", line) - } - st := planStatuses([]inventory.Plan{p}, now, nil) - if len(st) != 1 || !st[0].Late { - t.Errorf("920/AC1: status: %+v", st) - } - - // 920/AC2 - if line := planLine(mk("asked"), now); !strings.Contains(line, "building for 1h0m0s") { - t.Errorf("920/AC2: a plan with a tier module not built reads %q", line) - } -} - -// 920/AC3: a tier move writes the previous tier's summary into the note; with the new tier all built, the line still -// says "building for", for that note is no refusal. -func TestAPlanLineDoesNotHoldOnATiersSummary(t *testing.T) { - now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC) - built := now.Add(-time.Hour) - for _, note := range []string{"failed or refused its send (apply-failing says it): novox", "tier 0 built; sent mesh-sdk to novox first, in one send"} { - p := inventory.Plan{ID: "plan-s", Repository: "novox/mesh-sdk", Commit: "b2f5ebb2", State: inventory.PlanBuilding, - Tier: 1, Created: now.Add(-2 * time.Hour), Updated: now, TierEntered: now.Add(-time.Hour), Note: note, - Tiers: [][]string{{"a"}, {"b"}}, - Modules: map[string]*inventory.PlanModule{"a": {State: "built", BuiltAt: &built}, "b": {State: "built", BuiltAt: &built}}} - if line := planLine(p, now); !strings.Contains(line, "building for 1h0m0s") || strings.Contains(line, "held") { - t.Errorf("920/AC3: a plan with the note %q reads %q", note, line) - } - } -} diff --git a/cmd/mesh-controller/plan_failed.go b/cmd/mesh-controller/plan_failed.go deleted file mode 100644 index 2e3011ec..00000000 --- a/cmd/mesh-controller/plan_failed.go +++ /dev/null @@ -1,227 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "regexp" - "sort" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// failedModule is one module of a failed plan that failed to build, where it runs and what it said. -type failedModule struct { - module string - machines []string - why string -} - -// failedPlanFacts is one failed delivery plan asked of the operator (S23, novox/hq ADR 0449 §1). -type failedPlanFacts struct { - id, repository, commit string - modules []string - failed []failedModule - // note is where the plan stopped when no module failed to build (a rollout stopped at its first machine). - note string - // since is the plan's last change: when it failed. - since time.Time - // refusal is why `plans retry` would not run it, empty when it would. - refusal string - // passed is set when a newer plan of its repository and branch is done and the retry is refused for that - // (novox/hq ADR 0451): the condition asks nothing. - passed bool -} - -// machines are the machines the failed modules run on, sorted, each once. -func (p failedPlanFacts) machines() []string { - seen := map[string]bool{} - var out []string - for _, m := range p.failed { - for _, n := range m.machines { - if !seen[n] { - seen[n] = true - out = append(out, n) - } - } - } - sort.Strings(out) - return out -} - -// failureSaid is what failed, in the mesh's own words, for the condition's evidence. -func (p failedPlanFacts) failureSaid() string { - var parts []string - for _, m := range p.failed { - line := m.module - if len(m.machines) > 0 { - line += " on " + strings.Join(m.machines, ", ") - } - if m.why != "" { - line += ": " + oneLine(m.why) - } - parts = append(parts, line) - } - if len(parts) == 0 { - return "stopped: " + oneLine(p.note) - } - return strings.Join(parts, "; ") -} - -// refusalSaid is why a retry would not run, for the evidence. -func (p failedPlanFacts) refusalSaid() string { - if p.refusal == "" { - return "" - } - return "; a retry would not run: " + oneLine(p.refusal) -} - -// retryAction is the one answer a failed plan offers: the controller's `plans` verb with `retry` naming the plan, -// at the level approve. Only that argument: `plans` refuses a retry that carries others (seatverbs). -func retryAction(plan string) conditions.Action { - return conditions.Action{Label: "Retry", Verb: "mesh-controller.plans", Level: conditions.LevelApprove, - Arguments: map[string]string{"retry": plan}} -} - -var ( - planIDWord = regexp.MustCompile(`\bplan-[0-9]+\b`) - walkWd = regexp.MustCompile(`\bwalks\b`) - walkOne = regexp.MustCompile(`\bwalk\b`) -) - -// plainRefusal is a retry's refusal in words the operator reads: the plans named as "the plan", a walk said as a -// delivery. Whatever still breaks the plain rule (a hash, a dotted name) is left to the condition's evidence, and the -// words say only that a retry would not run. -func plainRefusal(refusal string) string { - return plainRefusalOr(refusal, "a retry would not run now; the details say why") -} - -// plainRefusalOr is plainRefusal with the sentence to say when the refusal's own words cannot be said plainly. -func plainRefusalOr(refusal, fallback string) string { - r := planIDWord.ReplaceAllString(refusal, "the plan") - r = walkWd.ReplaceAllString(r, "deliveries") - r = walkOne.ReplaceAllString(r, "delivery") - r = strings.TrimSpace(r) - if _, ok := conditions.Plain(r); !ok { - return fallback - } - return r -} - -// planFailedWords are the plain words of a failed plan: which modules failed and where, what they said when that -// is plain, and Retry when a retry would run, else the refusal's words. -func planFailedWords(p failedPlanFacts) (headline, explanation, needs string, actions []conditions.Action, resolved string) { - name := deliveryName(p.modules, p.repository) - headline = conditions.Capital(name) + " failed" - what := deliveryWhat(p.modules, p.repository) - var parts []string - for i, m := range p.failed { - if i == 2 { - parts = append(parts, fmt.Sprintf("%d more", len(p.failed)-2)) - break - } - line := m.module - if len(m.machines) > 0 { - line += " for " + readableList(m.machines) - } - if w := strings.TrimRight(cutWords(oneLine(m.why), 110), ".;:, "); w != "" { - if _, ok := conditions.Plain(w); ok { - line += " (" + w + ")" - } - } - parts = append(parts, line) - } - if len(parts) > 0 { - explanation = fmt.Sprintf("The delivery of %s failed: %s did not build.", what, readableList(parts)) - } else { - explanation = fmt.Sprintf("The delivery of %s stopped before it finished.", what) - } - switch { - case p.passed: - explanation += " A newer delivery has passed it, so a retry would send older code. Nothing is needed from you." - case p.refusal == "": - explanation += " Nothing is tried again until you say so." - needs = "try the failed builds again " + FromMeshMCPServer - actions = []conditions.Action{retryAction(p.id)} - default: - explanation += " " + conditions.Capital(plainRefusal(p.refusal)) + "." - needs = "look at why it failed; a retry would not run now." - } - resolved = conditions.Capital(name) + " is no longer failed" - return -} - -// cutWords is s cut to at most n bytes at a word. -func cutWords(s string, n int) string { - if len(s) <= n { - return s - } - t := s[:n] - if i := strings.LastIndexByte(t, ' '); i > n/2 { - t = t[:i] - } - return strings.TrimRight(t, " ,;:") + "…" -} - -// gatherFailedPlans is every delivery plan that failed within planFailedWithin and is asked of the operator -// (S23, novox/hq ADR 0449 §1): a failed plan that carries a delivery, is no rollout of held updates and was not -// stopped by a person. Each carries why `plans retry` would not run it, when it would not. -func gatherFailedPlans(ctx context.Context, inv *inventory.Inventory, now time.Time) ([]failedPlanFacts, error) { - plans, err := inv.PlansSince(ctx, now.Add(-planFailedWithin)) - if err != nil { - return nil, fmt.Errorf("the plans cannot be read: %w", err) - } - var failed []inventory.Plan - for _, p := range plans { - if failedAndAsked(p) { - failed = append(failed, p) - } - } - if len(failed) == 0 { - return nil, nil - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return nil, fmt.Errorf("the catalogue cannot be read: %w", err) - } - onPath := ownPathOf(entries) - runsOn := map[string][]string{} - for _, e := range entries { - runsOn[e.Manifest.Module] = e.On - } - var out []failedPlanFacts - for _, p := range failed { - out = append(out, failedFactsOf(p, plans, onPath, runsOn)) - } - return out, nil -} - -// failedAndAsked says a plan is one S23 asks about: failed, a delivery's, not a rollout of held updates, not -// stopped by a person. -func failedAndAsked(p inventory.Plan) bool { - return p.State == inventory.PlanFailed && p.Delivery != nil && p.Release == nil && p.Delivery.Stopped == "" -} - -// failedFactsOf is what S23 says of one failed plan, given every plan, the controller's own path and where each -// module runs. -func failedFactsOf(p inventory.Plan, plans []inventory.Plan, onPath map[string]string, - runsOn map[string][]string) failedPlanFacts { - f := failedPlanFacts{id: p.ID, repository: p.Repository, commit: p.Commit, modules: planModules(p), - note: p.Note, since: p.Updated} - for _, m := range f.modules { - if s := p.Modules[m]; s != nil && s.State == "failed" { - machines := append([]string(nil), runsOn[m]...) - sort.Strings(machines) - f.failed = append(f.failed, failedModule{module: m, machines: machines, why: s.Why}) - } - } - if err := retryRefusal(p, plans, onPath); err != nil { - f.refusal = err.Error() - var passed passedBy - f.passed = errors.As(err, &passed) - } - return f -} diff --git a/cmd/mesh-controller/plan_failed_test.go b/cmd/mesh-controller/plan_failed_test.go deleted file mode 100644 index 05691840..00000000 --- a/cmd/mesh-controller/plan_failed_test.go +++ /dev/null @@ -1,338 +0,0 @@ -package main - -import ( - "context" - "fmt" - "sort" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// **A failed delivery plan is asked of the operator** (novox/hq ADR 0449 §1 and §2, issue 1314). - -func aFailedPlan(id string, mutate func(p *inventory.Plan)) inventory.Plan { - p := inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Commit: "48bda475dunst0000", State: inventory.PlanFailed, - Tiers: [][]string{{"openrazer"}}, Updated: time.Date(2026, 10, 19, 18, 26, 0, 0, time.UTC), - Modules: map[string]*inventory.PlanModule{"openrazer": {State: "failed", Why: "the build could not fetch its source"}}, - Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}} - if mutate != nil { - mutate(&p) - } - return p -} - -func failedFactsFor(p inventory.Plan, others ...inventory.Plan) failedPlanFacts { - return failedFactsOf(p, append([]inventory.Plan{p}, others...), map[string]string{}, - map[string][]string{"openrazer": {"g14", "shanks"}}) -} - -// 1314/AC1 -func TestAFailedDeliveryPlanIsRaisedWithRetryAsItsOnlyAction(t *testing.T) { - now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) - p := aFailedPlan("plan-1792433168676870614", nil) - f := calm(now) - f.failed = []failedPlanFacts{failedFactsFor(p)} - got := watchPlansFailed(f) - if len(got) != 1 { - t.Fatalf("%+v", got) - } - o := got[0] - if o.Kind != "plan-failed" || o.Severity != conditions.Warning || o.Resolver != conditions.ResolverOperator || - o.Scope != conditions.ScopePlan || o.ID != p.ID { - t.Fatalf("kind, severity, resolver or subject: %+v", o) - } - if len(o.Actions) != 1 { - t.Fatalf("actions: %+v", o.Actions) - } - a := o.Actions[0] - if a.Label != "Retry" || a.Verb != "mesh-controller.plans" || a.Level != conditions.LevelApprove || - len(a.Arguments) != 1 || a.Arguments["retry"] != p.ID { - t.Fatalf("the action is not Retry with only the plan: %+v", a) - } - if o.Needs == "" { - t.Fatal("no Needs: the asker wants an action with a need") - } - for _, want := range []string{"openrazer", "g14", "shanks", "could not fetch its source"} { - if !strings.Contains(o.Headline+o.Explanation+o.Said, want) { - t.Errorf("the words do not name %q:\nheadline %q\nexplanation %q\nsaid %q", want, o.Headline, o.Explanation, o.Said) - } - } - if !strings.Contains(o.Said, "could not fetch its source") || !strings.Contains(o.Explanation, "could not fetch its source") { - t.Errorf("the failure's own words are not in both the evidence and the explanation: %q / %q", o.Said, o.Explanation) - } - words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs, Actions: o.Actions} - if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok { - t.Errorf("the words are not plain: %s\n%+v", why, words) - } -} - -// 1314/AC2 -func TestAFailedRolloutOfHeldUpdatesOrAPersonStoppedPlanIsNotRaised(t *testing.T) { - for name, p := range map[string]inventory.Plan{ - "held updates": aFailedPlan("plan-1", func(p *inventory.Plan) { p.Release = &inventory.PlanRelease{} }), - "stopped": aFailedPlan("plan-2", func(p *inventory.Plan) { p.Delivery.Stopped = "jochen" }), - "no delivery": aFailedPlan("plan-3", func(p *inventory.Plan) { p.Delivery = nil }), - "building": aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanBuilding }), - } { - if failedAndAsked(p) { - t.Errorf("%s: a plan not asked of the operator is asked", name) - } - } - if !failedAndAsked(aFailedPlan("plan-5", nil)) { - t.Error("a failed delivery plan is not asked") - } -} - -// 1314/AC3: every refusal of `plans retry` gives a condition with no action that says it. -func TestAFailedPlanThatCannotBeRetriedHasNoActionAndSaysWhy(t *testing.T) { - now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) - older := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Created = now.Add(-time.Hour) }) - newer := aFailedPlan("plan-2", func(p *inventory.Plan) { - p.State, p.Created = inventory.PlanBuilding, now.Add(-time.Minute) - }) - superseded := aFailedPlan("plan-3", func(p *inventory.Plan) { p.State, p.Note = inventory.PlanSuperseded, "superseded by plan-9" }) - done := aFailedPlan("plan-4", func(p *inventory.Plan) { p.State = inventory.PlanDone }) - first := time.Date(2026, 10, 19, 18, 20, 0, 0, time.UTC) - gate := aFailedPlan("plan-5", func(p *inventory.Plan) { - p.Note = "openrazer stopped at its first machine" - p.Modules = map[string]*inventory.PlanModule{"openrazer": {State: "built", FirstAt: &first, First: []string{"g14"}, - Why: "the build did not become healthy on g14", Gate: &inventory.PlanGate{Machines: []string{"g14"}, - Verdict: inventory.GateFailed, Why: "not healthy", From: "aaaaaaa", To: "bbbbbbb"}}} - }) - for name, c := range map[string]struct { - p inventory.Plan - others []inventory.Plan - says string - }{ - "a newer plan": {older, []inventory.Plan{newer}, "supersedes it"}, - "superseded": {superseded, nil, "superseded"}, - "done": {done, nil, "nothing to retry"}, - "gate failed": {gate, nil, "failed its gate"}, - } { - f := calm(now) - f.failed = []failedPlanFacts{failedFactsFor(c.p, c.others...)} - got := watchPlansFailed(f) - if len(got) != 1 { - t.Fatalf("%s: %+v", name, got) - } - o := got[0] - if len(o.Actions) != 0 { - t.Errorf("%s: a refused retry carries an action: %+v", name, o.Actions) - } - if !strings.Contains(o.Said, c.says) { - t.Errorf("%s: the evidence does not carry the refusal's words (%q): %q", name, c.says, o.Said) - } - if !strings.Contains(o.Explanation, "retry would not run") && !strings.Contains(o.Explanation, c.says) { - t.Errorf("%s: the explanation does not say the refusal: %q", name, o.Explanation) - } - words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs} - if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok { - t.Errorf("%s: the words are not plain: %s\n%+v", name, why, words) - } - } -} - -// 1314/AC4: the condition ends when the plan is no longer failed, kept by the keeper reconciling S23. -func TestAFailedPlanConditionEndsWhenThePlanLeavesFailed(t *testing.T) { - now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) - store := conditions.NewInMemory() - k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, - Teller: &conditions.Told{}, Now: func() time.Time { return now }}) - defer k.Close(t.Context()) - p := aFailedPlan("plan-1792433168676870614", nil) - f := calm(now) - f.failed = []failedPlanFacts{failedFactsFor(p)} - if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { - t.Fatal(err) - } - open, _ := k.Open(t.Context()) - if len(open) != 1 || open[0].Kind != "plan-failed" { - t.Fatalf("not raised: %+v", open) - } - // Retried: the plan is building, so it is no longer among the failed. - f = calm(now.Add(time.Minute)) - if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { - t.Fatal(err) - } - if open, _ = k.Open(t.Context()); len(open) != 0 { - t.Fatalf("the condition outlived the failure: %+v", open) - } - // Failed again: asked again, with Retry, and nothing retried it. - f = calm(now.Add(2 * time.Minute)) - again := aFailedPlan(p.ID, func(q *inventory.Plan) { q.Updated = now.Add(2 * time.Minute) }) - f.failed = []failedPlanFacts{failedFactsFor(again)} - if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { - t.Fatal(err) - } - open, _ = k.Open(t.Context()) - if len(open) != 1 || len(open[0].Actions) != 1 || open[0].Actions[0].Label != "Retry" { - t.Fatalf("a second failure was not asked again with Retry: %+v", open) - } -} - -// 1314/AC7 and ADR 0449 §2: no look at a failed plan retries it, however much time passes; the only thing -// S23 produces is an observation that offers an answer. -func TestNoClockRetriesAFailedPlan(t *testing.T) { - now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) - f := calm(now) - f.failed = []failedPlanFacts{failedFactsFor(aFailedPlan("plan-1", nil))} - for _, later := range []time.Duration{time.Hour, 24 * time.Hour, 6 * 24 * time.Hour} { - f.now = now.Add(later) - got := watchPlansFailed(f) - if len(got) != 1 || len(got[0].Actions) != 1 || got[0].Actions[0].Arguments["retry"] != "plan-1" { - t.Fatalf("after %s: %+v", later, got) - } - } - // A plan failed longer ago than the week it is kept is left to `plans`. - f.now = now.Add(planFailedWithin + time.Second) - if got := watchPlansFailed(f); len(got) != 0 { - t.Fatalf("a plan failed beyond the window is still raised: %+v", got) - } -} - -// A later plan of the same repository and branch, in a state, made after the failed one. -func laterPlan(id, branch, state string, after inventory.Plan) inventory.Plan { - q := aFailedPlan(id, func(p *inventory.Plan) { - p.State, p.Branch, p.Created = state, branch, after.Created.Add(time.Hour) - }) - return q -} - -// 1328/AC1: a later done plan of the same repository and branch makes `plans retry` refuse, naming the later plan. -func TestARetryIsRefusedWhenALaterPlanOfTheBranchIsDone(t *testing.T) { - failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) }) - later := laterPlan("plan-2", "main", inventory.PlanDone, failed) - err := retryRefusal(failed, []inventory.Plan{failed, later}, map[string]string{}) - if err == nil || !strings.Contains(err.Error(), "plan-2") || !strings.Contains(err.Error(), "passed") { - t.Fatalf("the retry was not refused, naming the later plan: %v", err) - } -} - -// 1328/AC2: with the later plan still open the refusal is today's; a later done plan of another branch refuses nothing. -func TestARetryIsRefusedAsBeforeWhileTheLaterPlanIsOpenAndNotForAnotherBranch(t *testing.T) { - failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 19, 12, 0, 0, 0, time.UTC) }) - open := laterPlan("plan-2", "main", inventory.PlanBuilding, failed) - err := retryRefusal(failed, []inventory.Plan{failed, open}, map[string]string{}) - if err == nil || !strings.Contains(err.Error(), "supersedes it") { - t.Fatalf("with the later plan open the refusal is not today's: %v", err) - } - other := laterPlan("plan-3", "feature/x", inventory.PlanDone, failed) - if err := retryRefusal(failed, []inventory.Plan{failed, other}, map[string]string{}); err != nil { - t.Fatalf("a later done plan of another branch refused the retry: %v", err) - } - earlier := laterPlan("plan-4", "main", inventory.PlanDone, failed) - earlier.Created = failed.Created.Add(-time.Hour) - if err := retryRefusal(failed, []inventory.Plan{failed, earlier}, map[string]string{}); err != nil { - t.Fatalf("an earlier done plan refused the retry: %v", err) - } -} - -// 1328/AC3: such a plan's plan-failed condition has no action and says it was passed by a newer delivery. -func TestAPassedFailedPlanHasNoActionAndSaysItWasPassed(t *testing.T) { - now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) - failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", now.Add(-5*time.Hour) }) - later := laterPlan("plan-2", "main", inventory.PlanDone, failed) - f := calm(now) - f.failed = []failedPlanFacts{failedFactsFor(failed, later)} - got := watchPlansFailed(f) - if len(got) != 1 || len(got[0].Actions) != 0 { - t.Fatalf("%+v", got) - } - o := got[0] - if !strings.Contains(o.Explanation, "newer delivery") || !strings.Contains(o.Explanation, "passed") { - t.Errorf("the words do not say it was passed by a newer delivery: %q", o.Explanation) - } - if strings.Contains(o.Explanation, "details say why") { - t.Errorf("the generic fallback was said: %q", o.Explanation) - } - words := conditions.Words{Headline: o.Headline, Explanation: o.Explanation, Resolved: o.Resolved, Needs: o.Needs} - if why, ok := conditions.PlainWords(words, "g14", "shanks"); !ok { - t.Errorf("not plain: %s\n%+v", why, words) - } -} - -// plansBehind serves plans as the inventory does: the open ones, the most recent n, and every one made since. -type plansBehind []inventory.Plan - -func (s plansBehind) OpenPlans(context.Context) ([]inventory.Plan, error) { - var out []inventory.Plan - for _, p := range s { - if p.Open() { - out = append(out, p) - } - } - return out, nil -} - -func (s plansBehind) RecentPlans(_ context.Context, limit int) ([]inventory.Plan, error) { - all := append([]inventory.Plan(nil), s...) - sort.SliceStable(all, func(i, j int) bool { return all[i].Created.After(all[j].Created) }) - if len(all) > limit { - all = all[:limit] - } - return all, nil -} - -func (s plansBehind) PlansSince(_ context.Context, since time.Time) ([]inventory.Plan, error) { - var out []inventory.Plan - for _, p := range s { - if p.Created.After(since) || p.Open() { - out = append(out, p) - } - } - return out, nil -} - -// 1328, review round 3: the plan that passed a failed one is seen by `plans retry` however many plans were made after it. -func TestARetrySeesThePassingPlanBeyondTheFiftyMostRecent(t *testing.T) { - failed := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Branch, p.Created = "main", time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC) }) - all := plansBehind{failed, laterPlan("plan-done", "main", inventory.PlanDone, failed)} - for i := 0; i < 60; i++ { // 60 newer plans of other repositories push the passing one out of the newest 50 - other := laterPlan(fmt.Sprintf("plan-x%d", i), "main", inventory.PlanFailed, failed) - other.Repository = "novox/other" - other.Created = failed.Created.Add(time.Duration(48+i) * time.Hour) - all = append(all, other) - } - plans, err := plansForRetry(t.Context(), all, failed) - if err != nil { - t.Fatal(err) - } - if err := retryRefusal(failed, plans, map[string]string{}); err == nil || !strings.Contains(err.Error(), "plan-done") { - t.Fatalf("the retry was not refused naming the passing plan beyond the 50 most recent: %v", err) - } -} - -// ADR 0451 §3: a failed plan that last changed 8 days ago raises nothing, and a raised condition ends once its -// plan's last change passes 7 days. -func TestAFailedPlanLastChangedEightDaysAgoRaisesNothingAndARaisedOneEnds(t *testing.T) { - now := time.Date(2026, 10, 19, 18, 30, 0, 0, time.UTC) - old := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Updated = now.Add(-8 * 24 * time.Hour) }) - f := calm(now) - f.failed = []failedPlanFacts{failedFactsFor(old)} - if got := watchPlansFailed(f); len(got) != 0 { - t.Fatalf("a plan last changed 8 days ago raised %+v", got) - } - store := conditions.NewInMemory() - k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store, - Teller: &conditions.Told{}, Now: func() time.Time { return now }}) - defer k.Close(t.Context()) - fresh := aFailedPlan("plan-1", func(p *inventory.Plan) { p.Updated = now.Add(-6 * 24 * time.Hour) }) - f.failed = []failedPlanFacts{failedFactsFor(fresh)} - if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { - t.Fatal(err) - } - if open, _ := k.Open(t.Context()); len(open) != 1 { - t.Fatalf("a plan last changed 6 days ago is not raised: %+v", open) - } - f.now = now.Add(2 * 24 * time.Hour) // its last change is now 8 days old - if err := k.Reconcile(t.Context(), "S23", watchPlansFailed(f)); err != nil { - t.Fatal(err) - } - if open, _ := k.Open(t.Context()); len(open) != 0 { - t.Fatalf("the condition outlived the 7 days: %+v", open) - } -} diff --git a/cmd/mesh-controller/plan_ids.go b/cmd/mesh-controller/plan_ids.go deleted file mode 100644 index d9dca230..00000000 --- a/cmd/mesh-controller/plan_ids.go +++ /dev/null @@ -1,26 +0,0 @@ -package main - -import ( - "fmt" - "regexp" -) - -// planID is every id the mesh gives a plan: a merge's walk or batch (plan-) or a release -// (release-). -var planID = regexp.MustCompile(`^(plan|release)-[0-9]+$`) - -// planIDArguments are the arguments of plans that name a plan. wait is among them for when it is declared -// (novox/hq issue 229); an argument not declared is refused before this is asked. -var planIDArguments = []string{"id", "wait", "stop", "close", "retry", "go"} - -// plainPlanIDs refuses a plans call naming a plan by anything but a plan's id (novox/hq issue 561): checked -// once, where the call becomes a command line, so nothing else a caller sent reaches a command or the store. -func plainPlanIDs(given map[string]string) error { - for _, arg := range planIDArguments { - if id, named := given[arg]; named && !planID.MatchString(id) { - return fmt.Errorf("plans %s: %q is not a plan's id (plan- or release-, as `plans` "+ - "lists them); nothing was done", arg, id) - } - } - return nil -} diff --git a/cmd/mesh-controller/plan_ids_test.go b/cmd/mesh-controller/plan_ids_test.go deleted file mode 100644 index f49039f4..00000000 --- a/cmd/mesh-controller/plan_ids_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package main - -import ( - "strings" - "testing" -) - -// **A plan's id is checked once, where a call becomes a command line** (novox/hq issue 561): every argument of -// plans that names a plan is refused unless it is an id the mesh gives a plan, so a quote never reaches a -// command, nor the store behind it. -func TestAPlanIdThatIsNoPlansIdIsRefusedBeforeACommandStarts(t *testing.T) { - for _, arg := range []string{"id", "stop", "close", "retry", "go"} { - for _, bad := range []string{"x' or '1'='1", "plan-1'", "plan-", "plan-1 2", "batch-1", "PLAN-1"} { - args := map[string]any{arg: bad} - if arg == "stop" || arg == "close" || arg == "go" { - args["why"] = "a test" - } - argv, err := argvFor("plans", args) - if err == nil { - t.Errorf("plans %s %q was composed: %v", arg, bad, argv) - continue - } - if !strings.Contains(err.Error(), "is not a plan's id") { - t.Errorf("plans %s %q was refused for another reason: %v", arg, bad, err) - } - } - for _, good := range []string{"plan-1791813626034291775", "release-1791811938267404408"} { - args := map[string]any{arg: good} - if arg == "stop" || arg == "close" || arg == "go" { - args["why"] = "a test" - } - if _, err := argvFor("plans", args); err != nil { - t.Errorf("plans %s %q was refused: %v", arg, good, err) - } - } - } -} diff --git a/cmd/mesh-controller/plan_keys.go b/cmd/mesh-controller/plan_keys.go deleted file mode 100644 index f867ad6b..00000000 --- a/cmd/mesh-controller/plan_keys.go +++ /dev/null @@ -1,268 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "sync" - "time" - - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - - "github.com/novox/mesh-controller/internal/builderkeys" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A walk takes its modules from the builder's done keys (novox/hq ADR 0503 §6, ADR 0501 §1, ADR 0498 §8; issue 1851, A6 -// and A7). -// -// A walk asks no build of anybody (R4g; ADR 0453 §5). A module of its tier is marked -// taken from a key (PlanModule.Key) and settled by the look: its fingerprint is read from the indexer's trunk value -// written for the commit the walk carries of its repository; the builder's key of that fingerprint, done with every -// artifact referenced, is taken in (recorded and registered, off the plans' hold) and the module is built; a key not done -// yet, or waiting for a reference, holds the tier and says so; a failed key fails the walk in the builder's own words. -// **No timer**: the builder's and the indexer's writes wake the look (builderkeys.Watch, watchNewModules). - -// planKeyReads is what a walk reads of the indexer and the builder (builderkeys.Reader, or a test's). -type planKeyReads interface { - ModuleAt(ctx context.Context, module, commit string) (sdkdelivery.Module, error) - Build(ctx context.Context, module, fp string) (builderkeys.Key, error) -} - -// openPlanKeys opens what a walk reads of the indexer and the builder. A variable so a test needs no bus. -var openPlanKeys = func() (planKeyReads, func(), error) { - r, closeKeys, err := openBuilderKeys() - if err != nil { - return nil, nil, err - } - return r, closeKeys, nil -} - -// takeFromKey marks a module of a walk's tier as taken from the builder's key: asked of nobody (ADR 0501 §6). -func takeFromKey(p *inventory.Plan, state *inventory.PlanModule, e inventory.Entry, now time.Time) { - state.BusWaitFrom = nil - state.State = "asked" - state.AskedAt = &now - state.Key = true - state.Build, state.Fingerprint = "", "" - state.KeyAt = p.CommitOn(e.Source.Repository, followedBranch(e.Source.Ref)) - state.Why = "waits for the builder's key" - state.Commit, state.BuiltAt, state.Made = "", nil, "" - fmt.Printf(" tier %d: %s is taken from the builder's key, asking no build\n", p.Tier, e.Manifest.Module) -} - -// keyTakes are the take-ins of keys a look started and has not seen end, by key build id, and what each said when it -// waits: kept in this process, so a look starts each once and a settle says why it waits. -var keyTakes = struct { - sync.Mutex - running map[string]bool - waits map[string]string - // taken are the keys whose take-in ended registered, or found the module already at a later build (superseded). - taken map[string]bool -}{running: map[string]bool{}, waits: map[string]string{}, taken: map[string]bool{}} - -// settleFromKeys settles every module of the tier taken from a key: built from its key's record, failed with the key's -// words or the take-in's refusal, or still waiting, saying for what. It says whether it changed the plan. It reads only -// the buckets and the records; a done key not recorded yet is taken in off the plans' hold, and its end asks the look. -func settleFromKeys(ctx context.Context, open *stores, keys planKeyReads, p *inventory.Plan, tier []string) (bool, error) { - return settleKeys(ctx, open.inventory, keys, p, tier, func(module string, trunk sdkdelivery.Module, id string) { - inBackground(func() { takeKeyIn(context.WithoutCancel(ctx), open, module, trunk, id) }) - }) -} - -// takeKeyIn is how a look takes a done key in; a variable so a test of what a walk does around it needs no builder. -var takeKeyIn = takeKeyOffTheHold - -// keyRecords is what settling from keys reads of the records (*inventory.Inventory, or a test's). -type keyRecords interface { - BuildByID(ctx context.Context, id string) (inventory.Build, bool, error) - RefusedOf(ctx context.Context, ids []string) (map[string]string, error) - SourceOf(ctx context.Context, module string) (inventory.Source, error) -} - -// settleKeys is settleFromKeys on the records handed in, starting a take-in with take. -func settleKeys(ctx context.Context, inv keyRecords, keys planKeyReads, p *inventory.Plan, tier []string, - take func(module string, trunk sdkdelivery.Module, id string)) (bool, error) { - changed := false - set := func(s *inventory.PlanModule, why string) { - if s.Why != why { - s.Why = why - changed = true - } - } - for _, m := range tier { - s := p.Modules[m] - if s == nil || !s.Key || s.State != "asked" { - continue - } - trunk, err := keys.ModuleAt(ctx, m, s.KeyAt) - if errors.Is(err, builderkeys.ErrNoValue) { - set(s, fmt.Sprintf("waits for the indexer's value of %s at %s", m, orNow(s.KeyAt))) - continue - } - if err != nil { - return changed, err - } - if trunk.FP2 == "" { - why := trunk.None2 - if why == "" { - why = "the indexer wrote no fingerprint of the new form" - } - set(s, fmt.Sprintf("waits: %s at %s has no fingerprint to take a build by: %s", m, orNow(s.KeyAt), why)) - continue - } - if s.Fingerprint != trunk.FP2 { - s.Fingerprint, changed = trunk.FP2, true - } - k, err := keys.Build(ctx, m, trunk.FP2) - if err != nil { - return changed, err - } - switch { - case k.Failed(): - // **A failed key fails the walk in the builder's own words** (`digests differ:`, `not stored:`, `not pinned:`, - // `version taken:`), and a rebuild of it is a new revision the next walk takes. - s.State, s.Why = "failed", k.Build.Error - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("%s failed to build in tier %d: %s", m, p.Tier, firstLine(k.Build.Error)) - changed = true - continue - case !k.Done() || len(k.Unreferenced()) > 0: - set(s, "waits: "+k.Says()) - continue - } - id := keyBuildID(k) - refused, err := inv.RefusedOf(ctx, []string{id}) - if err != nil { - return changed, fmt.Errorf("the refused registrations cannot be read: %w", err) - } - for kept, why := range refusalsKeptFor([]string{id}) { - refused[kept] = why - } - if why, ok := refused[id]; ok { - s.State, s.Why, s.Build = "failed", why, id - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("%s failed to build in tier %d: %s", m, p.Tier, firstLine(why)) - changed = true - continue - } - b, found, err := inv.BuildByID(ctx, id) - if err != nil { - return changed, err - } - if found && b.Worked() { - keyTakes.Lock() - taken := keyTakes.taken[id] - keyTakes.Unlock() - registered := taken - if !registered { - if registered, err = registeredAt(ctx, inv, m, trunk.Commit); err != nil { - return changed, err - } - } - if registered { - at := b.At - s.State, s.BuiltAt, s.Commit, s.Build = "built", &at, trunk.Commit, id - s.Made = fmt.Sprintf("taken from the builder's key %s (revision %d), no build asked", short(trunk.FP2), k.Revision) - s.Why = "" - fmt.Printf("%s: %s taken from the builder's key %s\n", p.ID, m, short(trunk.FP2)) - changed = true - continue - } - } - keyTakes.Lock() - running, waits := keyTakes.running[id], keyTakes.waits[id] - if !running { - keyTakes.running[id] = true - } - keyTakes.Unlock() - if waits != "" { - set(s, firstLine(waits)) - } else { - set(s, "taking in the builder's done key "+short(trunk.FP2)) - } - if !running { - take(m, trunk, id) - } - } - return changed, nil -} - -// registeredAt says the catalogue holds the module registered at the commit the walk takes it at: its key's take-in -// registered it there, whether with the key's artifacts or, unchanged, with the build it stands on (no move). A key -// recorded at an earlier commit (the same fingerprint at an older merge) is taken in again, to register it here. -func registeredAt(ctx context.Context, inv keyRecords, module, commit string) (bool, error) { - src, err := inv.SourceOf(ctx, module) - if errors.Is(err, inventory.ErrNoSuchModule) { - return false, nil - } - if err != nil { - return false, err - } - return sameCommitID(src.BuiltFrom, commit), nil -} - -func sameCommitID(a, b string) bool { - if a == "" || b == "" { - return false - } - if a == b { - return true - } - n := min(len(a), len(b)) - return n >= 7 && a[:n] == b[:n] -} - -func orNow(commit string) string { - if commit == "" { - return "its value now" - } - return short(commit) -} - -// takeKeyOffTheHold takes a walk's module in from its done key, off the plans' hold: a registration may sweep the -// artifact store, and the plans are held by one look at a time. Its end asks the look, which settles the module from the -// record; a refusal is kept by the key's build id, so the look fails the walk with it; a wait is said by the look. -func takeKeyOffTheHold(ctx context.Context, open *stores, module string, trunk sdkdelivery.Module, id string) { - defer func() { - keyTakes.Lock() - delete(keyTakes.running, id) - keyTakes.Unlock() - askToLook() - }() - keys, closeKeys, err := openBuilderKeys() - if err != nil { - keepKeyWait(id, "waits: the builder's keys cannot be read: "+err.Error()) - return - } - defer closeKeys() - _, _, err = takeInKey(ctx, open.inventory, keys, keyTake{Module: module, Trunk: trunk, - Ask: registrationAsk{Merge: trunk.Commit}}) - switch { - case err == nil, errors.Is(err, inventory.ErrSuperseded): - // Superseded is no failure: the module is already at what a later merge's key built, which a walk that carries - // this merge is answered by, as a build-agent outcome superseded was. - keepKeyWait(id, "") - keyTakes.Lock() - keyTakes.taken[id] = true - keyTakes.Unlock() - case errors.Is(err, errKeyWaits): - keepKeyWait(id, err.Error()) - default: - why := err.Error() - if rerr := open.inventory.RecordRefused(ctx, id, module, why); rerr != nil { - fmt.Printf("plans: %s's refusal could not be kept in the store, only in this controller: %v\n", module, rerr) - } - keepRefusalFor(id, why, time.Now()) - } -} - -func keepKeyWait(id, why string) { - keyTakes.Lock() - defer keyTakes.Unlock() - if why == "" { - delete(keyTakes.waits, id) - return - } - keyTakes.waits[id] = why -} diff --git a/cmd/mesh-controller/plan_keys_test.go b/cmd/mesh-controller/plan_keys_test.go deleted file mode 100644 index 3fe15089..00000000 --- a/cmd/mesh-controller/plan_keys_test.go +++ /dev/null @@ -1,205 +0,0 @@ -package main - -import ( - "context" - "strings" - "testing" - "time" - - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - "github.com/novox/mesh-controller/internal/builderkeys" - "github.com/novox/mesh-controller/internal/inventory" -) - -// planKeysOf is the indexer and the builder as a walk reads them, for a test: values by module and commit, keys by -// module. -type planKeysOf struct { - values map[string]sdkdelivery.Module // by module + "@" + commit - keys map[string]builderkeys.Key -} - -func (k planKeysOf) ModuleAt(_ context.Context, module, commit string) (sdkdelivery.Module, error) { - if v, ok := k.values[module+"@"+commit]; ok { - return v, nil - } - return sdkdelivery.Module{}, builderkeys.ErrNoValue -} - -func (k planKeysOf) Build(_ context.Context, module, fp string) (builderkeys.Key, error) { - if key, ok := k.keys[module]; ok && key.Fingerprint == fp { - return key, nil - } - return builderkeys.Key{Module: module, Fingerprint: fp}, nil -} - -// recordsOf is the build records and refusals a settle reads, for a test. -type recordsOf struct { - builds map[string]inventory.Build - refused map[string]string - sources map[string]inventory.Source -} - -func (r recordsOf) BuildByID(_ context.Context, id string) (inventory.Build, bool, error) { - b, ok := r.builds[id] - return b, ok, nil -} - -func (r recordsOf) RefusedOf(_ context.Context, ids []string) (map[string]string, error) { - out := map[string]string{} - for _, id := range ids { - if why, ok := r.refused[id]; ok { - out[id] = why - } - } - return out, nil -} - -func (r recordsOf) SourceOf(_ context.Context, module string) (inventory.Source, error) { - if s, ok := r.sources[module]; ok { - return s, nil - } - return inventory.Source{}, inventory.ErrNoSuchModule -} - -func aWalkOfOne(module string) (*inventory.Plan, map[string]inventory.Entry) { - p := &inventory.Plan{ID: "plan-1", State: inventory.PlanBuilding, Tiers: [][]string{{module}}, - Modules: map[string]*inventory.PlanModule{}, Repository: "novox/mesh-catalog", Commit: "c0ffee0123456789", - Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: "c0ffee0123456789"}}}} - byName := map[string]inventory.Entry{module: {Source: inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", - Path: "modules/" + module, Ref: "main"}}} - return p, byName -} - -// A6 (ADR 0503 §6, ADR 0498 check §7): a walk whose module has a done key goes asked to built with no build asked: the -// ask stub is never called, the key is taken in once, and `plans` says it was taken from the builder's key. -func TestAWalkTakesADoneKeyAndAsksNoBuild(t *testing.T) { - p, byName := aWalkOfOne("sudo") - askModule(t.Context(), p, "sudo", byName) - s := p.Modules["sudo"] - if s.State != "asked" || !s.Key || s.Build != "" || s.KeyAt != "c0ffee0123456789" { - t.Fatalf("the module is not taken from a key: %+v", s) - } - fp := "src2:" + strings.Repeat("a", 64) - k := builderkeys.Key{Module: "sudo", Fingerprint: fp, Found: true, Revision: 2, Build: sdkdelivery.BuilderBuild{ - State: sdkdelivery.BuildDone, Manifest: strings.Repeat("b", 64), Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a", - Digest: "sha256:1", Reference: "artifact-store://sudo/a/blobs/sha256:1"}}}} - keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp, Commit: "c0ffee0123456789"}}, - keys: map[string]builderkeys.Key{"sudo": k}} - records := recordsOf{builds: map[string]inventory.Build{}, sources: map[string]inventory.Source{}} - var taken []string - take := func(m string, _ sdkdelivery.Module, id string) { taken = append(taken, m+" "+id) } - t.Cleanup(func() { - keyTakes.Lock() - keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{} - keyTakes.Unlock() - }) - - changed, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take) - if err != nil || !changed || len(taken) != 1 || s.State != "asked" || s.Fingerprint != fp { - t.Fatalf("a done key not recorded yet is taken in once: %v %v %v %+v", changed, err, taken, s) - } - // A second look while the take-in runs starts no second one. - if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil || len(taken) != 1 { - t.Fatalf("a second look took the key in again: %v %v", taken, err) - } - // The take-in recorded and registered it: the look settles it built from the record. - id := keyBuildID(k) - records.builds[id] = inventory.Build{ID: id, Module: "sudo", Commit: "c0ffee0123456789", At: time.Now()} - records.sources["sudo"] = inventory.Source{BuiltFrom: "c0ffee0123456789"} - if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil { - t.Fatal(err) - } - if s.State != "built" || s.Build != id || !strings.Contains(s.Made, "taken from the builder's key") { - t.Fatalf("the module is not built from its key: %+v", s) - } -} - -// A7: a key building waits and says so, then moves on when done, with no timer and no ask; a failed key fails the walk -// in the builder's words; a value the indexer has not written yet waits, named. -func TestAWalkWaitsForAKeyAndFailsWithItsWords(t *testing.T) { - t.Cleanup(func() { - keyTakes.Lock() - keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{} - keyTakes.Unlock() - }) - p, byName := aWalkOfOne("view") - askModule(t.Context(), p, "view", byName) - s := p.Modules["view"] - records := recordsOf{builds: map[string]inventory.Build{}} - take := func(string, sdkdelivery.Module, string) {} - keys := planKeysOf{values: map[string]sdkdelivery.Module{}, keys: map[string]builderkeys.Key{}} - if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil || - !strings.Contains(s.Why, "waits for the indexer's value of view at c0ffee01") { - t.Fatalf("no indexer value yet: %q %v", s.Why, err) - } - fp := "src2:" + strings.Repeat("c", 64) - keys.values["view@c0ffee0123456789"] = sdkdelivery.Module{FP2: fp} - keys.keys["view"] = builderkeys.Key{Module: "view", Fingerprint: fp, Found: true, - Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildBuilding, By: "g14"}} - if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil || - s.State != "asked" || !strings.Contains(s.Why, "building on g14") || p.State != inventory.PlanBuilding { - t.Fatalf("a building key: %+v %v", s, err) - } - keys.keys["view"] = builderkeys.Key{Module: "view", Fingerprint: fp, Found: true, - Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildFailed, Error: "not stored: tools: the store answered 500"}} - if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil { - t.Fatal(err) - } - if s.State != "failed" || p.State != inventory.PlanFailed || !strings.Contains(p.Note, "not stored: tools") { - t.Fatalf("a failed key does not fail the walk in its words: %+v %q", s, p.Note) - } -} - -// A7: a take-in refused (a module from another repository, a manifest refused) fails the walk with the refusal. -func TestARefusedTakeInFailsTheWalk(t *testing.T) { - t.Cleanup(func() { - keyTakes.Lock() - keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{} - keyTakes.Unlock() - }) - p, byName := aWalkOfOne("sudo") - askModule(t.Context(), p, "sudo", byName) - fp := "src2:" + strings.Repeat("d", 64) - k := builderkeys.Key{Module: "sudo", Fingerprint: fp, Found: true, Revision: 1, Build: sdkdelivery.BuilderBuild{ - State: sdkdelivery.BuildDone, Manifest: strings.Repeat("b", 64), Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a", - Digest: "sha256:1", Reference: "artifact-store://sudo/a/blobs/sha256:1"}}}} - keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp}}, - keys: map[string]builderkeys.Key{"sudo": k}} - records := recordsOf{refused: map[string]string{keyBuildID(k): "sudo is built from novox/mesh-catalog, and this build is of agent/sudo"}} - if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], func(string, sdkdelivery.Module, string) {}); err != nil { - t.Fatal(err) - } - if p.State != inventory.PlanFailed || !strings.Contains(p.Note, "agent/sudo") { - t.Fatalf("a refused take-in did not fail the walk: %s %q", p.State, p.Note) - } -} - -// A6: a walk's module whose fingerprint did not change since an earlier merge has its key recorded at that merge's -// commit: the walk takes it in again, so the module is registered at the commit the walk carries, before it is built. -func TestAKeyRecordedAtAnEarlierCommitIsTakenInAgain(t *testing.T) { - t.Cleanup(func() { - keyTakes.Lock() - keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{} - keyTakes.Unlock() - }) - p, byName := aWalkOfOne("sudo") - askModule(t.Context(), p, "sudo", byName) - fp := "src2:" + strings.Repeat("e", 64) - k := builderkeys.Key{Module: "sudo", Fingerprint: fp, Found: true, Revision: 1, Build: sdkdelivery.BuilderBuild{ - State: sdkdelivery.BuildDone, Manifest: strings.Repeat("b", 64), Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a", - Digest: "sha256:1", Reference: "artifact-store://sudo/a/blobs/sha256:1"}}}} - keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp, Commit: "c0ffee0123456789"}}, - keys: map[string]builderkeys.Key{"sudo": k}} - id := keyBuildID(k) - records := recordsOf{builds: map[string]inventory.Build{id: {ID: id, Module: "sudo", Commit: "0ld0ld0ld0ld"}}, - sources: map[string]inventory.Source{"sudo": {BuiltFrom: "0ld0ld0ld0ld"}}} - var taken []string - if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], func(m string, _ sdkdelivery.Module, _ string) { - taken = append(taken, m) - }); err != nil { - t.Fatal(err) - } - if s := p.Modules["sudo"]; s.State != "asked" || len(taken) != 1 { - t.Fatalf("a key recorded at an earlier commit settled the walk at that commit: %+v, taken %v", s, taken) - } -} diff --git a/cmd/mesh-controller/plan_looks.go b/cmd/mesh-controller/plan_looks.go deleted file mode 100644 index d8d84206..00000000 --- a/cmd/mesh-controller/plan_looks.go +++ /dev/null @@ -1,718 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "slices" - "sort" - "strings" - "sync" - "time" - - "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A rollout advances on events, not on a timer (novox/hq ADR 0406 §5, issue 974). One look at the open plans runs on -// each thing that can let a plan move: a plan saved (a build's `built`, a tier's send kept, a command's move), a -// machine's report on a send, a silent key's expiry, and the controller taking the lease. The looks are asked for -// with askToLook and run one at a time by a worker, so an ask made while a look runs is not lost and many asks are -// one look. Only two things are still started by time, both named and both one-shot: a failed advance step is -// looked at again (retryLook), and the end of a walk whose end is not known is recorded when it can be -// (armWalkEnd). - -// looker runs the look at the plans that events ask for. -type looker struct { - signal chan struct{} // one place: asks made together are one look - look func(ctx context.Context) (failed bool) - // gap is the least time between two looks, so a plan that kept changing could not make the worker spin. - gap time.Duration - // first runs before the first look: what a controller that took the lease sets up. - first func(ctx context.Context) - - mu sync.Mutex - retry *time.Timer - backoff time.Duration - base time.Duration - most time.Duration -} - -const ( - retryLookFirst = 5 * time.Second - retryLookMost = time.Minute - lookGap = 200 * time.Millisecond -) - -func newLooker(look func(ctx context.Context) bool) *looker { - return &looker{signal: make(chan struct{}, 1), look: look, gap: lookGap, base: retryLookFirst, most: retryLookMost} -} - -// ask has a look run, soon; never blocks. -func (l *looker) ask() { - select { - case l.signal <- struct{}{}: - default: - } -} - -// run takes the asks one at a time until the context ends. -func (l *looker) run(ctx context.Context) { - if l.first != nil { - l.first(ctx) - } - for { - select { - case <-ctx.Done(): - l.mu.Lock() - if l.retry != nil { - l.retry.Stop() - } - l.mu.Unlock() - return - case <-l.signal: - } - l.after(l.look(ctx)) - if l.gap > 0 { - select { - case <-ctx.Done(): - case <-time.After(l.gap): - } - } - } -} - -// after takes the outcome of a look: a failed one is looked at again later, a good one forgets the failures. -func (l *looker) after(failed bool) { - l.mu.Lock() - defer l.mu.Unlock() - if !failed { - l.backoff = 0 - if l.retry != nil { - l.retry.Stop() - l.retry = nil - } - return - } - if l.retry != nil { - return // one is armed already - } - switch { - case l.backoff == 0: - l.backoff = l.base - case l.backoff*2 > l.most: - l.backoff = l.most - default: - l.backoff *= 2 - } - // retries: an advance step that failed (store, bus or send). One at a time, five seconds doubling to a minute, forgotten by - // the first look that fails nothing; the step is otherwise started by an event, and a failure is none. - l.retry = time.AfterFunc(l.backoff, func() { - l.mu.Lock() - l.retry = nil - l.mu.Unlock() - l.ask() - }) -} - -// theLooker is the serving controller's; nil in a command, whose looks are its own. -var ( - lookerMu sync.RWMutex - theLooker *looker -) - -// askToLook has the serving controller look at the open plans soon: a plan was saved, a machine reported on a send, -// a silent key expired. Nothing in a process that is not serving. -func askToLook() { - lookerMu.RLock() - l := theLooker - lookerMu.RUnlock() - if l != nil { - l.ask() - } -} - -// startPlanLooks starts the look at the open plans, and asks for the first: the controller took the lease, and a -// plan left by a replaced controller resumes from there (ADR 0406 §6). It replaces the rollout's 30 second ticker. -// It answers a channel that is closed once the look has returned after the context ended, for a test that restores what -// the look reads (issue 1338). -func startPlanLooks(ctx context.Context, open *stores) <-chan struct{} { - l := newLooker(func(ctx context.Context) bool { return lookAtPlans(ctx, open) }) - l.first = func(ctx context.Context) { rearmWalkEnds(ctx, open) } - lookerMu.Lock() - theLooker = l - lookerMu.Unlock() - go func() { - <-ctx.Done() - lookerMu.Lock() - if theLooker == l { - theLooker = nil - } - lookerMu.Unlock() - }() - done := make(chan struct{}) - go func() { - defer close(done) - l.run(ctx) - }() - l.ask() - return done -} - -// applySilentAfter is how long a machine may say nothing on a send before the plan names it silent: the mesh's -// ApplySilentAfter, a variable so a test of the wait needs no quarter of an hour. -var applySilentAfter = inventory.ApplySilentAfter - -// startRolloutEvents starts what a rollout advances on, for as long as the context lives (novox/hq ADR 0406 §5, issue 974): -// the wait for each machine's report as a key on the bus that expires (where the bus will not take it a timer of its own -// for that wait looks instead), a provisioner client's words, and the look at the plans, asked for first so a plan left -// by a replaced controller resumes. The serving controller's start and the tests of a rollout call this one. It answers -// a function that waits until the look at the plans has returned, which only a test that ends the context needs (issue -// 1338): the serving controller ignores it. -func startRolloutEvents(ctx context.Context, open *stores, js *broker.JetStream) (wait func()) { - if err := startRolloutSilent(ctx, js); err != nil { - fmt.Printf("plans: the wait for the machines' reports could not be put on the bus (%v); each is looked at by a timer of its own\n", err) - } - if err := startProvAppliedLooks(ctx, js); err != nil { - fmt.Printf("plans: the provisioner clients' words could not be watched (%v); a walk waits for them until the next look\n", err) - } - if err := startPlanMovedLooks(ctx, js, open); err != nil { - fmt.Printf("plans: the plans moved by other processes could not be followed (%v)\n", err) - } - done := startPlanLooks(ctx, open) - return func() { <-done } -} - -// servingPlanSaved is what the serving controller does for each plan it keeps: said as plan-moved, looked at (a build, a -// send kept or a command's move is followed by the look that asks the next tier, in place of the timer that did), and, -// for a walk that ended, its merges' check builds forgotten (novox/hq ADR 0350 §5). -func servingPlanSaved(ctx context.Context, bus link.Bus, open *stores) func(*inventory.Inventory, inventory.Plan) { - return func(_ *inventory.Inventory, p inventory.Plan) { - sayPlanMoved(ctx, bus, open.inventory, p) - planKept(p) - } -} - -// planLookPasses is the most times one look goes over the plans again because the first pass changed which are open. -const planLookPasses = 6 - -// lookAtPlans moves every open plan as far as the facts allow, as the ticker did each pass: what waits for a gate is -// released, the plans advanced, the wait for each machine's report kept as a key on the bus, and the phases of the -// walks that ended recorded. It says whether a step failed. Waits for the plans rather than giving up: an ask is -// answered once. -func lookAtPlans(ctx context.Context, open *stores) (failed bool) { - release, err := open.inventory.HoldPlans(ctx, true) - if err != nil { - fmt.Printf("plans: cannot hold them: %v\n", err) - return true - } - func() { - defer release() - // What waits for a gate, released one machine first, then the rest at once (ADR 0236, ADR 0353). - if _, err := releaseBacklog(ctx, open, ""); err != nil { - fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err) - failed = true - } - // Again while the plans that are open changed: a plan that ended lets a deferred one before it start, and a - // batch cut makes one. - for pass := 0; pass < planLookPasses; pass++ { - before := openPlansNow(ctx, open.inventory) - if advanceHeld(ctx, open) { - failed = true - } - if openPlansNow(ctx, open.inventory) == before { - break - } - } - if err := syncSilentKeys(ctx, open); err != nil { - fmt.Printf("plans: the wait for the machines' reports could not be kept on the bus: %v\n", err) - failed = true // so the look is retried - } - }() - keepWalkPhases(ctx, open) - return failed -} - -// advancePlans is one pass over the open plans taking the plans without waiting for them, for a caller that wants a -// single step rather than the look (the tests of the steps). -func advancePlans(ctx context.Context, open *stores) { - release, err := open.inventory.HoldPlans(ctx, false) - if err != nil { - if !errors.Is(err, inventory.ErrPlansBusy) { - fmt.Printf("plans: cannot hold them: %v\n", err) - } - return - } - defer release() - if _, err := releaseBacklog(ctx, open, ""); err != nil { - fmt.Printf("plans: what waits for a gate could not be looked at: %v\n", err) - } - advanceHeld(ctx, open) -} - -// openPlansNow is which plans are open and in what state, for telling a pass that changed them from one that did not. -func openPlansNow(ctx context.Context, inv *inventory.Inventory) string { - plans, err := inv.OpenPlans(ctx) - if err != nil { - return "unreadable " + time.Now().String() - } - ids := make([]string, 0, len(plans)) - for _, p := range plans { - ids = append(ids, p.ID+":"+string(p.State)) - } - sort.Strings(ids) - return strings.Join(ids, ",") -} - -// --- the wait for a machine's report, as keys on the bus --- - -var ( - silentMu sync.RWMutex - silentKeyed silentClock // nil in a process that is not the serving controller -) - -// setSilentClock makes clock the rollout's; nil takes it away. -func setSilentClock(c silentClock) { - silentMu.Lock() - defer silentMu.Unlock() - silentKeyed = c -} - -func silentClockOf() silentClock { - silentMu.RLock() - defer silentMu.RUnlock() - return silentKeyed -} - -// silentWaitsOf is every machine an open plan waits on for a report, with when it was sent: the machines a walk sent -// that have not reported on the send and are not silent yet (walkReports), the machine a gate of a first machine -// judges, and the machines a release's gates wait on. Pure. -func silentWaitsOf(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) map[silentWait]time.Time { - waits := map[silentWait]time.Time{} - put := func(w silentWait, at time.Time) { - if at.IsZero() { - return - } - if was, ok := waits[w]; !ok || at.Before(was) { - waits[w] = at - } - } - if !p.Open() { - return waits - } - if r := p.Release; r != nil { - gates := []*inventory.PlanGate{r.Gate} - for _, g := range r.Gates { - gates = append(gates, g) - } - for _, g := range gates { - if g == nil || g.Verdict != "" || g.Silent || g.Pending || g.Since == nil { - continue - } - for _, node := range g.Machines { - put(silentWait{Plan: p.ID, Tier: 0, Node: node}, *g.Since) - } - } - return waits - } - if p.Waiting() { - return waits - } - tierOf := func(module string) int { - for i, tier := range p.Tiers { - if slices.Contains(tier, module) { - return i - } - } - return 0 - } - waiting, _, _ := walkReports(p, reports, now, prov) - for module, s := range p.Modules { - if s == nil { - continue - } - for _, node := range waiting { - sent, ok := s.Rest[node] - if !ok { - continue - } - at := sent.At - if at.IsZero() && s.SentAt != nil { - at = *s.SentAt - } - put(silentWait{Plan: p.ID, Tier: tierOf(module), Node: node}, at) - } - if g := s.Gate; g != nil && g.Verdict == "" && !g.Silent && g.Since != nil && s.FirstAt != nil { - for _, node := range g.Machines { - put(silentWait{Plan: p.ID, Tier: tierOf(module), Node: node}, *g.Since) - } - } - } - return waits -} - -// silentTTL is how long a wait's key lives: what is left of ApplySilentAfter from the send, and a second over, so the -// look it wakes finds the machine silent and not a fraction early. -func silentTTL(sent, now time.Time) time.Duration { - left := sent.Add(applySilentAfter).Sub(now) + time.Second - if left < time.Second { - return time.Second - } - return left -} - -// silentDiff is what the keys on the bus need to be as the waits are: the waits to put a key for (none yet, or the -// key is of an earlier send) and the keys to delete (the wait is over). Pure. -func silentDiff(want, held map[silentWait]time.Time) (arm, drop []silentWait) { - for w, sent := range want { - if was, ok := held[w]; !ok || !was.Equal(sent) { - arm = append(arm, w) - } - } - for w := range held { - if _, ok := want[w]; !ok { - drop = append(drop, w) - } - } - return arm, drop -} - -// syncSilentKeys keeps the keys on the bus as the open plans' waits are: a wait without a key gets one that lives out -// its time, a send made again gets its key again, and a key whose wait is over — the machine reported on the send, the -// gate has its verdict, the plan ended — is deleted. A report deletes the key by way of the look it asks for (the -// look alone knows the report was on that send). Only the lease holder looks, so only it writes. -func syncSilentKeys(ctx context.Context, open *stores) error { - c := silentClockOf() - inv := open.inventory - plans, err := inv.OpenPlans(ctx) - if err != nil { - return err - } - reports, err := inv.LastReports(ctx) - if err != nil { - return err - } - now := time.Now().UTC() - want := map[silentWait]time.Time{} - for _, p := range plans { - for w, at := range silentWaitsOf(p, reports, now, provLookup(ctx)) { - want[w] = at - } - } - return keepSilentKeys(ctx, c, want, now) -} - -// keepSilentKeys puts the keys on the bus as the waits are. Where the bus will not take a key — no bucket, the keys -// cannot be read, one cannot be put — the refusal is raised as a condition naming the plan and the machine, never -// stood in for by a timer, and the error is given back so the look is retried and writes the key again (novox/hq ADR -// 0433 §4). The set of refusals is what this look found: a key written at the next look is no longer in it. -func keepSilentKeys(ctx context.Context, c silentClock, want map[silentWait]time.Time, now time.Time) error { - refused := func(err error) map[silentWait]error { - out := map[silentWait]error{} - for w := range want { - out[w] = err - } - return out - } - if c == nil { - noteRefusedSendKeys(refused(errors.New("the bus's clock for the waits is not there"))) - return nil - } - c = boundedClock{c} - held, err := c.Held(ctx) - if err != nil { - noteRefusedSendKeys(refused(err)) - return err - } - arm, drop := silentDiff(want, held) - var first error - notTaken := map[silentWait]error{} - for _, w := range arm { - if err := c.Arm(ctx, w, silentTTL(want[w], now), want[w]); err != nil { - notTaken[w] = err - if first == nil { - first = err - } - } - } - noteRefusedSendKeys(notTaken) - for _, w := range drop { - if err := c.Reported(ctx, w); err != nil && first == nil { - first = err - } - } - return first -} - -// silentCallBound is how long one call to the silent clock may take while the look holds the plans (novox/hq issue 1121). -// The vendored key listing reads its watcher without the context, so a consumer lost when the broker reloaded its user -// list left the look waiting for good, with the plans and every node held. -var silentCallBound = 20 * time.Second - -// boundedClock is a clock whose every call is answered within silentCallBound, or fails: a call that never returns is -// left behind on its own goroutine. -type boundedClock struct{ silentClock } - -func boundedCall[T any](ctx context.Context, what string, call func(context.Context) (T, error)) (T, error) { - ctx, cancel := context.WithTimeout(ctx, silentCallBound) - defer cancel() - type answer struct { - v T - err error - } - done := make(chan answer, 1) - go func() { v, err := call(ctx); done <- answer{v, err} }() - select { - case a := <-done: - return a.v, a.err - case <-ctx.Done(): - var zero T - return zero, fmt.Errorf("%s: no answer from the bus within %s: %w", what, silentCallBound, ctx.Err()) - } -} - -func (b boundedClock) Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error { - _, err := boundedCall(ctx, "arming a wait", func(ctx context.Context) (struct{}, error) { - return struct{}{}, b.silentClock.Arm(ctx, w, ttl, sent) - }) - return err -} - -func (b boundedClock) Held(ctx context.Context) (map[silentWait]time.Time, error) { - return boundedCall(ctx, "listing the waits", b.silentClock.Held) -} - -func (b boundedClock) Reported(ctx context.Context, w silentWait) error { - _, err := boundedCall(ctx, "dropping a wait", func(ctx context.Context) (struct{}, error) { - return struct{}{}, b.silentClock.Reported(ctx, w) - }) - return err -} - -// startRolloutSilent puts the wait for the machines' reports on the bus for as long as the context lives: the -// controller is the lease holder here, the only one that writes it. The watcher asks for a look when a key expires. -func startRolloutSilent(ctx context.Context, js *broker.JetStream) error { - api, err := jetstream.New(js.Conn()) - if err != nil { - return err - } - clock, err := newBusSilent(ctx, api, func(silentWait) { askToLook() }) - if err != nil { - return err - } - setSilentClock(clock) - go func() { - <-ctx.Done() - silentMu.Lock() - if silentKeyed == silentClock(clock) { - silentKeyed = nil - } - silentMu.Unlock() - }() - return nil -} - -// startProvAppliedLooks has the plans looked at when a machine's provisioner client says a word (`prov.applied..`): -// a send that handed modules to the provisioner is applied only once the client said so too, and without this the walk -// would wait for its silent key (ADR 0362, ADR 0406 §5, issue 974). New words only: the first look reads the rest. -func startProvAppliedLooks(ctx context.Context, js *broker.JetStream) error { - api, err := jetstream.New(js.Conn()) - if err != nil { - return err - } - stream, err := api.Stream(ctx, broker.ProvAppliedStream) - if err != nil { - return fmt.Errorf("the provisioner clients' words: %w", err) - } - consumer, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{DeliverPolicy: jetstream.DeliverNewPolicy}) - if err != nil { - return fmt.Errorf("watching the provisioner clients' words: %w", err) - } - consuming, err := consumer.Consume(func(jetstream.Msg) { askToLook() }) - if err != nil { - return fmt.Errorf("watching the provisioner clients' words: %w", err) - } - go func() { - <-ctx.Done() - consuming.Stop() - }() - return nil -} - -// startPlanMovedLooks has the plans looked at when any process says one moved: the serving controller follows its own -// `plan-moved`, so a plan a person moved with a verb at the control node's terminal — a process this one did not spawn, -// whose saves it does not hear — is looked at all the same (novox/hq ADR 0406 §2, §5, issue 974). New messages only; the -// first look reads the rest. A look its own save asks is the same look: asks are one. -func startPlanMovedLooks(ctx context.Context, js *broker.JetStream, open *stores) error { - api, err := jetstream.New(js.Conn()) - if err != nil { - return err - } - stream, err := api.Stream(ctx, broker.EventsStream) - if err != nil { - return fmt.Errorf("the events stream: %w", err) - } - consumer, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{ - FilterSubjects: []string{link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved)}, - DeliverPolicy: jetstream.DeliverNewPolicy}) - if err != nil { - return fmt.Errorf("following the plans moved: %w", err) - } - var bus link.Bus - if core, err := js.Conn().JetStream(); err == nil { - bus = link.OverNATS{Conn: js.Conn(), JS: core} - } - consuming, err := consumer.Consume(func(m jetstream.Msg) { - askToLook() - if bus != nil && open != nil { - // A walk's start or end, said by this controller or a verb: said again with the machines whose - // declaration it changes (novox/hq ADR 0505 §2, walk_declared.go). - go sayDeclared(context.WithoutCancel(ctx), bus, open, m.Data()) - } - }) - if err != nil { - return fmt.Errorf("following the plans moved: %w", err) - } - go func() { - <-ctx.Done() - consuming.Stop() - }() - return nil -} - -// --- the end of a walk whose end is not known --- - -var walkEnds = struct { - mu sync.Mutex - timers map[string]*time.Timer -}{timers: map[string]*time.Timer{}} - -// armWalkEnd has the walk recorded once its end can be known: a walk that ended with a machine silent is recorded -// without its total ApplySilentAfter and a minute after its last save (recordWalkPhases), which no event says. Armed at -// each save of the done walk, so it counts from the last. -func armWalkEnd(id string, savedAt time.Time) { - d := time.Until(savedAt.Add(inventory.ApplySilentAfter + time.Minute + time.Second)) - if d <= 0 { - return - } - walkEnds.mu.Lock() - defer walkEnds.mu.Unlock() - if t := walkEnds.timers[id]; t != nil { - t.Stop() - } - // retries: recording the end of a walk whose end is not known. One per walk, once, from its last save; the first - // look after the lease was taken arms it again for the walks still unrecorded. - walkEnds.timers[id] = time.AfterFunc(d, func() { - walkEnds.mu.Lock() - delete(walkEnds.timers, id) - walkEnds.mu.Unlock() - askToLook() - }) -} - -// rearmWalkEnds is armWalkEnd for the walks that ended lately, for a controller that has just taken the lease. -func rearmWalkEnds(ctx context.Context, open *stores) { - recent, err := open.inventory.RecentPlans(ctx, 30) - if err != nil { - fmt.Printf("plans: the walks ended lately could not be read to record their phases: %v\n", err) - return - } - for _, p := range recent { - if p.State == inventory.PlanDone && p.Release == nil && time.Since(p.Updated) <= 2*time.Hour { - armWalkEnd(p.ID, p.Updated) - } - } -} - -// planKept is what the serving controller does when a plan was saved: a look is asked for, so a send, a built tier or -// a command's move is followed by the plan's next step; and a walk that ended is armed to be recorded. -func planKept(p inventory.Plan) { - if p.State == inventory.PlanDone && p.Release == nil { - armWalkEnd(p.ID, time.Now()) - } - askToLook() -} - -// --- `sent`: a tier sent to a machine, said --- - -// sendMade is one machine a plan's tier was sent to. -type sendMade struct { - Tier int - Node string - Modules []string - At time.Time -} - -// sendsMade collects the sends one step of a plan made, to be said once the plan is saved. -type sendsMade struct { - mu sync.Mutex - list []sendMade -} - -type sendsMadeKey struct{} - -func withSendsMade(ctx context.Context, s *sendsMade) context.Context { - return context.WithValue(ctx, sendsMadeKey{}, s) -} - -// noteSends records that modules of the plan's tier were sent to the nodes. -func noteSends(ctx context.Context, tier int, nodes, modules []string, at time.Time) { - s, _ := ctx.Value(sendsMadeKey{}).(*sendsMade) - if s == nil { - return - } - s.mu.Lock() - defer s.mu.Unlock() - for _, n := range nodes { - s.list = append(s.list, sendMade{Tier: tier, Node: n, Modules: slices.Clone(modules), At: at}) - } -} - -// planSentBody is the body of the seat event `sent` (novox/hq ADR 0406 §5, issue 974): the plan, the tier, the machine -// and the modules it was sent, and when. -type planSentBody struct { - Event string `json:"event"` - Plan string `json:"plan"` - Tier int `json:"tier"` - Node string `json:"node"` - Modules []string `json:"modules"` - At time.Time `json:"at"` -} - -// sentBodies are the events a step's sends are said as. -func sentBodies(plan string, sends *sendsMade) []planSentBody { - sends.mu.Lock() - defer sends.mu.Unlock() - out := make([]planSentBody, 0, len(sends.list)) - for _, s := range sends.list { - out = append(out, planSentBody{Event: link.KeySent, Plan: plan, Tier: s.Tier, Node: s.Node, - Modules: s.Modules, At: s.At}) - } - return out -} - -// saySends says each send a step made as `sent`, once the plan that records it is saved. What an outsider hears; the -// controller's own next step does not wait for it. -func saySends(ctx context.Context, p inventory.Plan, sends *sendsMade) { - bus := checkEvents - if bus == nil { - return - } - for _, body := range sentBodies(p.ID, sends) { - raw, err := json.Marshal(body) - if err != nil { - continue - } - go func() { - stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) - defer stop() - if err := bus.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeySent, raw); err != nil { - fmt.Printf("%s: tier %d was sent to %s, and could not be said as sent: %v\n", body.Plan, body.Tier, body.Node, err) - } - }() - } -} diff --git a/cmd/mesh-controller/plan_looks_integration_test.go b/cmd/mesh-controller/plan_looks_integration_test.go deleted file mode 100644 index cf2f75fb..00000000 --- a/cmd/mesh-controller/plan_looks_integration_test.go +++ /dev/null @@ -1,188 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "sync/atomic" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -func TestAsksMadeWhileALookRunsAreOneLookAfterIt(t *testing.T) { - var failures, looks atomic.Int32 - block := make(chan struct{}) - l := lookerCounting(&failures, &looks, block) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - l.ask() - eventually(t, "the first look starts", func() bool { return looks.Load() == 1 }) - for i := 0; i < 20; i++ { - l.ask() - } - block <- struct{}{} // the first look ends - eventually(t, "the asks made meanwhile are looked at", func() bool { return looks.Load() == 2 }) - block <- struct{}{} - time.Sleep(60 * time.Millisecond) - if n := looks.Load(); n != 2 { - t.Errorf("%d looks for 21 asks; want 2", n) - } -} - -func TestAFailedLookIsLookedAtAgainAndAGoodOneEndsTheRetries(t *testing.T) { - var failures, looks atomic.Int32 - failures.Store(3) // three looks fail, then one is good - l := lookerCounting(&failures, &looks, nil) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - l.ask() - eventually(t, "four looks", func() bool { return looks.Load() == 4 }) - time.Sleep(200 * time.Millisecond) - if n := looks.Load(); n != 4 { - t.Errorf("%d looks; the good one should have ended the retries at 4", n) - } - l.mu.Lock() - armed, backoff := l.retry != nil, l.backoff - l.mu.Unlock() - if armed || backoff != 0 { - t.Errorf("a retry is armed (%v) or the wait kept (%v) after a good look", armed, backoff) - } -} - -func TestAPlanSavedAsksForALookAndADoneWalkIsArmedToBeRecorded(t *testing.T) { - var failures, looks atomic.Int32 - l := lookerCounting(&failures, &looks, nil) - lookerMu.Lock() - theLooker = l - lookerMu.Unlock() - t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - - planKept(inventory.Plan{ID: "walk-1", State: inventory.PlanDone}) - t.Cleanup(func() { - walkEnds.mu.Lock() - defer walkEnds.mu.Unlock() - if t := walkEnds.timers["walk-1"]; t != nil { - t.Stop() - } - delete(walkEnds.timers, "walk-1") - }) - eventually(t, "a look", func() bool { return looks.Load() == 1 }) - walkEnds.mu.Lock() - _, armed := walkEnds.timers["walk-1"] - walkEnds.mu.Unlock() - if !armed { - t.Error("a walk saved done is not armed to be recorded") - } - askToLook() // no panic, no looker, once it is taken away -} - -func TestAnExpiredSilentKeyAsksForALookAndAReportsDeleteDoesNot(t *testing.T) { - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - var failures, looks atomic.Int32 - l := lookerCounting(&failures, &looks, nil) - lookerMu.Lock() - theLooker = l - lookerMu.Unlock() - t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock(); setSilentClock(nil) }) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - if err := startRolloutSilent(ctx, js); err != nil { - t.Fatal(err) - } - c := silentClockOf() - w := silentWait{Plan: "plan-5", Tier: 0, Node: "ace"} - if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil { - t.Fatal(err) - } - if err := c.Reported(ctx, w); err != nil { - t.Fatal(err) - } - time.Sleep(1800 * time.Millisecond) - if n := looks.Load(); n != 0 { - t.Fatalf("a key a report deleted asked for %d looks", n) - } - if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil { - t.Fatal(err) - } - eventually(t, "the expiry asks for a look", func() bool { return looks.Load() == 1 }) -} - -func TestAProvisionerClientsWordAsksForALook(t *testing.T) { - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - var failures, looks atomic.Int32 - l := lookerCounting(&failures, &looks, nil) - lookerMu.Lock() - theLooker = l - lookerMu.Unlock() - t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - if err := startProvAppliedLooks(ctx, js); err != nil { - t.Fatal(err) - } - legacy, err := js.Conn().JetStream() - if err != nil { - t.Fatal(err) - } - if _, err := legacy.Publish(broker.ProvisionerAppliedSubject("ace", "plex"), []byte(`{"status":"applied"}`)); err != nil { - t.Fatal(err) - } - eventually(t, "a look on the client's word", func() bool { return looks.Load() >= 1 }) -} - -func TestAPlanMovedByAnotherProcessAsksForALook(t *testing.T) { - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - var failures, looks atomic.Int32 - l := lookerCounting(&failures, &looks, nil) - lookerMu.Lock() - theLooker = l - lookerMu.Unlock() - t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - if err := startPlanMovedLooks(ctx, js, nil); err != nil { - t.Fatal(err) - } - legacy, err := js.Conn().JetStream() - if err != nil { - t.Fatal(err) - } - if _, err := legacy.Publish(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), []byte(`{}`)); err != nil { - t.Fatal(err) - } - eventually(t, "a look on the plan moved", func() bool { return looks.Load() >= 1 }) -} diff --git a/cmd/mesh-controller/plan_looks_test.go b/cmd/mesh-controller/plan_looks_test.go deleted file mode 100644 index 0d62b8fb..00000000 --- a/cmd/mesh-controller/plan_looks_test.go +++ /dev/null @@ -1,265 +0,0 @@ -package main - -import ( - "context" - "errors" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -func rolloutOf(sentAgo time.Duration, reported bool) (inventory.Plan, []inventory.Reported) { - sent := time.Now().Add(-sentAgo).UTC() - p := inventory.Plan{ID: "plan-9", State: inventory.PlanRolling, Tiers: [][]string{{"a", "b"}}, Tier: 0, - Modules: map[string]*inventory.PlanModule{ - "a": {SentAt: &sent, Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d1", At: sent}}}, - "b": {SentAt: &sent, Rest: map[string]inventory.SentDeclaration{"ace": {Digest: "d1", At: sent}}}, - }} - r := inventory.Reported{Node: "ace", Sent: &sent} - if reported { - at := sent.Add(time.Minute) - r.At, r.Current, r.Declared, r.Outcome = &at, true, "d1", inventory.OutcomeApplied - } - return p, []inventory.Reported{r} -} - -// A machine a plan sent and heard nothing from is a wait with a key, from when it was sent (issue 974, novox/hq ADR -// 0406 §5); one that reported on the send, or that is silent already, is none. -func TestAPlanWaitsOnTheMachinesItSentAndHeardNothingFrom(t *testing.T) { - now := time.Now().UTC() - p, reports := rolloutOf(5*time.Minute, false) - waits := silentWaitsOf(p, reports, now, nil) - if len(waits) != 1 { - t.Fatalf("waits = %v; want the one machine", waits) - } - for w, at := range waits { - if w.Plan != "plan-9" || w.Node != "ace" || w.Tier != 0 || now.Sub(at) < 4*time.Minute || now.Sub(at) > 6*time.Minute { - t.Errorf("wait %+v sent at %v is wrong", w, at) - } - } - p, reports = rolloutOf(5*time.Minute, true) - if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 { - t.Errorf("a machine that reported on its send is waited on: %v", waits) - } - p, reports = rolloutOf(inventory.ApplySilentAfter+time.Minute, false) - if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 { - t.Errorf("a machine silent past its time is waited on again: %v", waits) - } - p, reports = rolloutOf(time.Minute, false) - p.State = inventory.PlanDone - if waits := silentWaitsOf(p, reports, now, nil); len(waits) != 0 { - t.Errorf("a plan that ended is waiting: %v", waits) - } -} - -// A release's open gate waits on its machines from when it began judging, and its verdict ends the wait. -func TestAReleasesOpenGateWaitsOnItsMachines(t *testing.T) { - since := time.Now().Add(-time.Minute).UTC() - g := &inventory.PlanGate{Machines: []string{"g14", "novox"}, Since: &since} - p := inventory.Plan{ID: "release-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: g}} - waits := silentWaitsOf(p, nil, time.Now(), nil) - if len(waits) != 2 { - t.Fatalf("waits = %v; want g14 and novox", waits) - } - g.Verdict = inventory.GatePassed - if waits := silentWaitsOf(p, nil, time.Now(), nil); len(waits) != 0 { - t.Errorf("a gate with its verdict still waits: %v", waits) - } -} - -// The keys follow the waits: a wait with no key, or a key of an earlier send, is armed; a key with no wait is deleted. -func TestTheKeysFollowTheWaits(t *testing.T) { - t1 := time.Now().UTC() - t2 := t1.Add(time.Minute) - a, b, c := silentWait{"p", 0, "a"}, silentWait{"p", 0, "b"}, silentWait{"p", 0, "c"} - arm, drop := silentDiff(map[silentWait]time.Time{a: t1, b: t2, c: t1}, map[silentWait]time.Time{a: t1, b: t1, silentWait{"p", 1, "a"}: t1}) - if len(arm) != 2 || !(arm[0] == b || arm[1] == b) || !(arm[0] == c || arm[1] == c) { - t.Errorf("armed %v; want b (sent again) and c (no key)", arm) - } - if len(drop) != 1 || drop[0] != (silentWait{"p", 1, "a"}) { - t.Errorf("dropped %v; want the key with no wait", drop) - } - if ttl := silentTTL(t1, t1.Add(inventory.ApplySilentAfter+time.Hour)); ttl != time.Second { - t.Errorf("a wait long over has a key of %v; want the least, one second", ttl) - } -} - -func lookerCounting(failures *atomic.Int32, looks *atomic.Int32, block chan struct{}) *looker { - l := newLooker(func(ctx context.Context) bool { - looks.Add(1) - if block != nil { - <-block - } - return failures.Add(-1) >= 0 - }) - l.gap, l.base, l.most = 0, 20*time.Millisecond, 80*time.Millisecond - return l -} - -func eventually(t *testing.T, what string, ok func() bool) { - t.Helper() - for end := time.Now().Add(15 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) { - if ok() { - return - } - } - t.Fatalf("never: %s", what) -} - -// Asks made while a look runs are not lost and are one look after it, not one each. - -// A look that failed a step is looked at again, one at a time with a longer wait each time, and a look that fails -// nothing ends the retries (novox/hq ADR 0406 §5; the one named retry). - -func TestTheRetryWaitDoublesToItsMost(t *testing.T) { - l := newLooker(func(context.Context) bool { return true }) - l.base, l.most = 5*time.Second, time.Minute - var seen []time.Duration - for i := 0; i < 6; i++ { - l.after(true) - seen = append(seen, l.backoff) - l.retry.Stop() - l.retry = nil - } - want := []time.Duration{5 * time.Second, 10 * time.Second, 20 * time.Second, 40 * time.Second, time.Minute, time.Minute} - for i := range want { - if seen[i] != want[i] { - t.Fatalf("waits %v; want %v", seen, want) - } - } -} - -// A plan saved asks for a look, and a walk saved done is armed to be recorded once its end can be known. - -// The tier-sends a step made are said once each, per machine, with the modules it was sent. -func TestTheSendsAStepMadeAreSaidPerMachine(t *testing.T) { - sends := &sendsMade{} - ctx := withSendsMade(context.Background(), sends) - at := time.Now().UTC() - noteSends(ctx, 2, []string{"ace"}, []string{"a", "b"}, at) - noteSends(ctx, 2, []string{"g14"}, []string{"a"}, at) - noteSends(context.Background(), 2, []string{"nobody"}, []string{"a"}, at) // outside a step: nothing kept - got := sentBodies("plan-9", sends) - if len(got) != 2 || got[0].Node != "ace" || len(got[0].Modules) != 2 || got[0].Tier != 2 || got[0].Plan != "plan-9" || - got[0].Event != "sent" || !got[0].At.Equal(at) || got[1].Node != "g14" { - t.Errorf("sent bodies = %+v", got) - } -} - -// A silent key that expires asks for a look; a key a report deleted does not (issue 974, novox/hq ADR 0406 §5). - -// A provisioner client's word on a module's files asks for a look, as a send that handed it modules is applied only -// once it has said so (issue 974, review 1): no walk waits for its silent key for what the client already said. - -type brokenClock struct{ silentClock } - -func (brokenClock) Held(context.Context) (map[silentWait]time.Time, error) { - return nil, errors.New("the bus is down") -} - -// 1157/AC1: a bus that will not say which keys it holds, or will not take one, does not leave the waits unwatched by -// nothing: the refusal raises a condition naming the plan and the machine, no timer stands in for the key, and the error -// is given back so the look is marked failed and retried (issue 974, review 1; ADR 0433 §4). -func TestARefusedSendKeyRaisesAConditionNamingThePlanAndTheMachine(t *testing.T) { // 1157/AC1 - w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"} - sent := time.Now().UTC() - for name, clock := range map[string]silentClock{"keys cannot be read": brokenClock{}, "a key cannot be put": armRefusingClock{}, - "no clock": nil} { - t.Run(name, func(t *testing.T) { - noteRefusedSendKeys(nil) - t.Cleanup(func() { noteRefusedSendKeys(nil) }) - err := keepSilentKeys(t.Context(), clock, map[silentWait]time.Time{w: sent}, sent) - if clock != nil && err == nil { - t.Fatal("no error given back") - } - got := observeRefusedSendKeys() - if len(got) != 1 { - t.Fatalf("observations %+v", got) - } - o := got[0] - if o.Scope != conditions.ScopePlan || o.ID != "plan-held" || o.Machine != "ace" || o.Kind != kindSendKeyRefused || - !strings.Contains(o.Summary, "plan-held") || !strings.Contains(o.Summary, "ace") { - t.Fatalf("the condition does not name the plan and the machine: %+v", o) - } - }) - } -} - -// 1157/AC2: at the plan's next look the send key is written again, and the condition clears once it is. -func TestTheNextLookWritesTheRefusedSendKeyAndTheConditionClears(t *testing.T) { // 1157/AC2 - w := silentWait{Plan: "plan-held", Tier: 0, Node: "ace"} - sent := time.Now().UTC() - noteRefusedSendKeys(nil) - t.Cleanup(func() { noteRefusedSendKeys(nil) }) - want := map[silentWait]time.Time{w: sent} - if err := keepSilentKeys(t.Context(), armRefusingClock{}, want, sent); err == nil || len(observeRefusedSendKeys()) != 1 { - t.Fatalf("the refusal was not raised: %v %+v", err, observeRefusedSendKeys()) - } - clock := &keptClock{} - if err := keepSilentKeys(t.Context(), clock, want, sent); err != nil { - t.Fatal(err) - } - if !clock.armed[w].Equal(sent) { - t.Fatalf("the next look did not write the key again: %v", clock.armed) - } - if got := observeRefusedSendKeys(); len(got) != 0 { - t.Fatalf("the condition stays after the key was written: %+v", got) - } -} - -type armRefusingClock struct{ silentClock } - -func (armRefusingClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, nil } -func (armRefusingClock) Arm(context.Context, silentWait, time.Duration, time.Time) error { - return errors.New("the bus refuses the key") -} - -// keptClock is a clock that takes every key. -type keptClock struct { - silentClock - armed map[silentWait]time.Time -} - -func (c *keptClock) Held(context.Context) (map[silentWait]time.Time, error) { return nil, nil } -func (c *keptClock) Arm(_ context.Context, w silentWait, _ time.Duration, sent time.Time) error { - if c.armed == nil { - c.armed = map[silentWait]time.Time{} - } - c.armed[w] = sent - return nil -} - -// A plan moved by a process the serving controller did not spawn — a verb run at the control node's terminal — is -// looked at: the controller follows its own plan-moved (issue 974, round 1). - -// A clock whose bus answers nothing, ever: a consumer lost when the broker reloaded its user list, a key listing that -// never reaches its end. -type hungClock struct{ silentClock } - -func (hungClock) Held(context.Context) (map[silentWait]time.Time, error) { - select {} // never answers, whatever the context says (the vendored Keys reads its watcher without it) -} - -// issue 1121: the look holds the plans while it keeps the silent keys, so a key listing that never ends held the plans -// and every node for good. Every call to the clock has a bound, and the refusal is raised as a condition. -func TestAClockThatNeverAnswersDoesNotHoldTheLookForever(t *testing.T) { - was := silentCallBound - silentCallBound = 50 * time.Millisecond - t.Cleanup(func() { silentCallBound = was }) - w := silentWait{Plan: "plan-hung", Tier: 0, Node: "ace"} - sent := time.Now().UTC() - done := make(chan error, 1) - go func() { done <- keepSilentKeys(t.Context(), hungClock{}, map[silentWait]time.Time{w: sent}, sent) }() - select { - case err := <-done: - if err == nil { - t.Fatal("a clock that never answered was reported as fine") - } - case <-time.After(5 * time.Second): - t.Fatal("the look is still waiting for a clock that never answers") - } -} diff --git a/cmd/mesh-controller/plan_moved_every_writer_integration_test.go b/cmd/mesh-controller/plan_moved_every_writer_integration_test.go deleted file mode 100644 index 2d376641..00000000 --- a/cmd/mesh-controller/plan_moved_every_writer_integration_test.go +++ /dev/null @@ -1,37 +0,0 @@ -//go:build integration - -package main - -import ( - "testing" - "time" -) - -func TestEveryMoveAMergeAndTheCutMakeOfAPlanIsSaid(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - heard := sayingEverySave(t) - hear(t, open, catalogueMerge("553b7191claude", "app", t0), t0.Add(time.Second)) - if _, bs := walks(t, open); len(bs) != 1 { - t.Fatalf("%d batch(es) opened", len(bs)) - } - heard.noUnsaid(t, open) - cutAt(t, open, t0.Add(91*time.Second)) - if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 { - t.Fatalf("%d walk(s), %d batch(es) after the cut", len(ws), len(bs)) - } - heard.noUnsaid(t, open) - counted := 0 - for _, r := range heard.revs { - counted += len(r) - } - cutAt(t, open, t0.Add(92*time.Second)) - again := 0 - for _, r := range heard.revs { - again += len(r) - } - if again != counted { - t.Errorf("a look that changed nothing said %d more move(s)", again-counted) - } -} diff --git a/cmd/mesh-controller/plan_retry.go b/cmd/mesh-controller/plan_retry.go deleted file mode 100644 index f7bf2a41..00000000 --- a/cmd/mesh-controller/plan_retry.go +++ /dev/null @@ -1,651 +0,0 @@ -package main - -import ( - "context" - "fmt" - "slices" - "sort" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// A plan follows what is done to the build queue (novox/hq ADR 0219). -// -// Two things a person does to builds by hand would otherwise leave a plan saying something untrue: -// -// - **Pausing the build seat.** A plan whose builds wait in the queue of a seat whose every holder -// is paused is not late — nothing will take its asks until somebody resumes — and saying LATE -// sends a reader looking for a fault that is a decision. It says what it waits on instead. -// - **A build that failed, been cancelled or killed, and is asked again.** `plans retry` re-asks a -// failed plan's failed modules and the plan goes on from that tier as if they had built the -// first time; `rebuild` of a module a plan holds unbuilt joins that plan rather than running -// beside it — beside it, the plan would either ask it again or stay failed on an outcome the -// rebuild has already replaced. - -// failedIn is the modules of a plan's current tier that failed to build, sorted. -func failedIn(p inventory.Plan) []string { - if p.Tier >= len(p.Tiers) { - return nil - } - var out []string - for _, m := range p.Tiers[p.Tier] { - if s := p.Modules[m]; s != nil && s.State == "failed" { - out = append(out, m) - } - } - sort.Strings(out) - return out -} - -// newerOpenPlan is an open plan of the same repository and branch made after this one: the plan -// that holds what this one held now (issue 254, ADR 0218). -func newerOpenPlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) { - for _, q := range plans { - if q.ID == p.ID || !q.Open() || !strings.EqualFold(q.Repository, p.Repository) || - (p.Branch != "" && q.Branch != "" && p.Branch != q.Branch) || !q.Created.After(p.Created) { - continue - } - return q, true - } - return inventory.Plan{}, false -} - -// newerDonePlan is a done plan of the same repository and branch made after this one: the newer commit it delivered -// runs now (novox/hq ADR 0451). -func newerDonePlan(p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) { - var found inventory.Plan - var ok bool - for _, q := range plans { - if q.ID == p.ID || q.State != inventory.PlanDone || !strings.EqualFold(q.Repository, p.Repository) || - (p.Branch != "" && q.Branch != "" && p.Branch != q.Branch) || !q.Created.After(p.Created) { - continue - } - if !ok || q.Created.After(found.Created) { - found, ok = q, true - } - } - return found, ok -} - -// passedBy is the refusal of a retry of a plan that a newer done plan of its repository and branch has passed. -type passedBy struct{ plan, by, repository, commit string } - -func (e passedBy) Error() string { - return fmt.Sprintf("%s was passed by %s, a newer plan of %s (at %s) that is done: retrying it would build and "+ - "send the older commit over the newer one that runs", e.plan, e.by, e.repository, short(e.commit)) -} - -// retryRefusal is why a plan cannot be retried, or nothing. -func retryRefusal(p inventory.Plan, plans []inventory.Plan, onPath map[string]string) error { - switch { - case p.State == inventory.PlanDone: - return fmt.Errorf("%s is done; there is nothing to retry", p.ID) - case p.State == inventory.PlanSuperseded: - return fmt.Errorf("%s was %s — what it had not built is in that plan", p.ID, p.Note) - case p.Open(): - return fmt.Errorf("%s is still %s; nothing in it failed to retry — `rebuild ` asks one module again", p.ID, p.State) - } - - // **A newer delivery of the branch has passed it** (novox/hq ADR 0451): retrying would build and send the older - // commit over the newer one that runs. - if q, found := newerDonePlan(p, plans); found { - return passedBy{plan: p.ID, by: q.ID, repository: q.Repository, commit: q.Commit} - } - if len(failedIn(p)) > 0 { - if q, found := newerOpenPlan(p, plans); found { - return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and retrying %s would build "+ - "what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID) - } - return oneWalkAtATime(p, plans, onPath) - } - stopped := stoppedRollouts(p) - // **A walk stopped by hand while its tier waited is resumed** (issue 1886): nothing in it failed, so nothing is - // asked again; it goes on from where it stopped, unless a newer walk holds what it has yet to send. - if len(stopped) == 0 && stoppedByHandWaiting(p) { - if q, found := newerOpenPlan(p, plans); found { - return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and resuming %s would build "+ - "what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID) - } - for _, m := range holdingABuild(p) { - if q, found := newerPlanFor(m, p, plans); found { - return fmt.Errorf("%s has a newer plan, %s (%s, %s at %s): resuming %s would send the older build of it "+ - "over the newer one", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID) - } - } - return oneWalkAtATime(p, plans, onPath) - } - // **A walk lost to silence is resumed** (novox/hq ADR 0531, issue 1886): its tier was sent whole and only the gate after it failed it, every - // machine it waited on silent; once they reported, it goes on from that gate. - if len(stopped) == 0 && lostToSilence(p) { - if q, found := newerOpenPlan(p, plans); found { - return fmt.Errorf("%s supersedes it: a newer merge of %s (%s at %s) is open, and resuming %s would send "+ - "what that one replaced", q.ID, q.Repository, q.ID, short(q.Commit), p.ID) - } - return oneWalkAtATime(p, plans, onPath) - } - if len(stopped) == 0 { - return fmt.Errorf("nothing in tier %d of %s failed to build or stopped rolling out — it stopped at: %s", - p.Tier, p.ID, p.Note) - } - // **A build that failed its gate is not sent again** (novox/hq ADR 0236): it was put back on its first - // machine, and retrying would judge the build the mesh put back, or send the failed one by hand. - for _, m := range stopped { - if g := p.Modules[m].Gate; g != nil && g.Verdict == inventory.GateFailed && !noVerdictOnItsBuild(g) { - return fmt.Errorf("%s failed its gate on %s (%s) and was put back: a build that failed its gate is not "+ - "sent again — a newer merge, or `rebuild %s`, makes a new build, judged at the gate again", - m, strings.Join(g.Machines, ", "), g.Why, m) - } - } - // **A rollout is retried unless the module has moved on**: a newer plan holding it sends — or - // sent — a newer build, and sending this one again would put the older build back on its machines. - for _, m := range stopped { - if q, found := newerPlanFor(m, p, plans); found { - return fmt.Errorf("%s has a newer plan, %s (%s, %s at %s): sending %s's build of it again would put "+ - "the older build back", m, q.ID, q.State, q.Repository, short(q.Commit), p.ID) - } - } - return oneWalkAtATime(p, plans, onPath) -} - -// oneWalkAtATime refuses a retry while an open walk, started or waiting for its word, is related to it (novox/hq -// ADR 0276, ADR 0407 decision 4, issue 973): a walk retried beside a related one would be two walks on the same -// things at once. A walk related to none may be retried beside them. The retried walk is judged by the modules it -// plans, which already hold their dependents, and the repositories it carries. -func oneWalkAtATime(p inventory.Plan, plans []inventory.Plan, onPath map[string]string) error { - mine := footprintOfWalk(p, nil) - for _, q := range plans { - if q.ID == p.ID || !q.Open() || q.Release != nil { - continue - } - if ownWalk(p, onPath) || ownWalk(q, onPath) { - return fmt.Errorf("%s is open (%s) and one of them is on the controller's own path: one walk at a time — "+ - "retry %s once it ended", q.ID, q.Named(), p.ID) - } - if ok, why := related(mine, footprintOfWalk(q, nil), nil); ok { - return fmt.Errorf("%s is open (%s) and %s: one walk at a time beside a related one — retry %s once it ended", - q.ID, q.Named(), why, p.ID) - } - } - return nil -} - -// stoppedRollouts is the modules of a plan's current tier whose rollout stopped at its first machine -// (issue 249, ADR 0218): built, sent to the first machine, never to the rest, and why it stopped kept. -func stoppedRollouts(p inventory.Plan) []string { - if p.Tier >= len(p.Tiers) { - return nil - } - var out []string - for _, m := range p.Tiers[p.Tier] { - if s := p.Modules[m]; s != nil && s.State == "built" && s.FirstAt != nil && s.SentAt == nil && - len(s.First) > 0 && s.Why != "" { - out = append(out, m) - } - } - sort.Strings(out) - return out -} - -// newerPlanFor is a plan made after this one that holds the module, superseded ones aside. -func newerPlanFor(module string, p inventory.Plan, plans []inventory.Plan) (inventory.Plan, bool) { - for _, q := range plans { - if q.ID == p.ID || q.State == inventory.PlanSuperseded || !q.Created.After(p.Created) { - continue - } - for _, tier := range q.Tiers { - for _, m := range tier { - if m == module { - return q, true - } - } - } - } - return inventory.Plan{}, false -} - -// sendRollout sends machines what the mesh would send them now, answering the ones it sent. A -// variable so a test of a retried rollout needs no machine. -var sendRollout = sendToEach - -// resumed sets a failed plan building again once nothing in its tier is failed. -func resumed(p *inventory.Plan, why string) { - if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 { - p.State = inventory.PlanBuilding - p.Note = why - goesOnAgain(p) - } -} - -// goesOnAgain clears a stop from a walk that walks again: a stop said why the walk ended, and a walk resumed has -// not ended. Kept, it would end a retried walk that fails again as stopped, not failed (novox/hq ADR 0308 -// decision 1), and keep the search for the merge that brought the failure from running (searchAfterFailures). -func goesOnAgain(p *inventory.Plan) { - if p.Delivery != nil { - p.Delivery.Stopped, p.Delivery.StoppedWhy = "", "" - } -} - -// planReader is what a retry reads the mesh's plans through. -type planReader interface { - OpenPlans(ctx context.Context) ([]inventory.Plan, error) - RecentPlans(ctx context.Context, limit int) ([]inventory.Plan, error) - PlansSince(ctx context.Context, since time.Time) ([]inventory.Plan, error) -} - -// plansForRetry are the plans a retry of p judges itself against. -func plansForRetry(ctx context.Context, inv planReader, p inventory.Plan) ([]inventory.Plan, error) { - plans, err := inv.OpenPlans(ctx) - if err != nil { - return nil, err - } - recent, err := inv.RecentPlans(ctx, 50) - if err != nil { - return nil, err - } - // **Every plan made after this one** (novox/hq ADR 0451 §1): a plan that passed it is seen however many plans - // were made since, not only when it is among the newest. - later, err := inv.PlansSince(ctx, p.Created) - if err != nil { - return nil, err - } - return append(append(plans, recent...), later...), nil -} - -// retryPlan asks a failed plan's failed modules again, under new ids, and sets it building again at -// that tier: what follows is the plan going on as if they had built the first time. -// A walk stopped by hand while its tier waited, with nothing failed, is resumed instead: nothing asked again -// (resumeStopped, issue 1886). -func retryPlan(ctx context.Context, open *stores, id string) (string, error) { - inv := open.inventory - release, err := inv.HoldPlans(ctx, true) - if err != nil { - return "", err - } - defer release() - p, err := inv.PlanByID(ctx, id) - if err != nil { - return "", err - } - plans, err := plansForRetry(ctx, inv, p) - if err != nil { - return "", err - } - // **A failed walk whose earlier merges are walked alone is not retried** (novox/hq ADR 0276): the search for - // the merge that brought the failure answers them now, and a retried walk would name them twice. - // **A walk stopped without some merges is not retried** (novox/hq ADR 0299): the merges it left out were stopped, - // and the others are another walk's now. - if p.Delivery != nil && (p.Delivery.WalkedAgainBy != "" || len(p.Delivery.LeftOut) > 0) { - return "", fmt.Errorf("%s was stopped without %s: those merges were stopped, and %s walks the others; a "+ - "newer merge, or `rebuild `, builds again", p.ID, mergesLeftOutWords(p.Delivery.LeftOut), - orWords(p.Delivery.WalkedAgainBy, "nothing")) - } - if p.Delivery != nil && len(p.Delivery.Merges) > 0 { - kept, err := inv.MergesOf(ctx, p.ID) - if err != nil { - return "", err - } - if len(kept) < len(p.Delivery.Merges) { - return "", fmt.Errorf("%s's earlier merges are walked alone, to find which one brought its failure: "+ - "those walks answer them; a newer merge, or `rebuild `, builds again", p.ID) - } - } - // Settled from the build records before anything is judged (novox/hq issue 457): a build of the tier - // that failed after the plan did is as failed as the one that failed it. What toRetry says is the - // failed set every step below works from. - var failed []string - if p.Tier < len(p.Tiers) { - recorded, byID, err := recordsOfAsked(ctx, inv, &p, p.Tiers[p.Tier]) - if err != nil { - return "", err - } - failed = toRetry(&p, recorded, byID) - } - onPath, err := ownPathNow(ctx, inv) - if err != nil { - return "", err - } - if err := retryRefusal(p, plans, onPath); err != nil { - return "", err - } - if again := unjudgedAtGate(p); len(failed) == 0 && len(again) > 0 { - return retryTierWhole(ctx, open, &p, again) - } - if len(failed) == 0 && len(stoppedRollouts(p)) == 0 && stoppedByHandWaiting(p) { - said := resumeStopped(&p) - if err := inv.SavePlan(ctx, &p); err != nil { - return "", err - } - return said, nil - } - if len(failed) == 0 && lostToSilence(p) { - reports, err := inv.LastReports(ctx) - if err != nil { - return "", err - } - said, err := resumeLost(&p, silentNodes(p, reports, time.Now().UTC(), provLookup(ctx))) - if err != nil { - return "", err - } - if err := inv.SavePlan(ctx, &p); err != nil { - return "", err - } - return said, nil - } - if len(failed) == 0 { - return retryRollouts(ctx, open, &p) - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return "", err - } - byName := map[string]inventory.Entry{} - for _, e := range entries { - byName[e.Manifest.Module] = e - } - var asked []string - for _, m := range failed { - askModule(ctx, &p, m, byName) - if s := p.Modules[m]; s.State == "asked" && s.Key { - asked = append(asked, m+" from the builder's key") - } else if s.State == "asked" { - asked = append(asked, m+" as "+s.Build) - } - } - waiting := busWaitingOnRetry(p) - note, said := retryWords(p, failed, asked, waiting) - resumed(&p, note) - if err := inv.SavePlan(ctx, &p); err != nil { - return "", err - } - if p.State != inventory.PlanBuilding { - return "", fmt.Errorf("%s could not be resumed: %s", p.ID, p.Note) - } - return said, nil -} - -// busWaitingOnRetry are the modules a retry left waiting for the bus. Read before the plan is resumed, while it still says -// failed, so read as the plan the retry resumes: a failed plan is not open, and busWaitingIn would name none, leaving the -// answer "asked" with nothing after it (issue 1403). -func busWaitingOnRetry(p inventory.Plan) []string { - p.State = inventory.PlanBuilding - return busWaitingIn(p) -} - -// retryWords are the plan's note and the verb's answer for a retry: what was asked again (as the builds it got), and what waits -// for the bus. A retry whose modules all wait for the bus asked none, and says so rather than "asked" with nothing after it -// (issue 1403). -func retryWords(p inventory.Plan, failed, asked, waiting []string) (note, said string) { - again := slicesWithout(failed, waiting) - switch { - case len(again) > 0: - note = fmt.Sprintf("tier %d retried by hand: %s asked again", p.Tier, strings.Join(again, ", ")) - said = fmt.Sprintf("%s retried at tier %d of %d: asked %s; the plan goes on from there as any plan does", - p.ID, p.Tier, len(p.Tiers), strings.Join(asked, ", ")) - default: - note = fmt.Sprintf("tier %d retried by hand: nothing could be asked yet", p.Tier) - said = fmt.Sprintf("%s retried at tier %d of %d: nothing asked yet; the plan goes on from there as any plan does", - p.ID, p.Tier, len(p.Tiers)) - } - if len(waiting) > 0 { - note += "; " + busWaitWords(p, waiting) - said += "; " + busWaitWords(p, waiting) - } - return note, said -} - -// retryRollouts sends each module whose rollout stopped to the machines it was first sent to, again, -// records that send as the first anew, and sets the plan rolling: from there it goes on as the plan -// would have — the rest sent once those report they applied it, the next tier after (ADR 0218). -func retryRollouts(ctx context.Context, open *stores, p *inventory.Plan) (string, error) { - // Every machine once, for all the modules stopped there (novox/hq issue 281). - stopped := stoppedRollouts(*p) - var machines []string - for _, m := range stopped { - for _, n := range p.Modules[m].First { - if !slices.Contains(machines, n) { - machines = append(machines, n) - } - } - } - sort.Strings(machines) - sent, err := sendRollout(ctx, open, machines) - if err != nil { - return "", fmt.Errorf("%s could not be sent to %s again, so %s stays failed: %w", - strings.Join(stopped, ", "), strings.Join(machines, ", "), p.ID, err) - } - now := time.Now().UTC() - var said []string - for _, m := range stopped { - s := p.Modules[m] - var again []string - for _, n := range sent { - if slices.Contains(s.First, n) { - again = append(again, n) - } - } - s.First, s.FirstAt, s.Why = again, &now, "" - said = append(said, m+" to "+strings.Join(again, ", ")) - } - p.State = inventory.PlanRolling - goesOnAgain(p) - p.Note = fmt.Sprintf("tier %d retried by hand; sent %s first again", p.Tier, strings.Join(said, "; ")) - if err := open.inventory.SavePlan(ctx, p); err != nil { - return "", err - } - return fmt.Sprintf("%s retried at tier %d of %d: sent %s first again; the rest follow once it reports it "+ - "applied, as the plan would have", p.ID, p.Tier, len(p.Tiers), strings.Join(said, "; ")), nil -} - -// noVerdictOnItsBuild says a failed gate said nothing about the module's build (novox/hq issue 281): its -// send changed nothing of the module there — the machine already ran that build, carried there by an -// earlier send of the same tier, or one identical to it. Such a module was blamed for its machine. -func noVerdictOnItsBuild(g *inventory.PlanGate) bool { - return g.Rollback == gateUnchanged || (g.From != "" && sameCommit(g.From, g.To)) -} - -// unjudgedAtGate is the modules of a plan's current tier stopped at a gate that was no verdict on their -// build, sorted. -func unjudgedAtGate(p inventory.Plan) []string { - if p.Tier >= len(p.Tiers) { - return nil - } - var out []string - for _, m := range p.Tiers[p.Tier] { - if s := p.Modules[m]; s != nil && s.Gate != nil && s.Gate.Verdict == inventory.GateFailed && noVerdictOnItsBuild(s.Gate) { - out = append(out, m) - } - } - sort.Strings(out) - return out -} - -// retryTierWhole retries a plan stopped at a gate that judged no build of the module it stopped on -// (issue 281): that module is asked again under a new id — its old build may be marked failed, and a new -// verdict is what takes the gate's condition away — and every module of the tier sent first and never -// passed is sent again, the tier whole, one send per machine, judged again. What passed stays passed. -func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again []string) (string, error) { - inv := open.inventory - entries, err := inv.Catalogued(ctx) - if err != nil { - return "", err - } - byName := map[string]inventory.Entry{} - for _, e := range entries { - byName[e.Manifest.Module] = e - } - var resent []string - for _, m := range p.Tiers[p.Tier] { - s := p.Modules[m] - if s == nil || s.FirstAt == nil || s.SentAt != nil || slices.Contains(again, m) || - (s.Gate != nil && s.Gate.Verdict == inventory.GatePassed) { - continue - } - sendAgain(s) - resent = append(resent, m) - } - var asked []string - for _, m := range again { - sendAgain(p.Modules[m]) - askModule(ctx, p, m, byName) - if s := p.Modules[m]; s.State == "asked" && s.Key { - asked = append(asked, m+" from the builder's key") - } else if s.State == "asked" { - asked = append(asked, m+" as "+s.Build) - } - } - if p.State == inventory.PlanFailed && len(failedIn(*p)) == 0 { - p.State = inventory.PlanBuilding - goesOnAgain(p) - p.Note = fmt.Sprintf("tier %d retried by hand: %s asked again; %s sent again with the tier", p.Tier, - strings.Join(again, ", "), orNone(strings.Join(resent, ", "))) - } - if err := inv.SavePlan(ctx, p); err != nil { - return "", err - } - if p.State != inventory.PlanBuilding { - return "", fmt.Errorf("%s could not be resumed: %s", p.ID, p.Note) - } - return fmt.Sprintf("%s retried at tier %d of %d: asked %s; %s sent again with the tier, one send per machine, "+ - "judged again", p.ID, p.Tier, len(p.Tiers), strings.Join(asked, ", "), orNone(strings.Join(resent, ", "))), nil -} - -// sendAgain forgets a module's first send, so its plan sends it again. -func sendAgain(s *inventory.PlanModule) { - s.First, s.FirstAt, s.Gate, s.GatedBy, s.Previous, s.Why = nil, nil, nil, "", "", "" -} - -// toRetry is the modules a retry asks again: every one of the tier that failed, the plan's state -// settled from the build records first (novox/hq issue 457). A plan fails on the first failure in its -// tier, and an outcome arriving after that finds no open plan to answer — it is kept only in the build -// records. Read from the plan alone, a retry asked only the build that failed first, and the records -// then failed the plan again on the next: each failed build of a tier took a retry of its own. What -// still runs is left asked, and its outcome is the plan's once the retry sets it building. -func toRetry(p *inventory.Plan, recorded map[string][]inventory.Build, byID map[string]inventory.Build) []string { - if p.Tier < len(p.Tiers) { - settleFromRecords(p, p.Tiers[p.Tier], recorded, byID) - } - return failedIn(*p) -} - -// stoppedByHandWaiting says a walk was stopped by hand while its current tier still waited (issue 1886): stopped by a -// person, nothing of the tier failed, and a module of it asked, built and not sent, or not yet asked. A stop that left -// merges out is not one: retryPlan refuses it before this is read (novox/hq ADR 0299). -func stoppedByHandWaiting(p inventory.Plan) bool { - if p.State != inventory.PlanFailed || p.Delivery == nil || p.Delivery.Stopped == "" || - p.Tier >= len(p.Tiers) || len(failedIn(p)) > 0 { - return false - } - asked, unsent, unasked := waitingIn(p) - return asked+unsent+unasked > 0 -} - -// waitingIn counts what a plan's current tier waits on: modules asked, built and not sent, not yet asked. -func waitingIn(p inventory.Plan) (asked, unsent, unasked int) { - if p.Tier >= len(p.Tiers) { - return 0, 0, 0 - } - for _, m := range p.Tiers[p.Tier] { - s := p.Modules[m] - switch { - case s == nil || s.State == "": - unasked++ - case s.State == "asked": - asked++ - case s.State == "built" && s.SentAt == nil: - unsent++ - } - } - return asked, unsent, unasked -} - -// holdingABuild is the modules of a stopped walk's current tier that hold a build of its commit, not yet sent: built and -// not sent, or asked under a build id. Only these could put an older build over a newer plan's (review of issue 1886): a -// module not yet asked is asked at its followed branch's head when the walk reaches it, and one asked on the builder's key -// is settled by the look from the key of the commit it reads then. Sorted. -func holdingABuild(p inventory.Plan) []string { - if p.Tier >= len(p.Tiers) { - return nil - } - var out []string - for _, m := range p.Tiers[p.Tier] { - s := p.Modules[m] - if s == nil || s.SentAt != nil || s.Key { - continue - } - if s.State == "built" || (s.State == "asked" && s.Build != "") { - out = append(out, m) - } - } - sort.Strings(out) - return out -} - -// resumeStopped sets a walk stopped by hand building again at the tier it stopped at, its stop cleared and its asks kept -// (issue 1886): the next look carries it on as any walk — a build done waits for its reference, and one done on the -// builder's key is handed on (novox/hq ADR 0522 §3). Its note keeps who stopped it and that a person resumed it. -func resumeStopped(p *inventory.Plan) string { - asked, unsent, unasked := waitingIn(*p) - later := 0 - for i := p.Tier + 1; i < len(p.Tiers); i++ { - later += len(p.Tiers[i]) - } - by, why := p.Delivery.Stopped, p.Delivery.StoppedWhy - waits := fmt.Sprintf("%d asked, %d built and not sent, %d not yet asked in it; %d not yet asked in the tiers after", - asked, unsent, unasked, later) - p.State = inventory.PlanBuilding - goesOnAgain(p) - p.Delivery.Kept = nil - p.Note = fmt.Sprintf("tier %d resumed by hand after the stop by %s (%s): %s", p.Tier, by, why, waits) - return fmt.Sprintf("%s resumed at tier %d of %d, stopped by %s: %s; this verb asks nothing — the next look carries it "+ - "on as any walk", p.ID, p.Tier+1, len(p.Tiers), by, waits) -} - -// silentGateWords say, in a plan's note, that the gate between two tiers found every machine it waits on silent past -// ApplySilentAfter, and failed the plan as lost (issue 1160, ADR 0531): one text for the writer and the reader (lostToSilence). -const silentGateWords = "every machine it waits on said nothing on its send within" - -// lostToSilence is whether a plan failed only at the gate after its tier, every machine it waited on silent (ADR 0531, -// issue 1886): nothing in its tier failed to build, stopped rolling out or is still to ask — the look had sent the whole tier. -func lostToSilence(p inventory.Plan) bool { - if p.State != inventory.PlanFailed || p.Tier >= len(p.Tiers) || !strings.Contains(p.Note, silentGateWords) || - len(failedIn(p)) > 0 || len(stoppedRollouts(p)) > 0 { - return false - } - asked, _, unasked := waitingIn(p) - return asked == 0 && unasked == 0 -} - -// resumeLost sets a walk lost to silence building again at the tier it failed after (ADR 0531, issue 1886), for the next look to -// judge its gate again on the machines' reports, unless every machine its tier was sent to is still silent: then -// nothing it needs is delivered anywhere yet, and the retry is refused as the gate would fail it again. -func resumeLost(p *inventory.Plan, silent map[string]bool) (string, error) { - var sent, still []string - for _, m := range p.Tiers[p.Tier] { - if s := p.Modules[m]; s != nil { - for n := range s.Rest { - if !slices.Contains(sent, n) { - sent = append(sent, n) - } - } - } - } - for _, n := range sent { - if silent[n] { - still = append(still, n) - } - } - sort.Strings(still) - if len(sent) > 0 && len(still) == len(sent) { - return "", fmt.Errorf("%s failed because every machine its tier %d waits on said nothing on its send, and %s "+ - "still said nothing: retry once it reported", p.ID, p.Tier+1, strings.Join(still, ", ")) - } - was := p.Note - p.State = inventory.PlanBuilding - goesOnAgain(p) - p.Note = fmt.Sprintf("tier %d resumed by hand after it failed as lost (%s): its gate is judged again on the "+ - "machines' reports", p.Tier, was) - return fmt.Sprintf("%s resumed at tier %d of %d, failed as lost to silence (%s); this verb asks and sends nothing — "+ - "the next look judges its gate again and carries it on as any walk", p.ID, p.Tier+1, len(p.Tiers), was), nil -} diff --git a/cmd/mesh-controller/plan_retry_stopped_test.go b/cmd/mesh-controller/plan_retry_stopped_test.go deleted file mode 100644 index a4552dfd..00000000 --- a/cmd/mesh-controller/plan_retry_stopped_test.go +++ /dev/null @@ -1,144 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// A walk stopped by hand while its tier waited — modules asked on the builder's keys, two built and never sent, the next -// tier not yet asked — is resumed by `plans retry` (issue 1886): set building again, its stop cleared, the asks it holds -// kept, so the next look carries it on as any walk. Nothing in it failed, so the retry of failed builds had nothing to do -// and refused it, leaving a stopped R4e with no verb to carry its 48 modules on. -func TestRetryResumesAWalkStoppedByHand(t *testing.T) { - at := time.Date(2026, 10, 23, 0, 1, 53, 0, time.UTC) - sent := at.Add(-time.Hour) - p := inventory.Plan{ID: "plan-r4e", Repository: "novox/mesh-sdk", Branch: "main", Commit: "7aacbc16", - Created: at.Add(-2 * time.Hour), State: inventory.PlanFailed, Tier: 1, - Note: "stopped by a person at tier 1: issue 1886", - Delivery: &inventory.PlanDelivery{Stopped: "a person (fix-1886@shanks)", StoppedWhy: "issue 1886", - Kept: []inventory.PlanKept{{Module: "mesh-controller"}}}, - Modules: map[string]*inventory.PlanModule{}} - tier0 := []string{"mesh-controller", "mesh-sdk", "pacman", "route-proxy"} - for _, m := range tier0 { - p.Modules[m] = &inventory.PlanModule{State: "built", BuiltAt: &sent, SentAt: &sent} - } - var tier1 []string - for _, m := range []string{"mesh-tools-go", "mesh-tools"} { - tier1 = append(tier1, m) - p.Modules[m] = &inventory.PlanModule{State: "built", AskedAt: &sent, BuiltAt: &sent, Build: "build-" + m} - } - onKeys := []string{"mesh-build-go", "mesh-build-image", "mesh-build-node", "mesh-delivery-checker", - "mesh-delivery-forge-bridge", "mesh-delivery-gate", "mesh-delivery-indexer", "mesh-delivery-view"} - for _, m := range onKeys { - tier1 = append(tier1, m) - p.Modules[m] = &inventory.PlanModule{State: "asked", AskedAt: &sent, Key: true, Build: "build-" + m} - } - tier2 := []string{"gitea", "mesh-catalog", "mongodb"} - for _, m := range tier2 { - p.Modules[m] = &inventory.PlanModule{} - } - p.Tiers = [][]string{tier0, tier1, tier2} - - if err := retryRefusal(p, []inventory.Plan{p}, ownPathOf(nil)); err != nil { - t.Fatalf("a walk stopped by hand with its tier waiting was refused a retry: %v", err) - } - said := resumeStopped(&p) - if p.State != inventory.PlanBuilding { - t.Errorf("resumed walk is %s, wanted building", p.State) - } - if p.Delivery.Stopped != "" || p.Delivery.StoppedWhy != "" || len(p.Delivery.Kept) > 0 { - t.Errorf("resumed walk still says it stopped: %+v", p.Delivery) - } - if !strings.Contains(p.Note, "resumed by hand") || !strings.Contains(p.Note, "a person (fix-1886@shanks)") { - t.Errorf("the note does not record the hand resume and the stop it undid: %q", p.Note) - } - for _, m := range onKeys { - s := p.Modules[m] - if s.State != "asked" || !s.Key || s.Build != "build-"+m { - t.Errorf("%s's ask was not kept for the look to hand on: %+v", m, s) - } - } - for _, w := range []string{"plan-r4e", "tier 2 of 3", "8 asked", "2 built and not sent", "0 not yet asked in it", - "3 not yet asked in the tiers after"} { - if !strings.Contains(said, w) { - t.Errorf("the answer %q does not say %q", said, w) - } - } - - // What it still refuses: a walk stopped with nothing left in its tier, and one a related open walk runs beside. - done := p - done.State, done.Delivery = inventory.PlanFailed, &inventory.PlanDelivery{Stopped: "a person"} - done.Modules = map[string]*inventory.PlanModule{} - for _, tier := range done.Tiers { - for _, m := range tier { - done.Modules[m] = &inventory.PlanModule{State: "built", BuiltAt: &sent, SentAt: &sent} - } - } - if err := retryRefusal(done, nil, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "nothing in tier 1") { - t.Errorf("a stopped walk with nothing left in its tier was retried: %v", err) - } - stopped := p - stopped.State, stopped.Delivery = inventory.PlanFailed, &inventory.PlanDelivery{Stopped: "a person"} - beside := inventory.Plan{ID: "plan-442", Repository: "novox/mesh-sdk", Branch: "release", Commit: "bbadbb2a", - Created: at, State: inventory.PlanBuilding, Tiers: [][]string{{"mesh-sdk"}}, - Modules: map[string]*inventory.PlanModule{"mesh-sdk": {State: "asked"}}} - if err := retryRefusal(stopped, []inventory.Plan{beside}, ownPathOf(nil)); err == nil || !strings.Contains(err.Error(), "one walk at a time") { - t.Errorf("a stopped walk was resumed beside a related open walk: %v", err) - } - // A newer open plan of its repository and branch supersedes it: resuming would build what that one replaced. - newerOpen := inventory.Plan{ID: "plan-newer", Repository: "novox/mesh-sdk", Branch: "main", Commit: "d00dfeed", - Created: at, State: inventory.PlanBuilding, Tiers: [][]string{{"zsh"}}, - Modules: map[string]*inventory.PlanModule{"zsh": {State: "asked"}}} - if err := retryRefusal(stopped, []inventory.Plan{stopped, newerOpen}, ownPathOf(nil)); err == nil || - !strings.Contains(err.Error(), "plan-newer supersedes it") { - t.Errorf("a stopped walk was resumed under a newer open plan of its branch: %v", err) - } - // A newer plan holding a build of its tier not yet sent refuses it: resuming would send the older build over the newer. - newerBuilt := inventory.Plan{ID: "plan-agent", Repository: "novox/mesh-controller", Branch: "main", Commit: "d00dfeed", - Created: at, State: inventory.PlanDone, Tiers: [][]string{{"mesh-tools"}}, - Modules: map[string]*inventory.PlanModule{"mesh-tools": {State: "built"}}} - if err := retryRefusal(stopped, []inventory.Plan{stopped, newerBuilt}, ownPathOf(nil)); err == nil || - !strings.Contains(err.Error(), "older build") { - t.Errorf("a stopped walk was resumed over a newer plan of a build it has not sent: %v", err) - } -} - -// A newer plan holding a module the stopped walk has not yet asked does not refuse its resume (review of issue 1886): that -// module is asked at its followed branch's head when the walk reaches it, so no older build goes over the newer. Open and -// related, the newer walk refuses it under one walk at a time; done, it does not. One holding a build the walk has not sent -// still refuses it. -func TestResumeIsNotRefusedByANewerPlanOfAModuleNotYetAsked(t *testing.T) { - at := time.Date(2026, 10, 23, 0, 1, 53, 0, time.UTC) - sent := at.Add(-time.Hour) - p := inventory.Plan{ID: "plan-r4e", Repository: "novox/mesh-sdk", Branch: "main", Commit: "7aacbc16", - Created: at.Add(-2 * time.Hour), State: inventory.PlanFailed, Tier: 1, - Delivery: &inventory.PlanDelivery{Stopped: "a person", StoppedWhy: "issue 1886"}, - Tiers: [][]string{{"mesh-sdk"}, {"build-agent", "mesh-delivery-gate"}, {"mesh-delivery-builder", "gitea"}}, - Modules: map[string]*inventory.PlanModule{ - "mesh-sdk": {State: "built", BuiltAt: &sent, SentAt: &sent}, - "build-agent": {State: "built", BuiltAt: &sent}, - "mesh-delivery-gate": {State: "asked", AskedAt: &sent, Key: true}, - "mesh-delivery-builder": {}, "gitea": {}}} - newer := func(state string, holds string) inventory.Plan { - return inventory.Plan{ID: "plan-1792715180015327197", Repository: "novox/mesh-delivery", Branch: "main", - Commit: "d00dfeed", Created: at, State: state, Tiers: [][]string{{holds}}, - Modules: map[string]*inventory.PlanModule{holds: {State: "asked"}}} - } - if err := retryRefusal(p, []inventory.Plan{p, newer(inventory.PlanDone, "mesh-delivery-builder")}, ownPathOf(nil)); err != nil { - t.Errorf("a done newer plan of a module not yet asked refused the resume: %v", err) - } - if err := retryRefusal(p, []inventory.Plan{p, newer(inventory.PlanDone, "mesh-delivery-gate")}, ownPathOf(nil)); err != nil { - t.Errorf("a done newer plan of a module on the builder's key refused the resume: %v", err) - } - if err := retryRefusal(p, []inventory.Plan{p, newer(inventory.PlanBuilding, "mesh-delivery-builder")}, ownPathOf(nil)); err == nil || - !strings.Contains(err.Error(), "one walk at a time") { - t.Errorf("an open newer walk related to it did not refuse the resume under one walk at a time: %v", err) - } - if err := retryRefusal(p, []inventory.Plan{p, newer(inventory.PlanDone, "build-agent")}, ownPathOf(nil)); err == nil || - !strings.Contains(err.Error(), "older build") { - t.Errorf("a newer plan of a built module not yet sent did not refuse the resume: %v", err) - } -} diff --git a/cmd/mesh-controller/plan_retry_tier_test.go b/cmd/mesh-controller/plan_retry_tier_test.go deleted file mode 100644 index 7efd44a8..00000000 --- a/cmd/mesh-controller/plan_retry_tier_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package main - -import ( - "reflect" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// A retry asks every failed build of the tier, not one (novox/hq issue 457). Seen 2026-10-11: gitea -// and plex both failed in tier 0 while the registry was held still; gitea's failure failed the plan, -// and plex's, arriving after, found no open plan and was kept only in the build records. The first -// retry asked gitea alone, the records then failed the plan again on plex, and a second retry asked -// plex. -func TestARetryAsksEveryFailedBuildOfTheTier(t *testing.T) { - asked := time.Date(2026, 10, 11, 1, 29, 0, 0, time.UTC) - failedAt := asked.Add(2 * time.Minute) - p := inventory.Plan{ID: "plan-457", State: inventory.PlanFailed, Tier: 0, - Tiers: [][]string{{"gitea", "plex"}}, Note: "gitea failed to build in tier 0", - Modules: map[string]*inventory.PlanModule{ - "gitea": {State: "failed", AskedAt: &asked, Build: "build-gitea", Why: "cannot reach the registry"}, - "plex": {State: "asked", AskedAt: &asked, Build: "build-plex"}, - }} - byID := map[string]inventory.Build{ - "build-plex": {ID: "build-plex", Module: "plex", At: failedAt, Failed: "cannot reach the registry"}, - } - - if got := toRetry(&p, nil, byID); !reflect.DeepEqual(got, []string{"gitea", "plex"}) { - t.Fatalf("a retry of a tier where gitea and plex failed asks %v", got) - } -} - -// A build of the tier still running when the plan failed is not asked again: its outcome is the plan's -// once the retry sets it building, and one that is recorded built is taken as built. -func TestARetryLeavesABuildThatRunsOrWorked(t *testing.T) { - asked := time.Date(2026, 10, 11, 1, 29, 0, 0, time.UTC) - builtAt := asked.Add(3 * time.Minute) - p := inventory.Plan{ID: "plan-457", State: inventory.PlanFailed, Tier: 0, - Tiers: [][]string{{"a", "b", "c"}}, - Modules: map[string]*inventory.PlanModule{ - "a": {State: "failed", AskedAt: &asked, Build: "build-a", Why: "broken"}, - "b": {State: "asked", AskedAt: &asked, Build: "build-b"}, - "c": {State: "asked", AskedAt: &asked, Build: "build-c"}, - }} - byID := map[string]inventory.Build{"build-c": {ID: "build-c", Module: "c", At: builtAt, Commit: "c0ffee"}} - - if got := toRetry(&p, nil, byID); !reflect.DeepEqual(got, []string{"a"}) { - t.Fatalf("asks %v, want a alone", got) - } - if s := p.Modules["c"]; s.State != "built" || s.Commit != "c0ffee" { - t.Errorf("c, recorded built, is %+v", s) - } - if s := p.Modules["b"]; s.State != "asked" { - t.Errorf("b, still building, is %+v", s) - } -} diff --git a/cmd/mesh-controller/planner_records_test.go b/cmd/mesh-controller/planner_records_test.go deleted file mode 100644 index 209d0710..00000000 --- a/cmd/mesh-controller/planner_records_test.go +++ /dev/null @@ -1,153 +0,0 @@ -//go:build integration - -package main - -import ( - "reflect" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// **The planner over edges the store derives**, not edges written by hand: modules registered, builds -// recorded with what they stood on and which repositories they read, the relation answered by -// inventory.Dependencies (dependenciesOf over the records), and the merge planned by reachOfMerge — the -// path a real merge takes, short of the bus. -// -// **A shared repository moves only what a change's files are in the build source of** (novox/hq ADR 0267, -// issues 338 and 363): each build records the build source it said, and a merge is mapped onto those of the -// newest builds. A build that said none (P below, as every build before ADR 0267) is read as before: P moves -// on any merge to the repository it packages, though through no edge. The rows tagged 338 held the opposite -// until ADR 0267 was built. -func TestASharedRepositoryIsPlannedFromTheRecordedBuildSources(t *testing.T) { - inv := inventory.ForTest(t) - ctx := t.Context() - asked := time.Now().Add(-time.Hour) - sourcesOf := map[string][]inventory.BuildSource{} - register := func(m catalogue.Manifest, repository, path string, against []string, read []inventory.ReadRepository) { - t.Helper() - if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: repository, Seat: "git", Path: path, - Ref: "main", BuiltFrom: "old", Asked: asked}); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + m.Module, Repository: repository, Ref: "main", - Module: m.Module, Commit: "old", On: "builder", Path: path, Against: against, Read: read, Asked: asked, - Sources: sourcesOf[m.Module]}); err != nil { - t.Fatal(err) - } - } - controllerRead := []inventory.ReadRepository{{Repository: "novox/mesh-controller", Ref: "main"}} - agent := catalogue.Manifest{Module: "build-agent", Version: "1", - Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}} - - // The shape of issue 338, each build saying its build source (ADR 0267). - gomod := []string{"go.mod", "go.sum"} - sourcesOf["mesh-controller"] = []inventory.BuildSource{{Paths: append([]string{"module.json", "cmd/mesh-controller/", - "internal/conditions/", "internal/broker/"}, gomod...)}} - sourcesOf["build-agent"] = []inventory.BuildSource{ - {Paths: []string{"modules/build-agent/Dockerfile", "modules/build-agent/module.json"}}, - {Repository: "novox/mesh-controller", Ref: "main", Paths: append([]string{"cmd/mesh-builder/", "internal/broker/"}, gomod...)}} - sourcesOf["route-proxy"] = []inventory.BuildSource{ - {Paths: []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}}, - {Repository: "novox/mesh-controller", Ref: "main", Paths: append([]string{"modules/route-proxy/cmd/route-proxy/", "internal/broker/"}, gomod...)}} - register(catalogue.Manifest{Module: "mesh-controller", Version: "1"}, "novox/mesh-controller", "", nil, nil) - register(agent, "novox/mesh-catalog", "modules/build-agent", nil, controllerRead) - register(catalogue.Manifest{Module: "route-proxy", Version: "1"}, "novox/mesh-catalog", "modules/route-proxy", nil, controllerRead) - register(catalogue.Manifest{Module: "gitea", Version: "1"}, "novox/mesh-catalog", "modules/gitea", nil, nil) - // A, B and C in one repository; D built against A's artifact, E declaring B, P packaging the repository. - for _, n := range []string{"a", "b", "c"} { - register(catalogue.Manifest{Module: n, Version: "1"}, "novox/one", "modules/"+n, nil, nil) - } - register(catalogue.Manifest{Module: "d", Version: "1"}, "novox/two", "d", - []string{catalogue.ArtifactStoreScheme + "a/runtime@sha256:" + strings.Repeat("0", 64)}, nil) - register(catalogue.Manifest{Module: "e", Version: "1", Build: &catalogue.Build{ - On: []catalogue.BuildsOn{{Arg: "BASE", Module: "b", Artifact: "runtime"}}}}, "novox/two", "e", nil, nil) - register(catalogue.Manifest{Module: "p", Version: "1"}, "novox/two", "p", nil, - []inventory.ReadRepository{{Repository: "novox/one", Ref: "main"}}) - - entries, err := inv.Catalogued(ctx) - if err != nil { - t.Fatal(err) - } - read, err := inv.ReadRepositories(ctx) - if err != nil { - t.Fatal(err) - } - edges, err := inv.Dependencies(ctx) - if err != nil { - t.Fatal(err) - } - - // The edges the hand-written rows of TestASharedRepositoryMovesWhatPackagesItAsItDoesToday use are - // the ones derived here. - var shared []inventory.Edge - in338 := map[string]bool{"mesh-controller": true, "build-agent": true, "route-proxy": true, "gitea": true} - for _, e := range edges { - if in338[e.From] && in338[e.To] { - shared = append(shared, e) - } - } - if !reflect.DeepEqual(shared, sharedRepositoryEdges) { - t.Errorf("derived %v\nthe hand-written rows use %v", shared, sharedRepositoryEdges) - } - // Each kind derived from its record: built against (stands-on), build.on (declared); a read draws none. - for _, e := range edges { - if e.Kind == inventory.EdgePackages { - t.Errorf("a packages edge was drawn (ADR 0267 rule 4): %v", e) - } - } - for _, want := range []inventory.Edge{ - dep("d", inventory.EdgeStandsOn, "a"), - dep("e", inventory.EdgeDeclared, "b"), - dep("d", inventory.EdgeBuiltBy, "build-agent"), - } { - found := false - for _, e := range edges { - found = found || e == want - } - if !found { - t.Errorf("no %s %s %s derived: %v", want.From, want.Kind, want.To, edges) - } - } - - for _, c := range []struct { - what, repo string - paths []string - want string - issue338 bool - }{ - {"A and B changed, C untouched: D after A, E after B; P, which said no build source, reads all", "one", - []string{"modules/a/x.go", "modules/b/x.go"}, "a,b,p | d,e", false}, - {"C alone: C, and P, read whole as before; P after nothing", "one", - []string{"modules/c/x.go"}, "c,p", false}, - {"a README of the repository P packages: P, read whole as before", "one", - []string{"README.md"}, "p", false}, - {"the dependent's repository: D alone", "two", []string{"d/main.go"}, "d", false}, - {"a README of the controller's repository: no module", "mesh-controller", - []string{"README.md"}, "", true}, - {"the controller's own command: the controller alone", "mesh-controller", - []string{"cmd/mesh-controller/main.go"}, "mesh-controller", true}, - {"a package only the controller builds from: the controller alone", "mesh-controller", - []string{"internal/conditions/condition.go"}, "mesh-controller", true}, - {"the route proxy's program: the route proxy alone", "mesh-controller", - []string{"modules/route-proxy/cmd/route-proxy/main.go"}, "route-proxy", true}, - {"a package all three build from: all three", "mesh-controller", - []string{"internal/broker/broker.go"}, "mesh-controller,route-proxy | build-agent", false}, - {"the route proxy's directory in the catalogue: it alone", "mesh-catalog", - []string{"modules/route-proxy/module.json"}, "route-proxy", false}, - {"the build agent's directory: it alone, nothing it builds", "mesh-catalog", - []string{"modules/build-agent/module.json"}, "build-agent", false}, - } { - _, got := planMerge(t, c.repo, c.paths, entries, read, edges) - if got != c.want { - tag := "" - if c.issue338 { - tag = " (issue 338, flipped by ADR 0267)" - } - t.Errorf("%s: planned %q, wanted %q%s", c.what, got, c.want, tag) - } - } -} diff --git a/cmd/mesh-controller/planner_rules_test.go b/cmd/mesh-controller/planner_rules_test.go deleted file mode 100644 index 4e14a594..00000000 --- a/cmd/mesh-controller/planner_rules_test.go +++ /dev/null @@ -1,672 +0,0 @@ -package main - -import ( - "fmt" - "math/rand/v2" - "sort" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// The delivery planner's rules, as recorded (novox/hq ADR 0162 §1, ADR 0238 §3), held by one table and -// one property over reachOfMerge — the planner's one answer to "what does this merge move, and in which -// order". A row that fails here on main is a planner that breaks a recorded rule: the row stays, the -// expectation is not bent to the code. -// -// The kinds of edge, as the code reads them (release_plan.go): -// -// kind widens the plan orders the tiers -// stands-on yes yes, after its base is built -// declared yes yes, after its base is built -// packages no no — retired by novox/hq ADR 0267; one recorded before is read and ignored -// built-by no yes, after the build machine — except for what the build machine stands -// on, and for the controller whose worker it binds -// worker-of no yes, the build seat's holder after the controller (hq issue 206) - -const ( - repoOne = "http://forge.internal:20000/novox/one.git" - repoTwo = "http://forge.internal:20000/novox/two.git" -) - -// dep is one edge of the catalogue's relation: from depends on to, in the way kind says. -func dep(from, kind, to string) inventory.Edge { - return inventory.Edge{From: from, To: to, Kind: kind} -} - -// tiered is a plan's tiers as one line: a tier's modules by comma, tiers by " | ". Empty for no plan. -func tiered(tiers [][]string) string { - var out []string - for _, t := range tiers { - out = append(out, strings.Join(t, ",")) - } - return strings.Join(out, " | ") -} - -// planMerge is what reachOfMerge plans for a merge of these files into a repository's main: its tiers as -// one line, and the files no build reads. It also holds the plan to its own shape: every module it builds -// is in exactly one tier. -func planMerge(t *testing.T, repo string, paths []string, entries []inventory.Entry, - read map[string][]inventory.ReadRepository, edges []inventory.Edge) (mergeReach, string) { - t.Helper() - m := link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Commit: "head", Paths: paths} - r := reachOfMerge(m, entries, read, edges) - seen := map[string]int{} - for _, tier := range r.Plan.Tiers { - for _, name := range tier { - seen[name]++ - } - } - for name := range r.Plan.Modules { - if seen[name] != 1 { - t.Errorf("%s/%v: %s is in %d tiers of %v", repo, paths, name, seen[name], r.Plan.Tiers) - } - } - if len(seen) != len(r.Plan.Modules) { - t.Errorf("%s/%v: the tiers %v hold modules the plan does not (%d)", repo, paths, r.Plan.Tiers, len(r.Plan.Modules)) - } - return r, tiered(r.Plan.Tiers) -} - -// **A merge moves the modules whose directory it changed, and everything built on them; nothing else.** -// Each row is a catalogue (modules in directories of one or two repositories), its dependencies with -// their kinds, the files one commit changed, and the exact plan: the moved set and its tier order. -func TestAPlanIsWhatTheChangeTouchedAndWhatIsBuiltOnIt(t *testing.T) { - in := func(repo, dir string, names ...string) []inventory.Entry { - var out []inventory.Entry - for _, n := range names { - d := dir + "/" + n - if dir == "" { - d = n - } - out = append(out, fromRepo(n, repo, d)) - } - return out - } - // Modules a to f, x and z in novox/one under modules/; g in novox/two at g/. - entries := append(in(repoOne, "modules", "a", "b", "c", "d", "e", "f", "x", "z"), in(repoTwo, "", "g")...) - const ( - standsOn = inventory.EdgeStandsOn - declared = inventory.EdgeDeclared - packages = inventory.EdgePackages - builtBy = inventory.EdgeBuiltBy - workerOf = inventory.EdgeWorkerOf - ) - // The two real cycles the kinds resolve themselves (ADR 0162 §1, hq issue 206). - runtime := append(in(repoOne, "modules", "runtime", "builder"), fromRepo("controller", repoTwo, "")) - for _, c := range []struct { - what string - entries []inventory.Entry - edges []inventory.Edge - repo string - paths []string - want string // the tiers, " | " between them - unread string - cycle bool - }{ - // The operator's case: two modules changed, a third beside them in the same repository untouched, - // each changed one with a dependent. - {what: "A and B changed, C untouched beside them, D on A and E on B", - edges: []inventory.Edge{dep("d", standsOn, "a"), dep("e", declared, "b")}, - repo: "one", paths: []string{"modules/a/main.go", "modules/b/module.json"}, want: "a,b | d,e"}, - {what: "only A's directory: A and what stands on it", - edges: []inventory.Edge{dep("d", standsOn, "a"), dep("e", declared, "b")}, - repo: "one", paths: []string{"modules/a/main.go"}, want: "a | d"}, - {what: "only C's directory: C alone, nothing is built on it", - edges: []inventory.Edge{dep("d", standsOn, "a"), dep("e", declared, "b")}, - repo: "one", paths: []string{"modules/c/Dockerfile"}, want: "c"}, - {what: "only the dependent changed: its base does not move", - edges: []inventory.Edge{dep("d", standsOn, "a")}, - repo: "one", paths: []string{"modules/d/main.go"}, want: "d"}, - {what: "transitive: F on D on A, A changed", - edges: []inventory.Edge{dep("f", standsOn, "d"), dep("d", standsOn, "a")}, - repo: "one", paths: []string{"modules/a/x"}, want: "a | d | f"}, - {what: "transitive across kinds stops at a packages edge: F declared on D, D packages A (ADR 0267)", - edges: []inventory.Edge{dep("f", declared, "d"), dep("d", packages, "a")}, - repo: "one", paths: []string{"modules/a/x"}, want: "a"}, - - // Each kind alone: X depends on A, A changed (widening), then both changed (ordering). - {what: "stands-on (built against A's artifact) widens", edges: []inventory.Edge{dep("x", standsOn, "a")}, - repo: "one", paths: []string{"modules/a/x"}, want: "a | x"}, - {what: "stands-on orders", edges: []inventory.Edge{dep("x", standsOn, "a")}, - repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"}, - {what: "declared (build.on) widens", edges: []inventory.Edge{dep("x", declared, "a")}, - repo: "one", paths: []string{"modules/a/x"}, want: "a | x"}, - {what: "declared orders", edges: []inventory.Edge{dep("x", declared, "a")}, - repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"}, - {what: "packages, recorded before ADR 0267, widens nothing", edges: []inventory.Edge{dep("x", packages, "a")}, - repo: "one", paths: []string{"modules/a/x"}, want: "a"}, - {what: "packages does not order", edges: []inventory.Edge{dep("x", packages, "a")}, - repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"}, - {what: "built-by never widens", edges: []inventory.Edge{dep("x", builtBy, "a")}, - repo: "one", paths: []string{"modules/a/x"}, want: "a"}, - // Whichever builder runs builds it (novox/hq ADR 0360 §3): built-by orders nothing, as before it ordered. - {what: "built-by does not order", edges: []inventory.Edge{dep("x", builtBy, "a")}, - repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a,x"}, - {what: "worker-of never widens", edges: []inventory.Edge{dep("x", workerOf, "a")}, - repo: "one", paths: []string{"modules/a/x"}, want: "a"}, - {what: "worker-of orders", edges: []inventory.Edge{dep("x", workerOf, "a")}, - repo: "one", paths: []string{"modules/a/x", "modules/x/y"}, want: "a | x"}, - {what: "a change to the base alone does not move what it builds", edges: []inventory.Edge{dep("x", builtBy, "a"), - dep("d", builtBy, "a"), dep("e", standsOn, "a")}, - repo: "one", paths: []string{"modules/a/module.json"}, want: "a | e"}, - - // The cycles the kinds resolve: the build machine stands on the runtime image the runtime image is - // built by; the build seat's holder follows the controller that is built by it. - {what: "the build machine's base comes first, built by the build machine that runs", entries: runtime, - edges: []inventory.Edge{dep("runtime", builtBy, "builder"), dep("builder", standsOn, "runtime")}, - repo: "one", paths: []string{"modules/runtime/Dockerfile", "modules/builder/main.go"}, want: "runtime | builder"}, - {what: "the runtime image alone takes the build machine on it along", entries: runtime, - edges: []inventory.Edge{dep("runtime", builtBy, "builder"), dep("builder", standsOn, "runtime")}, - repo: "one", paths: []string{"modules/runtime/Dockerfile"}, want: "runtime | builder"}, - {what: "the build machine alone moves alone", entries: runtime, - edges: []inventory.Edge{dep("runtime", builtBy, "builder"), dep("builder", standsOn, "runtime")}, - repo: "one", paths: []string{"modules/builder/main.go"}, want: "builder"}, - {what: "the build seat's holder follows the controller it binds the worker of", entries: runtime, - edges: []inventory.Edge{dep("controller", builtBy, "builder"), dep("builder", workerOf, "controller")}, - repo: "two", paths: []string{"cmd/main.go"}, want: "controller"}, - - // Two repositories. - {what: "a dependent in another repository follows its base", - edges: []inventory.Edge{dep("g", standsOn, "a")}, - repo: "one", paths: []string{"modules/a/x"}, want: "a | g"}, - {what: "a directory of the same name in another repository is not this one's", - edges: []inventory.Edge{dep("g", standsOn, "a")}, - repo: "two", paths: []string{"modules/a/x"}, want: "", unread: "modules/a/x"}, - {what: "the dependent's own repository moves the dependent alone", - edges: []inventory.Edge{dep("g", standsOn, "a")}, - repo: "two", paths: []string{"g/main.go"}, want: "g"}, - - // A diamond. - {what: "a diamond, one side changed", edges: []inventory.Edge{dep("d", standsOn, "a"), dep("d", standsOn, "b")}, - repo: "one", paths: []string{"modules/a/x"}, want: "a | d"}, - {what: "a diamond, both sides changed", edges: []inventory.Edge{dep("d", standsOn, "a"), dep("d", standsOn, "b")}, - repo: "one", paths: []string{"modules/a/x", "modules/b/x"}, want: "a,b | d"}, - {what: "a diamond on one base", edges: []inventory.Edge{dep("d", standsOn, "a"), dep("d", declared, "b"), - dep("a", standsOn, "z"), dep("b", standsOn, "z")}, - repo: "one", paths: []string{"modules/z/x"}, want: "z | a,b | d"}, - {what: "a diamond of mixed kinds orders on the ordering side only", - edges: []inventory.Edge{dep("d", standsOn, "a"), dep("d", packages, "b")}, - repo: "one", paths: []string{"modules/b/x"}, want: "b"}, - - // A cycle the catalogue should never produce: what remains is one last tier, and said. - {what: "a cycle is one last tier, not lost", edges: []inventory.Edge{dep("a", standsOn, "b"), dep("b", standsOn, "a"), - dep("f", standsOn, "c")}, - repo: "one", paths: []string{"modules/a/x", "modules/c/x"}, want: "c | f | a,b", cycle: true}, - - // Files no build reads. - {what: "a README at the root of a repository whose modules all live below it", - edges: []inventory.Edge{dep("d", standsOn, "a")}, - repo: "one", paths: []string{"README.md"}, want: "", unread: "README.md"}, - {what: "a directory no module lives in", edges: []inventory.Edge{dep("d", standsOn, "a")}, - repo: "one", paths: []string{"modules/lib/x.go", "modules/README.md"}, want: "", - unread: "modules/lib/x.go,modules/README.md"}, - {what: "a module's directory beside a root file", edges: []inventory.Edge{dep("d", standsOn, "a")}, - repo: "one", paths: []string{"merge-check.sh", "modules/a/x"}, want: "a | d", unread: "merge-check.sh"}, - {what: "a directory whose name begins with a module's", edges: []inventory.Edge{dep("d", standsOn, "a")}, - repo: "one", paths: []string{"modules/ab/x"}, want: "", unread: "modules/ab/x"}, - } { - e := entries - if c.entries != nil { - e = c.entries - } - r, got := planMerge(t, c.repo, c.paths, e, nil, c.edges) - if got != c.want { - t.Errorf("%s: planned %q, wanted %q", c.what, got, c.want) - } - if u := strings.Join(r.Unread, ","); u != c.unread { - t.Errorf("%s: unread %q, wanted %q", c.what, u, c.unread) - } - // A packages edge in the last tier is no cycle; hasCycle said one on main at 8170fc5. - if hasCycle(r.Plan.Tiers, c.edges) != c.cycle { - t.Errorf("%s: a cycle said %v, wanted %v (%v)", c.what, !c.cycle, c.cycle, r.Plan.Tiers) - } - } -} - -// **A shared repository moves only what a change's files are in the build source of** (novox/hq ADR 0267, -// issue 338, issue 363). The controller is built from its repository's root as a Go bundle; the route proxy -// and the build seat's holder build images whose context is that repository and which name the package they -// compile. Each newest trunk build said its build source — the import closure of its program — and a merge -// is mapped onto those. The rows tagged 338 held the opposite until ADR 0267 was built: every merge to the -// controller's repository planned all three, in three tiers. -// -// The build sources are this repository's own programs as GoBuildSource reads them (held to that by -// TestThisRepositorysProgramsHaveBuildSourcesOfTheirOwn in internal/builder), cut to what the rows need. -func TestASharedRepositoryMovesOnlyWhatItsBuildSourceHolds(t *testing.T) { - const catalogueRepo = "http://forge.internal:20000/novox/mesh-catalog.git" - const controllerRepo = "http://forge.internal:20000/novox/mesh-controller.git" - entries := []inventory.Entry{ - fromRepo("mesh-controller", controllerRepo, ""), - fromRepo("build-agent", catalogueRepo, "modules/build-agent"), - fromRepo("route-proxy", catalogueRepo, "modules/route-proxy"), - fromRepo("gitea", catalogueRepo, "modules/gitea"), - } - gomod := []string{"go.mod", "go.sum", "vendor/modules.txt"} - with := func(paths ...string) []string { return append(append([]string{}, gomod...), paths...) } - read := map[string][]inventory.ReadRepository{ - "mesh-controller": {{Own: true, Paths: with("module.json", "cmd/mesh-controller/", "internal/conditions/", - "internal/broker/", "internal/builder/", "internal/inventory/", "internal/inventory/migrations/**", - "vendor/github.com/nats-io/nats.go/")}}, - "build-agent": { - {Repository: "novox/mesh-controller", Ref: "main", Paths: with("cmd/mesh-builder/", "internal/broker/", - "internal/builder/", "internal/inventory/", "internal/inventory/migrations/**", "vendor/github.com/nats-io/nats.go/")}, - {Own: true, Paths: []string{"modules/build-agent/Dockerfile", "modules/build-agent/module.json"}}, - }, - "route-proxy": { - {Repository: "novox/mesh-controller", Ref: "main", Paths: with("modules/route-proxy/cmd/route-proxy/", "internal/broker/", - "vendor/github.com/nats-io/nats.go/")}, - {Own: true, Paths: []string{"modules/route-proxy/Dockerfile", "modules/route-proxy/module.json"}}, - }, - } - // With no build source said — before each module's first trunk build under ADR 0267, or while an - // earlier merge's build of it is pending — a module is read as before. - unsaid := map[string][]inventory.ReadRepository{ - "build-agent": {{Repository: "novox/mesh-controller", Ref: "main"}}, - "route-proxy": {{Repository: "novox/mesh-controller", Ref: "main"}}, - } - edges := sharedRepositoryEdges - for _, c := range []struct { - what, repo string - paths []string - read map[string][]inventory.ReadRepository - want string - unread string - }{ - // The operator's acceptance: a merge of the controller's own code plans the controller alone. - {"the controller's own command: the controller alone (338)", "mesh-controller", - []string{"cmd/mesh-controller/main.go"}, read, "mesh-controller", ""}, - {"a package only the controller builds from: the controller alone (338)", "mesh-controller", - []string{"internal/conditions/condition.go", "internal/conditions/bus.go"}, read, "mesh-controller", ""}, - {"a test beside the controller's command: nothing is built from it", "mesh-controller", - []string{"cmd/mesh-controller/main_test.go"}, read, "", "cmd/mesh-controller/main_test.go"}, - {"a README of the controller's repository: no module (338)", "mesh-controller", - []string{"README.md"}, read, "", "README.md"}, - {"the route proxy's program alone: the route proxy alone (338)", "mesh-controller", - []string{"modules/route-proxy/cmd/route-proxy/main.go"}, read, "route-proxy", ""}, - {"the build seat's program alone: its holder alone", "mesh-controller", - []string{"cmd/mesh-builder/main.go"}, read, "build-agent", ""}, - {"a package the build seat's program and the controller build from: both", "mesh-controller", - []string{"internal/builder/builder.go"}, read, "mesh-controller | build-agent", ""}, - {"a migration the controller and the build seat's program embed: both", "mesh-controller", - []string{"internal/inventory/migrations/0088-a-build-says-its-build-source.sql"}, read, - "mesh-controller | build-agent", ""}, - // A package all three build from: all three; the build seat's holder after the controller whose worker - // it binds (worker-of, issue 206), the proxy beside the controller: built-by orders nothing (ADR 0360 §3). - {"a package all three build from: all three", "mesh-controller", - []string{"internal/broker/broker.go"}, read, "mesh-controller,route-proxy | build-agent", ""}, - {"a vendored package all three build from: all three", "mesh-controller", - []string{"vendor/github.com/nats-io/nats.go/nats.go"}, read, "mesh-controller,route-proxy | build-agent", ""}, - {"go.sum: all three", "mesh-controller", - []string{"go.sum"}, read, "mesh-controller,route-proxy | build-agent", ""}, - {"a file added to the proxy's package: the proxy", "mesh-controller", - []string{"modules/route-proxy/cmd/route-proxy/new.go"}, read, "route-proxy", ""}, - // Before any build source is said: as before. - {"no build source said: a README moves all three, as before", "mesh-controller", - []string{"README.md"}, unsaid, "mesh-controller,route-proxy | build-agent", ""}, - // In the catalogue, where they live, each by its own build source. - {"the route proxy's recipe: it alone", "mesh-catalog", - []string{"modules/route-proxy/Dockerfile"}, read, "route-proxy", ""}, - {"the route proxy's manifest: it alone", "mesh-catalog", - []string{"modules/route-proxy/module.json"}, read, "route-proxy", ""}, - {"the route proxy's README: nothing", "mesh-catalog", - []string{"modules/route-proxy/README.md"}, read, "", "modules/route-proxy/README.md"}, - {"the build agent's manifest: it alone, nothing it builds", "mesh-catalog", - []string{"modules/build-agent/module.json"}, read, "build-agent", ""}, - {"another module of the catalogue: neither", "mesh-catalog", - []string{"modules/gitea/index.ts"}, read, "gitea", ""}, - } { - r, got := planMerge(t, c.repo, c.paths, entries, c.read, edges) - if got != c.want { - t.Errorf("%s: planned %q, wanted %q", c.what, got, c.want) - } - if u := strings.Join(r.Unread, ","); u != c.unread { - t.Errorf("%s: unread %q, wanted %q", c.what, u, c.unread) - } - } - // Files not all said: everything the repository builds, as before. - m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "head", - Paths: []string{"README.md"}, PathsTruncated: true} - if got := tiered(reachOfMerge(m, entries, read, edges).Plan.Tiers); got != "mesh-controller,route-proxy | build-agent" { - t.Errorf("a merge whose files were not all said: planned %q, wanted all three", got) - } -} - -// **A module whose recorded build source a plan has overtaken is read whole** (novox/hq ADR 0267): a merge -// that added an import to the route proxy is planned; before a build of it works — still building, failed, -// or its plan closed before reaching it — a merge changing only the newly imported package must still move -// the proxy, since the build source its last build said does not hold that package. -func TestAModuleAPlanOvertookIsReadWhole(t *testing.T) { - built := time.Date(2026, 10, 10, 1, 0, 0, 0, time.UTC) - read := map[string][]inventory.ReadRepository{ - "route-proxy": { - {Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"modules/route-proxy/cmd/route-proxy/", "go.mod"}, Built: built}, - {Own: true, Paths: []string{"modules/route-proxy/module.json"}, Built: built}, - }, - "mesh-controller": {{Own: true, Paths: []string{"module.json", "cmd/mesh-controller/"}, Built: built}}, - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Paths: []string{"internal/newly/imported.go"}} - if readsFrom(read["route-proxy"], m) { - t.Fatal("the said build source holds the new package: the fixture is wrong") - } - proxy := func(state string) map[string]*inventory.PlanModule { - return map[string]*inventory.PlanModule{"route-proxy": {State: state}, "gitea": {State: "built"}} - } - for _, c := range []struct { - what string - plans []inventory.Plan - whole bool - }{ - {"no plan", nil, false}, - {"a plan still building it", []inventory.Plan{{State: inventory.PlanBuilding, Created: built.Add(-time.Hour), - Modules: proxy("building")}}, true}, - {"a plan made after its build that failed before building it", []inventory.Plan{{State: "failed", - Created: built.Add(time.Minute), Modules: proxy("waiting")}}, true}, - {"a plan made after its build that built it", []inventory.Plan{{State: inventory.PlanDone, - Created: built.Add(time.Minute), Modules: proxy("built")}}, false}, - {"a plan closed before its build", []inventory.Plan{{State: "failed", Created: built.Add(-time.Hour), - Modules: proxy("waiting")}}, false}, - } { - view := planningView(read, c.plans) - if readsFrom(view["route-proxy"], m) != c.whole { - t.Errorf("%s: read whole %v, wanted %v", c.what, !c.whole, c.whole) - } - if (ownSource(view["route-proxy"]) == nil) != c.whole { - t.Errorf("%s: its own build source kept %v", c.what, ownSource(view["route-proxy"]) != nil) - } - if ownSource(view["mesh-controller"]) == nil { - t.Errorf("%s: a module no plan holds lost its build source", c.what) - } - } -} - -// **A missed merge that moves only a module packaging the repository is acted on** (novox/hq ADR 0267, -// issue 266): the catch-up asks wouldMove, which counts it; once a plan or build of it is made after the -// merge, the merge is history for it, and the catch-up leaves it. -func TestAMissedMergeMovingOnlyAPackagingModuleIsActedOnOnce(t *testing.T) { - merged := time.Date(2026, 10, 10, 1, 0, 0, 0, time.UTC) - entries := []inventory.Entry{ - fromRepo("mesh-controller", "http://forge.internal:20000/novox/mesh-controller.git", ""), - fromRepo("route-proxy", "http://forge.internal:20000/novox/mesh-catalog.git", "modules/route-proxy"), - } - read := map[string][]inventory.ReadRepository{ - "mesh-controller": {{Own: true, Paths: []string{"module.json", "cmd/mesh-controller/"}, Built: merged.Add(-time.Hour)}}, - "route-proxy": {{Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"modules/route-proxy/cmd/route-proxy/"}, - Built: merged.Add(-time.Hour), Looked: merged.Add(-time.Hour)}}, - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-controller", Base: "main", Commit: "c1", - MergedAt: merged.Format(time.RFC3339), Paths: []string{"modules/route-proxy/cmd/route-proxy/main.go"}} - if got := wouldMove(m, entries, planningView(read, nil)); len(got) != 1 || got[0].Manifest.Module != "route-proxy" { - t.Fatalf("a missed merge of the proxy's program would move %v", got) - } - // Acted on at once: the plan answering this very merge is a look, however close the clocks. - atOnce := []inventory.Plan{{State: inventory.PlanBuilding, Commit: "c1", Created: merged.Add(2 * time.Second), - Modules: map[string]*inventory.PlanModule{"route-proxy": {State: "building"}}}} - if got := wouldMove(m, entries, planningView(read, atOnce)); len(got) != 0 { - t.Fatalf("a merge whose own plan holds the proxy would move %v again", got) - } - acted := []inventory.Plan{{State: inventory.PlanBuilding, Created: merged.Add(2 * time.Minute), - Modules: map[string]*inventory.PlanModule{"route-proxy": {State: "building"}}}} - if got := wouldMove(m, entries, planningView(read, acted)); len(got) != 0 { - t.Fatalf("a merge acted on for the proxy would move %v again", got) - } - // A plan that closed without building it looked at nothing: the merge is still news for it. - closed := []inventory.Plan{{State: "failed", Created: merged.Add(2 * time.Minute), - Modules: map[string]*inventory.PlanModule{"route-proxy": {State: "waiting"}}}} - if got := wouldMove(m, entries, planningView(read, closed)); len(got) != 1 { - t.Fatalf("a plan that never built the proxy hid the merge from it: %v", got) - } - // A look just before the merge, on clocks a little apart, is no look after it. - skewed := []inventory.Plan{{State: inventory.PlanBuilding, Created: merged.Add(30 * time.Second), - Modules: map[string]*inventory.PlanModule{"route-proxy": {State: "building"}}}} - if got := wouldMove(m, entries, planningView(read, skewed)); len(got) != 1 { - t.Fatalf("a look within the clocks' margin made the merge history: %v", got) - } -} - -// sharedRepositoryEdges is what dependenciesOf derives for the catalogue of the test above, sorted as it -// sorts them: no packages edge (novox/hq ADR 0267 rule 4). -var sharedRepositoryEdges = []inventory.Edge{ - dep("build-agent", inventory.EdgeWorkerOf, "mesh-controller"), - dep("gitea", inventory.EdgeBuiltBy, "build-agent"), - dep("mesh-controller", inventory.EdgeBuiltBy, "build-agent"), - dep("route-proxy", inventory.EdgeBuiltBy, "build-agent"), -} - -// **The planner's invariant, over random catalogues.** For any catalogue whose dependencies form no cycle -// and any set of changed files in one repository: -// -// - the plan is exactly the modules of that repository whose directory holds a changed file (every file, -// for a module built from the root), and everything reachable from them along stands-on and declared — -// never along packages (novox/hq ADR 0267), built-by or worker-of; -// - every stands-on, declared, built-by and worker-of edge with both ends in the plan has the module -// depended on in an earlier tier; -// - no cycle is said. -// -// Seeded, so a failure is replayed by its seed and case. -func TestAPlanIsTheTouchedModulesAndWhatIsReachableAlongTheWideningEdges(t *testing.T) { - kinds := []string{inventory.EdgeStandsOn, inventory.EdgeDeclared, inventory.EdgePackages, - inventory.EdgeBuiltBy, inventory.EdgeWorkerOf} - widens := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true} - // Built-by orders nothing since novox/hq ADR 0360 §3. - orders := map[string]bool{inventory.EdgeStandsOn: true, inventory.EdgeDeclared: true, - inventory.EdgeWorkerOf: true} - // Directory names drawn from one pool, so two repositories hold directories of the same name, and one - // is a prefix of another. - dirs := []string{"a", "ab", "b", "c", "lib/x", "lib/y", "modules/a", "modules/a/sub"} - files := []string{"README.md", "merge-check.sh", "lib/z.go", "docs/x.md", "modules/README.md"} - - falseCycles, firstFalseCycle := 0, "" - for _, seed := range []uint64{1, 2, 3, 0x338, 0x162} { - rng := rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15)) - for n := 0; n < 100; n++ { - repos := 1 + rng.IntN(3) - repoName := func(i int) string { return fmt.Sprintf("r%d", i) } - count := 1 + rng.IntN(12) - var entries []inventory.Entry - repoOf, dirOf := map[string]int{}, map[string]string{} - for i := 0; i < count; i++ { - name := fmt.Sprintf("m%02d", i) - repo := rng.IntN(repos) - dir := dirs[rng.IntN(len(dirs))] - if rng.IntN(12) == 0 { - dir = "" // built from the repository's root - } - repoOf[name], dirOf[name] = repo, dir - entries = append(entries, fromRepo(name, "http://forge.internal:20000/novox/"+repoName(repo)+".git", dir)) - } - // A graph with no cycle: a module depends only on modules made before it. - var edges []inventory.Edge - for i := 1; i < count; i++ { - for j := 0; j < i; j++ { - if rng.IntN(4) == 0 { - edges = append(edges, dep(fmt.Sprintf("m%02d", i), kinds[rng.IntN(len(kinds))], fmt.Sprintf("m%02d", j))) - } - } - } - merged := rng.IntN(repos) - var paths []string - for k := 1 + rng.IntN(4); k > 0; k-- { - if rng.IntN(3) == 0 { - paths = append(paths, files[rng.IntN(len(files))]) - } else { - paths = append(paths, dirs[rng.IntN(len(dirs))]+"/f.go") - } - } - - // What the rules say. - want := map[string]bool{} - for _, e := range entries { - name := e.Manifest.Module - if repoOf[name] != merged { - continue - } - for _, p := range paths { - if d := dirOf[name]; d == "" || p == d || strings.HasPrefix(p, d+"/") { - want[name] = true - } - } - } - for grew := true; grew; { - grew = false - for _, e := range edges { - if widens[e.Kind] && want[e.To] && !want[e.From] { - want[e.From], grew = true, true - } - } - } - - r, _ := planMerge(t, repoName(merged), paths, entries, nil, edges) - got := map[string]bool{} - tierOf := map[string]int{} - for i, tier := range r.Plan.Tiers { - for _, name := range tier { - got[name], tierOf[name] = true, i - } - } - replay := func() string { - return fmt.Sprintf("seed %#x case %d: repository %s, files %v\n modules %v\n edges %v\n tiers %v", - seed, n, repoName(merged), paths, describe(entries), edges, r.Plan.Tiers) - } - if !sameSet(got, want) { - t.Fatalf("planned %v, wanted %v\n%s", keys(got), keys(want), replay()) - } - for _, e := range edges { - if orders[e.Kind] && got[e.From] && got[e.To] && tierOf[e.To] >= tierOf[e.From] { - t.Fatalf("%s %s %s, and %s is in tier %d, not before %s's %d\n%s", e.From, e.Kind, e.To, - e.To, tierOf[e.To], e.From, tierOf[e.From], replay()) - } - } - if hasCycle(r.Plan.Tiers, edges) { - falseCycles++ - if firstFalseCycle == "" { - firstFalseCycle = replay() - } - } - } - } - // Said once, after the other invariants have run over every case, so it hides none of them (hasCycle - // counted a packages edge on main at 8170fc5: 19 of these 500 cases). - if falseCycles > 0 { - t.Errorf("a cycle said of a graph with none in %d of 500 cases; "+ - "the first:\n%s", falseCycles, firstFalseCycle) - } -} - -func sameSet(a, b map[string]bool) bool { - if len(a) != len(b) { - return false - } - for k := range a { - if !b[k] { - return false - } - } - return true -} - -func keys(m map[string]bool) []string { - out := make([]string, 0, len(m)) - for k := range m { - out = append(out, k) - } - sort.Strings(out) - return out -} - -func describe(entries []inventory.Entry) []string { - var out []string - for _, e := range entries { - out = append(out, fmt.Sprintf("%s@%s:%q", e.Manifest.Module, - strings.TrimSuffix(strings.TrimPrefix(e.Source.Repository, "http://forge.internal:20000/novox/"), ".git"), - e.Source.Path)) - } - return out -} - -// **A plan says why each module is in it** (novox/hq ADR 0267, issue 363): the files of its build source the -// merge changed, or why it is read whole — never that it packages a repository as though that moved it. -func TestAPlanSaysWhyEachModuleIsInIt(t *testing.T) { - const controllerRepo = "http://forge.internal:20000/novox/mesh-controller.git" - controller := fromRepo("mesh-controller", controllerRepo, "") - agent := fromRepo("build-agent", "http://forge.internal:20000/novox/mesh-catalog.git", "modules/build-agent") - gitea := fromRepo("gitea", "http://forge.internal:20000/novox/mesh-catalog.git", "modules/gitea") - built := time.Date(2026, 10, 10, 12, 26, 0, 0, time.UTC) - read := map[string][]inventory.ReadRepository{ - "mesh-controller": {{Own: true, Paths: []string{"module.json", "cmd/mesh-controller/", "internal/link/"}, Built: built}}, - "build-agent": { - {Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"cmd/mesh-builder/", "internal/link/"}, Built: built}, - {Own: true, Paths: []string{"modules/build-agent/module.json"}, Built: built}, - }, - } - merge := func(repo string, paths ...string) link.SourceMoved { - return link.SourceMoved{Owner: "novox", Repo: repo, Base: "main", Paths: paths} - } - open := []inventory.Plan{{ID: "plan-1", State: inventory.PlanBuilding, Created: built.Add(time.Minute), - Modules: map[string]*inventory.PlanModule{"build-agent": {State: "asked"}}}} - for _, c := range []struct { - what string - e inventory.Entry - read map[string][]inventory.ReadRepository - m link.SourceMoved - says string - }{ - {"the controller's own closure", controller, read, merge("mesh-controller", "README.md", "internal/link/handacts.go"), - "its build source changed: 1 changed file(s) in it, e.g. internal/link/handacts.go"}, - {"the build seat's closure, through its context", agent, read, merge("mesh-controller", "internal/link/handacts.go"), - "its build source in novox/mesh-controller changed: 1 changed file(s) in it, e.g. internal/link/handacts.go"}, - {"an open plan has yet to build it", agent, planningView(read, open), merge("mesh-controller", "cmd/mesh-controller/main.go"), - "read whole: plan plan-1 has not built it yet, so every file of novox/mesh-controller, which its build context is, is its build source"}, - {"nothing recorded, built from its root", controller, nil, merge("mesh-controller", "README.md"), - "read whole: no build source recorded, so every file of its repository is its build source"}, - {"nothing recorded, in its directory", gitea, nil, merge("mesh-catalog", "modules/gitea/index.ts"), - "read whole: no build source recorded, so its directory is its build source; e.g. modules/gitea/index.ts"}, - {"a root manifest is not in a module's directory", gitea, nil, merge("mesh-catalog", "module.json", "modules/gitea/x.ts"), - "read whole: no build source recorded, so its directory is its build source; e.g. modules/gitea/x.ts"}, - {"a context on another branch says nothing of this one", agent, map[string][]inventory.ReadRepository{"build-agent": { - {Repository: "novox/mesh-controller", Ref: "release", Paths: []string{"internal/link/"}}, - {Repository: "novox/mesh-controller", Ref: "main"}}}, merge("mesh-controller", "internal/link/handacts.go"), - "read whole: no build source recorded, so every file of novox/mesh-controller, which its build context is, is its build source"}, - {"files not all said", controller, read, link.SourceMoved{Owner: "novox", Repo: "mesh-controller", PathsTruncated: true, - Paths: []string{"x"}}, "read whole: the merge's changed files were not all said"}, - } { - if got := whyMoved(c.e, c.read[c.e.Manifest.Module], c.m); got != c.says { - t.Errorf("%s:\n said %q\n wanted %q", c.what, got, c.says) - } - } -} - -// **A context naming the module's own repository is the module's own build source** (novox/hq ADR 0420 §4, -// issue 1025). A module registered from the controller's repository whose image compiles cmd/x through a -// context naming that same repository is moved by a change to the program, and by none outside its closure. -func TestAContextNamingTheModulesOwnRepositoryIsItsOwnBuildSource(t *testing.T) { - const controllerRepo = "http://forge.internal:20000/novox/mesh-controller.git" - entries := []inventory.Entry{fromRepo("mover", controllerRepo, "modules/mover")} - said := []inventory.ReadRepository{ - {Repository: "novox/mesh-controller", Ref: "main", Paths: []string{"go.mod", "cmd/x/", "internal/y/"}}, - {Own: true, Paths: []string{"modules/mover/module.json"}}, - } - read := map[string][]inventory.ReadRepository{"mover": inventory.OwnRepositoryAsOwn(said, controllerRepo)} - for _, c := range []struct { - what string - paths []string - want string - }{ - {"the program its image compiles", []string{"cmd/x/main.go"}, "mover"}, - {"a package the program imports", []string{"internal/y/y.go"}, "mover"}, - {"its own manifest", []string{"modules/mover/module.json"}, "mover"}, - {"a file outside its closure", []string{"cmd/other/main.go"}, ""}, - } { - if _, got := planMerge(t, "mesh-controller", c.paths, entries, read, nil); got != c.want { - t.Errorf("%s: planned %q, wanted %q", c.what, got, c.want) - } - } - // A context of another repository stays a context read, and a context of this one that said no paths too. - other := []inventory.ReadRepository{{Repository: "novox/mesh-catalog", Paths: []string{"a/"}}, {Repository: "novox/mesh-controller"}} - if got := inventory.OwnRepositoryAsOwn(other, controllerRepo); len(got) != 2 || got[0].Own || got[1].Own { - t.Errorf("a context that is not this repository's with paths became its own: %+v", got) - } -} diff --git a/cmd/mesh-controller/plans_data_test.go b/cmd/mesh-controller/plans_data_test.go deleted file mode 100644 index f7fc58c0..00000000 --- a/cmd/mesh-controller/plans_data_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package main - -import ( - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// flagsOf reads a command line's flags (and its first positional after --what-if) into a map. -func flagsOf(argv []string) map[string]string { - out := map[string]string{} - for i := 0; i < len(argv); i++ { - if !strings.HasPrefix(argv[i], "--") { - continue - } - name := strings.TrimPrefix(argv[i], "--") - if i+1 < len(argv) && !strings.HasPrefix(argv[i+1], "--") { - out[name] = argv[i+1] - i++ - } else { - out[name] = "true" - } - } - return out -} - -// Without the data form the text answer is unchanged (AC2): no data flag, and the data-only arguments are not passed. -func TestPlansWhatIfWithoutDataIsUnchanged(t *testing.T) { - argv, err := argvFor("plans", map[string]any{"repository": "novox/x", "paths": "a,b"}) - if err != nil { - t.Fatal(err) - } - if got := strings.Join(argv, " "); got != "plans --what-if novox/x --paths a,b" { - t.Errorf("the text what-if reads %q", got) - } - if _, err := argvFor("plans", map[string]any{"repository": "novox/x", "paths": "a", "head": "h"}); err == nil { - t.Error("a head without data was passed over, not refused") - } - argv, _ = argvFor("plans", map[string]any{"repository": "novox/x", "paths": "a", "data": "false"}) - if got := strings.Join(argv, " "); got != "plans --what-if novox/x --paths a" { - t.Errorf("data false reads %q", got) - } -} - -func catalogueVerb(name string) (catalogue.Verb, bool) { return controllerVerb(name) } - -func slicesHas(s []string, v string) bool { - for _, x := range s { - if x == v { - return true - } - } - return false -} diff --git a/cmd/mesh-controller/plans_follow.go b/cmd/mesh-controller/plans_follow.go deleted file mode 100644 index 3edf71d9..00000000 --- a/cmd/mesh-controller/plans_follow.go +++ /dev/null @@ -1,159 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "os" - "strconv" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// Following a plan (novox/hq issue 229): `plans wait ` holds its answer until the plan moves, reaches its -// next tier or ends, or until its bound passes, and says which. Before it, an agent following a rollout had -// `plans` answer once, in prose, and wrote a loop of its own round the mesh's tools to be told the plan moved. -// -// **No wait is longer than a call is answered within** (link.LongestAnswer, inside the console's thirty -// seconds): a longer one would answer "still running" and leave its caller to ask `calls` — the loop again. -// So a caller following a plan to its end asks again; each call answers the moment the plan moves. - -// maxPlanWait is the longest bound a wait takes, and the bound when none is given: what a call is answered -// within, less the time a verb's command takes to start and read the store. -var maxPlanWait = link.LongestAnswer - planWaitStart - -// planWaitStart is what the verb's window allows beyond the bound: starting the command and opening the stores. -// A store slow enough to push the read and the answer past it degrades as any long call does: the caller is -// answered that the call is still running, and `calls` with its id says what came of it. -const planWaitStart = 5 * time.Second - -// planWaitEvery is how often a wait reads the plan again. A variable so a test reads it more often -// (novox/hq issue 591). -var planWaitEvery = time.Second - -// The things a wait waits until. -var planWaitUntil = []string{"moved", "tier", "done"} - -// planWaitAsked checks what a wait is asked for: until (moved when empty) and a bound (a duration, or a number -// of seconds; maxPlanWait when empty). -func planWaitAsked(until, bound string) (string, time.Duration, error) { - if until == "" { - until = "moved" - } - known := false - for _, u := range planWaitUntil { - known = known || u == until - } - if !known { - return "", 0, fmt.Errorf("a wait is until %s, not %q", strings.Join(planWaitUntil, ", "), until) - } - d := maxPlanWait - if bound = strings.TrimSpace(bound); bound != "" { - var err error - if secs, isNumber := strconv.Atoi(bound); isNumber == nil { - d = time.Duration(secs) * time.Second - } else if d, err = time.ParseDuration(bound); err != nil { - return "", 0, fmt.Errorf("the bound %q is not a duration (such as 20s) or a number of seconds", bound) - } - if d <= 0 || d > maxPlanWait { - return "", 0, fmt.Errorf("the bound is at most %s, the longest a call is answered within; %s was asked "+ - "— to follow a plan longer, ask again: each call answers the moment the plan moves", maxPlanWait, d) - } - } - return until, d, nil -} - -// planEnded is a plan that will not move again: done, failed or superseded. A batch being assembled is not one. -func planEnded(p inventory.Plan) bool { return !p.Open() && !p.Batch() } - -// waitOnPlan reads the plan until what it waits for happens or the bound passes, and answers the plan as last -// read and what ended the wait: moved (any save of it), tier (its tier changed), done (it ended — which ends -// every wait, since nothing comes after it), or bound. Moved and tier are judged against the plan as first read. -func waitOnPlan(ctx context.Context, read func(context.Context) (inventory.Plan, error), until string, - bound, every time.Duration) (inventory.Plan, string, error) { - first, err := read(ctx) - if err != nil { - return inventory.Plan{}, "", err - } - if planEnded(first) { - return first, "done", nil - } - deadline := time.Now().Add(bound) - tick := time.NewTicker(every) - defer tick.Stop() - last := first - for { - select { - case <-ctx.Done(): - return last, "", ctx.Err() - case <-tick.C: - } - if !time.Now().Before(deadline) { - return last, "bound", nil - } - p, err := read(ctx) - if err != nil { - return last, "", err - } - last = p - switch { - case planEnded(p): - return p, "done", nil - case until == "moved" && p.Revision != first.Revision: - return p, "moved", nil - case until == "tier" && p.Tier != first.Tier: - return p, "tier", nil - } - } -} - -// planWaited is what `plans wait` answers: the plan, what ended the wait, and the line a person reads (`text`, -// which the verb answers as its output and not again in the data). -type planWaited struct { - // Came is what ended the wait: moved, tier, done, or bound when nothing did. - Came string `json:"came"` - Until string `json:"until"` - Waited string `json:"waited"` - Plan planStatus `json:"plan"` - // Revision is the plan's as read: a caller asking again with the same plan sees whether it moved since. - Revision int64 `json:"revision"` - Text string `json:"text"` -} - -func planWaitedDoc(p inventory.Plan, st planStatus, until, came string, waited time.Duration, line string) planWaited { - said := map[string]string{ - "moved": "moved", "tier": "reached another tier", "done": "ended", - "bound": "did not " + map[string]string{"moved": "move", "tier": "reach another tier", "done": "end"}[until], - }[came] - text := fmt.Sprintf("%s %s in %s (waited until %s)\n%s — %s\n", p.ID, said, waited.Round(time.Second), until, - p.ID, line) - return planWaited{Came: came, Until: until, Waited: waited.Round(time.Millisecond).String(), Plan: st, - Revision: p.Revision, Text: text} -} - -// planWaitCommand is `plans wait `: the plan followed until it moves, or the bound passes. -func planWaitCommand(ctx context.Context, inv *inventory.Inventory, id, untilAsked, boundAsked string) error { - until, bound, err := planWaitAsked(untilAsked, boundAsked) - if err != nil { - return err - } - began := time.Now() - p, came, err := waitOnPlan(ctx, func(ctx context.Context) (inventory.Plan, error) { - return inv.PlanByID(ctx, id) - }, until, bound, planWaitEvery) - if err != nil { - return err - } - // Said as `plans ` says its first line, and as `status` says the plan. - now, one := time.Now(), []inventory.Plan{p} - bounds := readTierBounds(ctx, inv, now) - line := planLineSaid(p, now, bounds.of(p.Repository), func(module string) []string { - on, _ := inv.Running(ctx, module) - return on - }, waitsOf(ctx, inv, one)) + heldSaid(p, heldNow(ctx, inv)) - doc := planWaitedDoc(p, planStatuses(one, now, bounds)[0], until, came, now.Sub(began), line) - return json.NewEncoder(os.Stdout).Encode(doc) -} diff --git a/cmd/mesh-controller/plans_follow_integration_test.go b/cmd/mesh-controller/plans_follow_integration_test.go deleted file mode 100644 index 13968d55..00000000 --- a/cmd/mesh-controller/plans_follow_integration_test.go +++ /dev/null @@ -1,62 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestAPlanFollowedFromTheStoreAnswersWhenItMoves(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - // Read every 50ms rather than every second, so the wait answers as soon as the plan moves and the bound - // below passes in a fraction of a second (novox/hq issue 591); the move still waits 1.5s, long enough - // for the wait to have read the plan before it. - was := planWaitEvery - planWaitEvery = 50 * time.Millisecond - t.Cleanup(func() { planWaitEvery = was }) - now := time.Now().UTC() - walk := inventory.Plan{ID: "plan-229", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c229c229", - Created: now, State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, - Modules: map[string]*inventory.PlanModule{"a": {}, "b": {}}} - if err := open.inventory.SavePlan(ctx, &walk); err != nil { - t.Fatal(err) - } - moved := make(chan error, 1) - go func() { - time.Sleep(1500 * time.Millisecond) - p, err := open.inventory.PlanByID(ctx, walk.ID) - if err == nil { - p.Tier, p.State = 1, inventory.PlanRolling - err = open.inventory.SavePlan(ctx, &p) - } - moved <- err - }() - began := time.Now() - var got planWaited - out := printed(t, func() error { - return plansCommand(ctx, []string{"wait", walk.ID, "--until", "tier", "--bound", "15s"}) - }) - if err := <-moved; err != nil { - t.Fatal(err) - } - if err := json.Unmarshal([]byte(out), &got); err != nil { - t.Fatalf("plans wait did not answer data: %v: %q", err, out) - } - if got.Came != "tier" || got.Plan.Tier != 1 || got.Plan.State != inventory.PlanRolling { - t.Fatalf("the wait answered %+v", got) - } - if waited := time.Since(began); waited > 3*time.Second { - t.Errorf("it answered after %s, not when the plan moved", waited) - } - out = printed(t, func() error { - return plansCommand(ctx, []string{"wait", walk.ID, "--bound", "300ms"}) - }) - if err := json.Unmarshal([]byte(out), &got); err != nil || got.Came != "bound" || got.Until != "moved" { - t.Fatalf("a plan that did not move answered %+v (%v)", got, err) - } -} diff --git a/cmd/mesh-controller/plans_follow_test.go b/cmd/mesh-controller/plans_follow_test.go deleted file mode 100644 index 33b19bc8..00000000 --- a/cmd/mesh-controller/plans_follow_test.go +++ /dev/null @@ -1,152 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// A plan as the store would answer it at each read: the reads are counted, and each answers the next. -type planReads struct { - plans []inventory.Plan - n int -} - -func (r *planReads) read(context.Context) (inventory.Plan, error) { - p := r.plans[min(r.n, len(r.plans)-1)] - r.n++ - return p, nil -} - -func followedPlan(rev int64, state string, tier int) inventory.Plan { - return inventory.Plan{ID: "plan-1", Repository: "novox/x", Commit: "abc", State: state, Tier: tier, - Tiers: [][]string{{"a"}, {"b"}}, Revision: rev} -} - -// **`plans` waits for a plan, and says what ended the wait** (novox/hq issue 229): an agent following a -// rollout is told when the plan moved, reached its next tier or ended, or that the bound passed — in one -// call, without a loop of its own. -func TestAWaitForAPlanEndsWhenWhatItWaitsForHappens(t *testing.T) { - cases := []struct { - name string - until string - reads []inventory.Plan - came string - times int - }{ - {"moved: any save", "moved", []inventory.Plan{ - followedPlan(1, inventory.PlanBuilding, 0), followedPlan(1, inventory.PlanBuilding, 0), - followedPlan(2, inventory.PlanBuilding, 0)}, "moved", 3}, - {"tier: a save inside the tier is not enough", "tier", []inventory.Plan{ - followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanBuilding, 0), - followedPlan(3, inventory.PlanRolling, 1)}, "tier", 3}, - {"tier: a plan that ended has no next tier to wait for", "tier", []inventory.Plan{ - followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanFailed, 0)}, "done", 2}, - {"done: a new tier is not the end", "done", []inventory.Plan{ - followedPlan(1, inventory.PlanBuilding, 0), followedPlan(2, inventory.PlanRolling, 1), - followedPlan(3, inventory.PlanDone, 1)}, "done", 3}, - {"done: a plan already ended answers at once", "done", []inventory.Plan{ - followedPlan(4, inventory.PlanSuperseded, 1)}, "done", 1}, - {"the bound passes", "moved", []inventory.Plan{followedPlan(1, inventory.PlanBuilding, 0)}, "bound", -1}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - r := &planReads{plans: c.reads} - p, came, err := waitOnPlan(context.Background(), r.read, c.until, 50*time.Millisecond, time.Millisecond) - if err != nil { - t.Fatal(err) - } - if came != c.came { - t.Fatalf("the wait ended as %q, want %q", came, c.came) - } - if c.times > 0 && r.n != c.times { - t.Errorf("read the plan %d times, want %d: it answered before or after what it waited for", r.n, c.times) - } - if want := c.reads[min(r.n, len(c.reads))-1]; p.Revision != want.Revision { - t.Errorf("answered revision %d, not the plan as it was last read (%d)", p.Revision, want.Revision) - } - }) - } -} - -// What a wait is asked for is checked before anything is read: what it waits until, and a bound no longer than -// a call is answered within. -func TestAWaitIsRefusedWhatItCannotDo(t *testing.T) { - for _, bad := range []struct{ until, bound string }{{"landed", ""}, {"", "1h"}, {"", "-3s"}, {"", "soon"}} { - if _, _, err := planWaitAsked(bad.until, bad.bound); err == nil { - t.Errorf("until %q, bound %q was taken", bad.until, bad.bound) - } - } - until, bound, err := planWaitAsked("", "") - if err != nil || until != "moved" || bound != maxPlanWait { - t.Fatalf("the defaults are %q %s %v, want moved and %s", until, bound, err, maxPlanWait) - } - if _, bound, _ := planWaitAsked("tier", "12"); bound != 12*time.Second { - t.Errorf("a bare number is seconds; read %s", bound) - } -} - -// **The answer is data, with its prose as a field beside it** (novox/hq issue 229): the verb's `answer` carries -// the plan and what ended the wait, and its output is the line a person reads — never JSON inside a string. -func TestAWaitForAPlanAnswersAsData(t *testing.T) { - p := followedPlan(7, inventory.PlanRolling, 1) - st := planStatuses([]inventory.Plan{p}, time.Now(), tierBounds{})[0] - doc := planWaitedDoc(p, st, "tier", "tier", 3*time.Second, "rolling, tier 2 of 2") - stdout, _ := json.Marshal(doc) - a := answerOf([]string{"plans", "wait", "plan-1"}, stdout, "", true) - data, ok := a.Answer.(map[string]any) - if !ok { - t.Fatalf("the answer is not data: %v", a.Answer) - } - if data["came"] != "tier" || data["revision"] != float64(7) { - t.Errorf("the data does not say what ended the wait: %v", data) - } - if plan, _ := data["plan"].(map[string]any); plan["state"] != "rolling" || plan["tier"] != float64(1) { - t.Errorf("the data does not carry the plan: %v", data["plan"]) - } - if _, text := data["text"]; text { - t.Errorf("the prose is in the data too: %v", data) - } - if strings.Contains(a.Output, "{") || !strings.Contains(a.Output, "plan-1") || !strings.Contains(a.Output, "tier") { - t.Errorf("the output is not the line a person reads: %q", a.Output) - } -} - -// The verb composes the wait, and asks for a window as long as its bound. -func TestThePlansVerbComposesAWait(t *testing.T) { - argv, err := argvFor("plans", map[string]any{"wait": "plan-1", "until": "done", "bound": "15s"}) - if err != nil { - t.Fatal(err) - } - if got := strings.Join(argv, " "); got != "plans wait plan-1 --until done --bound 15s" { - t.Fatalf("composed %q", got) - } - if _, err := argvFor("plans", map[string]any{"until": "done"}); err == nil { - t.Error("until without wait was taken") - } - if _, err := argvFor("plans", map[string]any{"wait": "plan-1", "bound": "1h"}); err == nil { - t.Error("a bound past what a call can wait was taken") - } -} - -// Live, with the store: `plans wait` run as the verb runs it answers when the plan reaches its next tier, as -// data, and says the bound passed when nothing moved (novox/hq issue 229). - -// The verb's schema says the longest bound in words, and the words are the value (review 75 on #266): the -// catalogue cannot read maxPlanWait, so this holds them together. -func TestThePlansSchemaSaysTheLongestBoundThereIs(t *testing.T) { - v, ok := controllerVerb("plans") - if !ok { - t.Fatal("no plans verb") - } - props, _ := v.Input["properties"].(map[string]any) - bound, _ := props["bound"].(map[string]any) - said, _ := bound["description"].(string) - if !strings.Contains(said, maxPlanWait.String()+", the default, is the longest") { - t.Fatalf("the schema says %q; the longest bound is %s", said, maxPlanWait) - } -} diff --git a/cmd/mesh-controller/plans_one_at_a_time_test.go b/cmd/mesh-controller/plans_one_at_a_time_test.go deleted file mode 100644 index 3f3eef97..00000000 --- a/cmd/mesh-controller/plans_one_at_a_time_test.go +++ /dev/null @@ -1,49 +0,0 @@ -//go:build integration - -package main - -import ( - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq issue 213: for the moment a machine hands its controller over, the container and the -// process both run the plan timer on one store. Only the one holding the plans moves them; the other -// leaves them alone, and moves them once they are let go. -func TestAControllerLeavesThePlansToTheOneHoldingThem(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - now := time.Now().UTC() - // Every tier done: the next step is the plan's last, and needs nothing but the store. - plan := inventory.Plan{ID: "plan-213", Repository: "r", Commit: "abc", Created: now, Updated: now, - State: inventory.PlanRolling, Tier: 1, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {State: "built"}}} - if err := open.inventory.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - - // The other controller: its own connections to the same store, holding the plans. - other, err := inventory.Open(ctx) - if err != nil { - t.Fatal(err) - } - t.Cleanup(other.Close) - release, err := other.HoldPlans(ctx, false) - if err != nil { - t.Fatal(err) - } - t.Cleanup(release) // before the close above: a pool waits for a connection still held - - advancePlans(ctx, open) - if p, err := open.inventory.PlanByID(ctx, "plan-213"); err != nil || !p.Open() { - t.Fatalf("a controller moved a plan another held: %+v %v", p, err) - } - - release() - advancePlans(ctx, open) - if p, err := open.inventory.PlanByID(ctx, "plan-213"); err != nil || p.State != inventory.PlanDone { - t.Fatalf("the plan did not move once it was let go: %+v %v", p, err) - } -} diff --git a/cmd/mesh-controller/proposals.go b/cmd/mesh-controller/proposals.go index 3015d102..76c30286 100644 --- a/cmd/mesh-controller/proposals.go +++ b/cmd/mesh-controller/proposals.go @@ -56,7 +56,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/outward" ) @@ -952,26 +951,10 @@ func setLayerIn(open *stores) setOnWarrant { changed = "nothing changed" } } - // **And sends it** (novox/hq ADR 0344): the layer the operator approved is walked to the machines it reaches - // at once. A walk that cannot be opened is said as the act's failure — the layer is set and nothing sends it. - walked, err := openSettingsWalk(ctx, open, inventory.PlanSettings{Module: p.Module, Node: p.Node, Ask: askID, - Change: changed}) - if err != nil { - return "", fmt.Errorf("%s (ask %s) is set, and the walk that sends it could not be opened: %v — `push %s` sends it", - changed, askID, err, pushTo(p.Node)) - } - return changed + " (ask " + askID + "); " + walked, nil + return changed + " (ask " + askID + ")", nil } } -// pushTo is the push that sends a layer by hand when its walk could not: the machine, or every one behind. -func pushTo(node string) string { - if node == "" { - return "--behind" - } - return node -} - // keyOfLine is what a refusal names of a line: the key of a change's line ("+ key: …"), or the line's first words. func keyOfLine(line string) string { if len(line) > 2 && strings.ContainsRune("+~-=", rune(line[0])) && line[1] == ' ' { diff --git a/cmd/mesh-controller/provider_hold_test.go b/cmd/mesh-controller/provider_hold_test.go deleted file mode 100644 index 5212a9f7..00000000 --- a/cmd/mesh-controller/provider_hold_test.go +++ /dev/null @@ -1,174 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "slices" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A provider down is said once, at the provider (novox/hq ADR 0240 rule 5, "how it is checked"): one -// unhealthy database provider and three consumers failing their checks that need it — one condition, at the -// provider, urgent, the consumers listed as waiting; their gates wait, not fail; once the provider is -// healthy, a consumer still failing is its own. A consumer failing while its provider is healthy is raised -// on its own from the start. -func TestOneProviderDownAndThreeConsumersFailingAreOneCondition(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - k := conditionsFrom - register(t, open, catalogue.Manifest{Module: "db", Version: "1", - Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}) - consumers := []string{"shop", "wiki", "crm"} - for _, c := range consumers { - register(t, open, catalogue.Manifest{Module: c, Version: "1", Requires: []string{"postgres-database"}}) - } - if _, err := inv.Assign(ctx, "anchor", "db"); err != nil { - t.Fatal(err) - } - machineOf := map[string]string{"shop": "laptop", "wiki": "laptop", "crm": "anchor"} - for _, c := range consumers { - if _, err := inv.Assign(ctx, machineOf[c], c); err != nil { - t.Fatal(err) - } - if err := inv.RecordBindings(ctx, machineOf[c], []inventory.Binding{{Machine: machineOf[c], Consumer: c, - Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil { - t.Fatal(err) - } - } - - at := h0 - say := func(machine string, rs ...link.ResourceHealth) { - t.Helper() - at = at.Add(time.Second) - for i := range rs { - rs[i].Since = at - } - if err := stateHealth(ctx, inv, k, machine, link.Health{Contract: link.ReadinessContract, At: at, Resources: rs}, at); err != nil { - t.Fatal(err) - } - } - dbDown := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: "container", Target: "db", State: link.StateUnhealthy, - Reason: "its command: the database refuses connections", Check: "exec"} - dbUp := dbDown - dbUp.State, dbUp.Reason = link.StateHealthy, "" - failing := func(module string) link.ResourceHealth { - return link.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module, - State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"} - } - fine := func(module string) link.ResourceHealth { - r := failing(module) - r.State, r.Reason = link.StateHealthy, "" - return r - } - openKeys := func() []conditions.Condition { - t.Helper() - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - return list - } - - // Two looks of the provider down and its consumers failing, in either order on each machine. - for look := 0; look < 2; look++ { - say("laptop", failing("shop"), failing("wiki")) - say("anchor", dbDown, failing("crm")) - } - raised := openKeys() - if len(raised) != 1 || raised[0].Key != "module.db.anchor.unhealthy" { - var keys []string - for _, c := range raised { - keys = append(keys, c.Key) - } - t.Fatalf("one provider down and three consumers failing raised %v; want the provider's alone", keys) - } - c := raised[0] - if c.Severity != conditions.Urgent { - t.Errorf("consumers wait on the provider and its condition is %s", c.Severity) - } - said := c.Evidence[0].Said - for _, w := range []string{"shop on laptop", "wiki on laptop", "crm on anchor"} { - if !strings.Contains(said, w) { - t.Errorf("the provider's condition does not list %s as waiting on it: %s", w, said) - } - } - - // Their gates wait, not fail: the judging is neither a pass nor a fault, past the bound too. - f, err := gatherGateFacts(ctx, open, "") - if err != nil { - t.Fatal(err) - } - f.open, f.openErr, f.judged = raised, nil, false - for _, m := range consumers { - h, why := moduleHealthWord(m, machineOf[m], h0, f) - if h != healthWaiting || !strings.Contains(why, "waits on db on anchor") { - t.Errorf("%s's gate: %v %q; want it waiting on its provider", m, h, why) - } - } - // And a whole judging past the bound puts nothing back: it waits. - long := h0.Add(-time.Hour) - for _, m := range []string{"anchor", "laptop"} { - f.reports[m] = inventory.Reported{Node: m, Outcome: inventory.OutcomeApplied, At: &f.now, Current: true} - } - gatherWas := gatherGateFacts - defer func() { gatherGateFacts = gatherWas }() - gatherGateFacts = func(context.Context, *stores, string) (gateFacts, error) { return f, nil } - g := &inventory.PlanGate{Machines: []string{"laptop"}, Since: &long} - verdict, err := judgeMoves(ctx, open, g, []judged{{module: "shop", node: "laptop"}}, time.Now()) - if err != nil || verdict != "" || !strings.Contains(g.Last, "waits on db on anchor") { - t.Fatalf("a held consumer's gate past its bound: verdict %q (%v), last %q; want it waiting", verdict, err, g.Last) - } - - // The provider healthy again: a consumer still failing is now its own, at once (its streak stood). - say("anchor", dbUp, fine("crm")) - say("laptop", failing("shop"), fine("wiki")) - var keys []string - for _, c := range openKeys() { - keys = append(keys, c.Key) - } - if !slices.Equal(keys, []string{"module.shop.laptop.unhealthy"}) { - t.Fatalf("after the provider recovered: %v; want shop's own and nothing else", keys) - } -} - -// A consumer failing while its provider is healthy is raised on its own. -func TestAConsumerFailingBesideAHealthyProviderIsItsOwn(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - register(t, open, catalogue.Manifest{Module: "db", Version: "1", - Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}) - register(t, open, catalogue.Manifest{Module: "shop", Version: "1", Requires: []string{"postgres-database"}}) - for _, a := range [][2]string{{"anchor", "db"}, {"laptop", "shop"}} { - if _, err := inv.Assign(ctx, a[0], a[1]); err != nil { - t.Fatal(err) - } - } - if err := inv.RecordBindings(ctx, "laptop", []inventory.Binding{{Machine: "laptop", Consumer: "shop", - Provision: "postgres-database", Provider: catalogue.Chosen{Node: "anchor", Module: "db"}}}); err != nil { - t.Fatal(err) - } - healthy := link.ResourceHealth{Module: "db", Resource: "db.server", Kind: "container", Target: "db", State: link.StateHealthy, Check: "exec"} - shop := link.ResourceHealth{Module: "shop", Resource: "shop.web", Kind: "container", Target: "shop", State: link.StateUnhealthy, - Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"} - for i := 1; i <= 2; i++ { - at := h0.Add(time.Duration(i) * time.Minute) - _ = stateHealth(ctx, inv, conditionsFrom, "anchor", link.Health{Contract: 2, At: at, Resources: []link.ResourceHealth{healthy}}, at) - if err := stateHealth(ctx, inv, conditionsFrom, "laptop", link.Health{Contract: 2, At: at, Resources: []link.ResourceHealth{shop}}, at); err != nil { - t.Fatal(err) - } - } - list, _ := conditionsFrom.Open(ctx) - if len(list) != 1 || list[0].Key != "module.shop.laptop.unhealthy" { - t.Fatalf("a consumer failing beside a healthy provider raised %v", list) - } -} diff --git a/cmd/mesh-controller/provisioned_test.go b/cmd/mesh-controller/provisioned_test.go index a9756c35..71ecac88 100644 --- a/cmd/mesh-controller/provisioned_test.go +++ b/cmd/mesh-controller/provisioned_test.go @@ -55,46 +55,6 @@ func TestTheProvisionersAccountIsTheClientsProcessUser(t *testing.T) { } } -// The first-machine gate counts a handed module healthy only once its node's prov.applied for the new build has -// come (ADR 0349 §15, kept by ADR 0362 for a walk finishing at its gate); a failure there fails it. -func TestTheGateWaitsForTheProvisionersReport(t *testing.T) { - now := time.Date(2026, 10, 14, 23, 0, 0, 0, time.UTC) - since := now.Add(-time.Minute) - reported := now.Add(-50 * time.Second) - digest := "sha256:" + strings.Repeat("d", 64) - facts := func(said map[string]inventory.ProvReport) gateFacts { - return gateFacts{now: now, - reports: map[string]inventory.Reported{"g14": {Node: "g14", Outcome: inventory.OutcomeApplied, - At: &reported, Current: true}}, - sent: map[string]inventory.SentDeclaration{"g14": {Digest: "sent", At: since, - Provisioned: map[string]string{"dunst": digest}}}, - provisioned: func(node, module string) (inventory.ProvReport, bool) { - r, ok := said[node+"."+module] - return r, ok - }} - } - m := catalogue.Manifest{Module: "dunst"} - if h, why := judgeHealth("dunst", "", m, "g14", since, facts(nil)); h != healthNotYet || - !strings.Contains(why, "has not reported dunst's files placed") { - t.Fatalf("judged without the provisioner's report: %v %q", h, why) - } - failed := map[string]inventory.ProvReport{"g14.dunst": {Status: inventory.ProvFailed, Templates: digest, - Reason: "a value is not there", At: now.Add(-10 * time.Second)}} - if h, why := judgeHealth("dunst", "", m, "g14", since, facts(failed)); h != healthBroken || - !strings.Contains(why, "a value is not there") { - t.Fatalf("a failed placing is not the build's failure there: %v %q", h, why) - } - placed := map[string]inventory.ProvReport{"g14.dunst": {Status: inventory.ProvApplied, Templates: digest, - At: now.Add(-10 * time.Second)}} - if h, why := judgeHealth("dunst", "", m, "g14", since, facts(placed)); strings.Contains(why, "provisioner") { - t.Fatalf("placed, the module is still held on the provisioner: %v %q", h, why) - } - // A module the send did not hand is judged as before. - if h, why := judgeHealth("picom", "", catalogue.Manifest{Module: "picom"}, "g14", since, facts(nil)); strings.Contains(why, "provisioner") { - t.Fatalf("a module not handed waits on the provisioner: %v %q", h, why) - } -} - // A machine whose provisioner client failed to place a module's files has a failed apply, said as apply-failing // once the failure outlasts an apply interval, with the client's reason (ADR 0362 §3). func TestAFailedPlacingRaisesApplyFailing(t *testing.T) { diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 1e8efc6f..578a3414 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -6,7 +6,6 @@ import ( "encoding/hex" "encoding/json" "errors" - "flag" "fmt" "github.com/novox/mesh-controller/internal/conditions" "io" @@ -189,13 +188,7 @@ func serve(ctx context.Context) (err error) { // by a replaced controller resumes from the first look, which this asks for. No timer looks every so often. // **Its own grants on the bus before it writes** (novox/hq issue 1800): a bucket this controller newly writes is // granted before the first write, not after its gate. - startWithOwnGrants(ctx, ownGrantsOf(open), func() { startRolloutEvents(ctx, open, server.JetStream()) }) - // And the merge window (novox/hq ADR 0276, ADR 0407): a batch is cut into its walk when the window closes, - // which no merge or outcome says. The window is two keys on the bus that expire and a watcher that hears it; - // nothing polls (issue 973). - if err := startMergeWindow(ctx, open, server.JetStream()); err != nil { - return fmt.Errorf("the merge window could not be put on the bus: %w", err) - } + startWithOwnGrants(ctx, ownGrantsOf(open), func() {}) // And the pending assignments settled on a tick of their own (novox/hq ADR 0261): made once their module // is registered, ended with why when its build will not register it, raised and cleared as conditions. // Never by a read. @@ -208,12 +201,9 @@ func serve(ctx context.Context) (err error) { if err := server.Follows(following{open: open}); err != nil { return err } - // And the merges it did not hear while it was away, caught up once it serves (novox/hq ADR 0457, issue 1336). - go catchUpAtStart(ctx, open) // And the merges mesh-delivery made, kept a week with the ask each was made on: what registers a module that // moved between repositories on the operator's Approve (novox/hq ADR 0420 §7, issue 1025). - if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream, catalog: open.inventory.Catalogued, - kept: following{open: open}.resumeFirstBuilds}); err != nil { + if err := server.KeepsDeliveryMerges(keepingDeliveryMerges{js: server.JetStream, catalog: open.inventory.Catalogued}); err != nil { return err } // No timer reads the merges back (novox/hq ADR 0407 decision 3, issue 973): the forge's event and the durable @@ -267,7 +257,6 @@ func serve(ctx context.Context) (err error) { // (novox/hq to-be 45 §9). checkEvents = bus // And every walk this controller keeps in a new state, for the delivery it walks (novox/hq ADR 0239). - inventory.PlanSaved = servingPlanSaved(ctx, bus, open) if err := server.Checks(following{open: open}); err != nil { return err } @@ -339,429 +328,9 @@ func serve(ctx context.Context) (err error) { return server.Serve(ctx) } -// declare sends one node a declaration, signed. -// -// Signed here rather than trusted from the broker: a node connects to the broker and takes -// instruction from the control plane behind it, and those are two identities. If a node believed -// whatever arrived on its queue, a compromised broker could forge declarations — and since the -// host applies whatever the link delivers, that is the whole machine (novox/hq ADR 0004). -func declare(ctx context.Context, args []string) error { - if len(args) != 2 { - return errors.New("declare ") - } - node, path := args[0], args[1] - - raw, err := os.ReadFile(path) - if err != nil { - return err - } - - ident, err := openIdentity(ctx) - if err != nil { - return err - } - defer ident.Close() - - // The node has to exist before it can be told anything. Publishing to a queue nobody consumes - // would sit there looking like success. - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - inv := open.inventory - if _, err := inv.NodeByName(ctx, node); err != nil { - return err - } - - // **With the inventory, so the bus is raised** (novox/hq ADR 0134, design 30). A module's - // declaration and how it hears what it consumes move together: its consumer is derived from the - // same records this declaration is composed from. Raised only when the control plane started - // serving, a module that gained a `consumes` was sent a declaration it could act on and a - // consumer that never delivered the event — and nothing anywhere said the two disagreed - // (found on review, 2026-09-28). Everything the raise does is idempotent. - server, err := connectLink(ctx, inv, nil, nil) - if err != nil { - return err - } - defer server.Close() - - if err := link.Declare(ctx, server.Bus(), ident, node, raw, 15*time.Second); err != nil { - return err - } - // Written down like every other send (novox/hq issue 204): a declaration a person sent by hand - // is still what the machine was last told, and status must not read it as current for the one - // the mesh would compose. Which builds it carried is recorded as not known (novox/hq issue 259): - // the mesh did not compose it, so a push that does not name this machine treats it as held. - // The epoch it carried, if a person wrote one in, is what the machine heard. - var carried struct { - Epoch uint64 `json:"epoch"` - Sequence int64 `json:"sequence"` - Generation int64 `json:"generation"` - } - _ = json.Unmarshal(raw, &carried) - // And recorded as every send is (novox/hq issue 234): by hand, from what it carried; the modules it - // named are not known, since the mesh did not compose it. - if _, err := recordSent(ctx, inv, node, raw, nil, carried.Epoch, sentRecord{sequence: carried.Sequence, - epoch: carried.Epoch, generation: carried.Generation, toldGeneration: carried.Generation, - sender: senderOf(callerOf(ctx)) + ", a declaration sent by hand"}); err != nil { - return err - } - fmt.Printf("sent %s a signed declaration (%d bytes)\n", node, len(raw)) - return nil -} - // OverlayCIDRVar is the range the mesh allocates node addresses from. const OverlayCIDRVar = "MESH_OVERLAY_CIDR" -// pushCommand sends nodes everything they should be: their place on the network, and what their -// assignments resolve to. -// -// One declaration, not two. A node holding its network and not its modules, or the reverse, is -// half-configured for as long as that lasts — and the two are computed from the same picture of -// the mesh, so sending them apart would let them disagree. -func pushCommand(ctx context.Context, args []string) error { - set := flag.NewFlagSet("push", flag.ContinueOnError) - // Only the machines that need it. - // - // **A command rather than a timer, to begin with.** Something that re-pushes on a schedule is - // a scheduler over this, and building the scheduler first would mean two paths to one act - // with nothing to compare them against. A person can run this; so can cron; so can whatever - // eventually watches. - behind := set.Bool("behind", false, - "only machines whose last declaration was refused or partly failed") - // For a named node, wait until it reports applying exactly what it was sent, so `push ` - // means "this node is now what it was told" — a command right after does not race the apply - // (novox/hq ADR 0010). 0 waits for nothing, which is the old fire-and-forget. - wait := set.Duration("wait", 0, - "for a named node, how long to wait for it to report applying what it was sent (0: do not wait)") - // A push by hand is a repair, and says why (novox/hq to-be 45 §7): required through the seat, - // recorded when given at a shell — see handacts.go for why a shell is not refused. - why := addHandActFlags(set) - // Every machine at once is said, not defaulted to (novox/hq ADR 0217): a bare `push` sent the - // whole mesh on 2026-10-05 when its usage was wanted. - all := set.Bool("all", false, "every machine") - // A running module's data moving is sent only when said, per module (novox/hq ADR 0217). - move := set.String("move", "", "modules whose data may move with this push, comma-separated") - positionals, err := parseAround(set, args) - if err != nil { - return err - } - args = positionals - if len(args) > 1 { - return errors.New("push | push --behind | push --all — one machine, the ones behind, or all of them") - } - if len(args) == 0 && !*behind && !*all { - return errors.New("push names a machine, or says --behind (the ones not running what they " + - "should) or --all (every machine) — a push to the whole mesh is not the default (novox/hq ADR 0217)") - } - if *all && (*behind || len(args) == 1) { - return errors.New("push --all is every machine; it takes no machine and no --behind") - } - movable := map[string]bool{} - for _, m := range splitModules(*move) { - movable[m] = true - } - var heldBack []string - if len(args) == 1 && *behind { - // Naming a machine and asking for the ones that need it are two different requests, and - // guessing which was meant would sometimes push to a machine somebody did not name. - return errors.New("push or push --behind, not both: one names a machine and the " + - "other asks which machines need one") - } - recorded := append([]string(nil), args...) - if *behind { - recorded = append(recorded, "--behind") - } - if *all { - recorded = append(recorded, "--all") - } - if *move != "" { - recorded = append(recorded, "--move", *move) - } - why.record(ctx, "push", recorded) - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - inv := open.inventory - - ident, err := openIdentity(ctx) - if err != nil { - return err - } - defer ident.Close() - - // Every node, not only the ones on the private network. A machine that was never given the - // network module still takes modules, and iterating the network here is what used to make - // "on the network" and "managed" the same thing. - nodes, err := inv.Nodes(ctx) - if err != nil { - return err - } - - // Which machines are not in the state they were sent, when that is what was asked for. - var needsOne map[string]inventory.Doing - if *behind { - wrong, err := inv.NotDoingWhatTheyWereTold(ctx) - if err != nil { - return err - } - needsOne = map[string]inventory.Doing{} - for _, d := range wrong { - needsOne[d.Node] = d - } - // **And every machine not running what the mesh would send it.** "Behind" used to mean - // only "failed or refused", so a machine that applied cleanly and whose declaration has - // since changed was not behind — and novox/hq ADR 0010's question, *did my change go - // out?*, was answerable only for the machines that broke. - would, err := wouldSend(ctx, open, nodes) - if err != nil { - return err - } - waiting, err := inv.Waiting(ctx, would) - if err != nil { - return err - } - for _, m := range waiting { - if _, already := needsOne[m.Node]; already { - continue - } - needsOne[m.Node] = inventory.Doing{Node: m.Node, Outcome: "waiting"} - } - if len(needsOne) == 0 { - // Said rather than doing nothing quietly. "Nothing needed one" and "this did not run" - // must never look the same. - fmt.Println("no machine named: a push of the whole mesh, and every machine is doing what it was told — nothing sent") - return nil - } - } - // The private network every declaration of the push is composed with: not computable refuses the push up front. - if _, err := generatorsOf(ctx, open); err != nil { - return err - } - - server, err := connectLink(ctx, nil, nil, nil) - if err != nil { - return err - } - defer server.Close() - - // Which machines this push is about, before any of them is worked out. - var asked []string - for _, n := range nodes { - if len(args) == 1 && n.Name != args[0] { - continue - } - if *behind { - doing, needs := needsOne[n.Name] - if !needs { - continue - } - // A machine that has been failing the same way for a long time is not going to stop - // because it was asked again. Said, and pushed to anyway — refusing would leave no - // way to retry after fixing the cause, and this is a command somebody ran. - // - // Only for machines that reported something. One that is merely waiting has no report - // to be old, and saying it had been failing since the zero time would be a sentence - // about nothing. - if since := time.Since(doing.At); doing.Outcome != "waiting" && since > 6*time.Hour { - fmt.Printf("%s has been %s since %s; pushing again anyway, but the cause is "+ - "unlikely to be timing\n", - n.Name, doing.Outcome, doing.At.Local().Format("2006-01-02 15:04")) - } - } - asked = append(asked, n.Name) - } - // **A push that named no machine says so, first.** Through the console a machine the caller - // meant to name could be lost on the way (novox/hq issue 244): the call arrived empty, ran as - // `push --behind`, and every machine behind was pushed by someone who thought they had pushed one. - // Its whole-mesh reach is the first line of the answer, with the machines it is about to send. - if len(args) == 0 { - which := "every machine" - if *behind { - which = "every machine that is behind" - } - fmt.Printf("no machine named: this is a push of the WHOLE mesh — %s (%d): %s\n", - which, len(asked), strings.Join(asked, ", ")) - } - - // **A whole-mesh push sends no build a gate has not seen** (novox/hq ADR 0236): a machine where one - // waits is left, named, for the release plan — `push ` still sends one machine by a person's word. - if len(args) == 0 { - f, err := readMoveFacts(ctx, inv) - if err != nil { - return err - } - var kept []string - for _, n := range asked { - moves, err := machineMoves(ctx, open, f, n, false) - if err != nil { - return err - } - if len(moves) > 0 { - fmt.Printf(" %s is left: %d build(s) wait there for a gate, which a release plan sends one machine "+ - "at a time (`upgrade backlog` lists them; `push %s` sends it by name)\n", n, len(moves), n) - continue - } - kept = append(kept, n) - } - asked = kept - } - // **The bus is replaced only as a planned step** (novox/hq ADR 0236, to-be 45 §8): a machine whose bus - // would move is not sent by a push — named, it is refused; otherwise it is left and said. - busKept, err := busHeld(ctx, inv, asked) - if err != nil { - return err - } - if len(busKept) > 0 { - if len(args) == 1 { - return fmt.Errorf("%s. Nothing was sent", busKept[args[0]]) - } - var kept []string - for _, n := range asked { - if why, h := busKept[n]; h { - fmt.Printf(" %s is left: %s\n", n, why) - continue - } - kept = append(kept, n) - } - asked = kept - } - - // **The machine holding the bus first** (novox/hq issue 249): its declaration carries the bus's - // user list, and a module's new grants are refused by the bus until that list says them. Among - // the machines asked it goes first; not among them and behind, it is added — a named push whose - // module gained a state would otherwise send the code and leave the right to use it for the - // cascade below, after. - holder, holderBehind, err := brokerBehind(ctx, open, asked) - if err != nil { - return err - } - // **Not when its own modules are held back** (novox/hq issue 259, ADR 0221): added rather than - // named, it is sent its whole declaration, and a build its policy records or a plan has not sent - // it yet would go with the user list. Named and left, with what that costs. - saidHeld := map[string]bool{} - if holderBehind && len(args) == 1 { - held, err := heldMachines(ctx, open, []string{holder}) - if err != nil { - return err - } - if why, isHeld := held[holder]; isHeld { - sayHeld(os.Stdout, holder, why) - fmt.Printf("%s holds the bus, and the user list it would carry has changed: until it is "+ - "sent, the bus may refuse what this push's machines were newly granted\n", holder) - holderBehind = false - saidHeld[holder] = true - } - } - asked = brokerFirst(asked, holder, holderBehind) - - // Held from composing to sending, so a converge on one of them cannot send between the two - // and be overtaken by what was composed before it (novox/hq ADR 0100). - held, release, err := holdNodes(ctx, open, asked) - if err != nil { - return err - } - // Each machine's unmet seat dependencies (novox/hq ADR 0207), said after the sends: in full - // for a machine named, as a count for each of many — the full list is `status`'s. - unheld := map[string][]catalogue.Unheld{} - sending, refusals := composeEach(asked, allotting(held, inv), func(node string) (sendable, error) { - // What this machine's send reads is read afresh: the send before it made and recorded things. - held := freshInput(held) - plan, _, err := resolveNode(held, open, node) - if err != nil { - return sendable{}, err - } - // A module assigned here that this machine cannot host is said and left out, not fatal: the - // healthy modules beside it are still resolved and sent. Reported so it is not silently - // dropped — the remedy is to move it, and until then the rest of the node converges. - reportUnhostable(node, plan) - reportKept(held, plan) - unheld[node] = plan.Unheld - // The private network is in here with everything else. It used to be composed separately - // and prepended, which meant every machine with an address was on it and no machine could - // be kept off. It is a module now, so it arrives the way a module does. - declared, err := composeSend(held, open, node) - if err == nil { - reportLeftOut(node, declared) - } - return declared, err - }) - - defer release() - // A machine whose declaration would move a running module's data is held, and only it (novox/hq - // ADR 0217): said, with the command that sends it, and left out of everything below. - toSend, err := holdingBack(ctx, inv, sending, movable, &heldBack) - if err != nil { - return err - } - // **What it recreates, said before it is sent** (novox/hq ADR 0245): a person's push moves every - // module whose build changed, and recreates what changed in it — a gated send said so, a push did not. - sayWhatAPushRecreates(ctx, open, toSend, os.Stdout) - // Each machine's memberships first, then the declarations (novox/hq issue 249, ADR 0160): a push - // is the one most operators run, and on 2026-10-01 it was the one path that issued none. - bus := overTheBus{open: open, server: server, signer: ident} - sentDigest, err := deliver(ctx, bus, holder, toSend) - if err != nil { - return err - } - release() - told := make([]string, 0, len(toSend)) - for _, r := range toSend { - told = append(told, r.node) - } - fmt.Printf("\n%d node(s) told: %s\n", len(toSend), strings.Join(told, ", ")) - reportUnheldPushed(os.Stdout, len(args) == 1, asked, unheld) - - // **A named push leaves the mesh consistent, not just the machine it named** (novox/hq - // issue 057, ADR 0083). Assigning a cross-node consumer mints a provision, and the PROVIDER's - // grant list is a pure read of secrets already issued — so after the named node is current, - // other machines can be behind *as a consequence*: their declaration now differs from what - // they were last sent. Those are flushed too, by name, in the push's own output. - // - // Compared against what each machine was last SENT, not against a before/after of this push: - // the mint usually happened at `assign` or `module issue`, before this command ran, so the - // only durable signal is "what it should be" versus "what it last received". Bounded: a - // flushed send may itself mint, so this converges over a few rounds. - // - // **Except a machine a policy or a plan holds back** (novox/hq issue 259, ADR 0221): one whose - // modules would move to a build their upgrade policy records rather than rolls out, or that an - // open plan has not sent it yet. It is named, with why, and left for a push that names it. - if len(args) == 1 { - handled := map[string]bool{args[0]: true} - for n := range saidHeld { - handled[n] = true - } - // The cascade is a push too, and a machine in it whose data would move is held the same way - // (novox/hq ADR 0217). - keep := keeping(func(held context.Context, sending []readyNode) ([]readyNode, error) { - return holdingBack(held, inv, sending, movable, &heldBack) - }) - refused, err := flushBehind(ctx, open, nodes, handled, composeForPush(open), bus, holder, keep, os.Stdout) - refusals = append(refusals, refused...) - if err != nil { - return err - } - } - - // A named node is a request to make THAT node current now, so it waits for the node to say it - // applied exactly this. A whole-mesh or --behind push does not wait: it is a sweep, and blocking - // on the slowest machine would hold back the report on all the others. - if *wait > 0 && len(args) == 1 { - if err := waitForApplied(ctx, inv, args[0], sentDigest[args[0]], *wait); err != nil { - return err - } - } - if len(heldBack) > 0 { - sort.Strings(heldBack) - return fmt.Errorf("held %s: a running module's data would move — see above; nothing was sent "+ - "there (novox/hq ADR 0217)", strings.Join(heldBack, ", ")) - } - return couldNotBeResolved(refusals, len(sending)) -} - // waitForApplied blocks until the node reports it applied exactly the declaration just sent, or the // wait runs out. A report of failure or refusal for that same declaration ends the wait at once — // there is nothing to wait for, and the reason is the node's own. @@ -868,32 +437,6 @@ func composeEach(names []string, allot func(node string) (order, error), return sending, refusals } -// sendRound holds the named nodes, composes each and sends each that composed, and gives the hold -// back on every way out — a body that cannot be marshalled and a send that fails included -// (novox/hq ADR 0100). A node that cannot be composed is a refusal, not an error: the others are -// still sent. Their memberships go before their declarations, as every send's do (issue 249). -func sendRound(ctx context.Context, open *stores, names []string, - compose func(held context.Context, node string) (sendable, error), - d delivery, holder string, keep keeping) ([]string, error) { - held, release, err := holdNodes(ctx, open, names) - if err != nil { - return nil, err - } - defer release() - sending, refused := composeEach(names, allotting(held, open.inventory), func(node string) (sendable, error) { - return compose(held, node) - }) - if keep != nil { - if sending, err = keep(held, sending); err != nil { - return refused, err - } - } - if _, err := deliver(held, d, holder, sending); err != nil { - return refused, err - } - return refused, nil -} - // keeping is what a round of sends keeps of the machines it composed, before any is delivered: nil // keeps them all. A push holds back the machines whose running modules' data would move (holdingBack). type keeping func(held context.Context, sending []readyNode) ([]readyNode, error) @@ -1105,65 +648,6 @@ func (b overTheBus) grant(ctx context.Context, sending []readyNode) error { return issueMemberships(ctx, b.open, b.server, sending) } -func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (string, error) { - // The contents the declaration names by digest are in the bucket before it is sent (novox/hq ADR 0486 §1, issue 1696): a node - // that applies it finds every one. A node reading no references has none. - // **Under the node's lock** from the put to the publication, so the sweep of its bucket cannot remove an object between them. - unlock := lockFilesOf(s.node) - err := b.putFiles(ctx, s) - if err != nil { - unlock() - return "", err - } - declared := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second) - unlock() - if declared != nil { - return "", declared - } - // After it is away, not before. A digest recorded for something that failed to send would make - // the machine look current for a declaration it never received. - digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch, - sentRecordOf(ctx, s.declared)) - if err != nil { - return "", err - } - // A send that carried the mesh's record of the controller and the node-engine — a person's push included — - // ends the condition of one kept over it (novox/hq issue 201). - settleCoreDowngrades(ctx, b.open.inventory, s.node, s.declared.Builds) - if len(s.declared.Bindings) > 0 { - // And where it bound each consumer of a provision that keeps its data (novox/hq ADR 0232): - // what the next resolution keeps it at. On the same outliving context as the send's record. - kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) - err := b.open.inventory.RecordBindings(kept, s.node, s.declared.Bindings) - cancel() - if err != nil { - return "", fmt.Errorf("%s was sent its declaration, and where its consumers are bound to their "+ - "data could not be recorded: %w", s.node, err) - } - } - if s.declared.BusUsers != "" { - // And the user list it carried, so the next send reads whether it must go first from the - // list alone (novox/hq issue 249). On the same outliving context as the send's record. - kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) - err := b.open.inventory.RecordSentBusUsers(kept, s.node, digestOf([]byte(s.declared.BusUsers))) - cancel() - if err != nil { - return "", err - } - // The list sent issued the credential of every module it carried, at the fingerprint each is registered at: - // the grants the assigner's sixth rule reads (novox/hq ADR 0528 clause 2, issue 1394). - if bus, ok := b.server.Bus().(link.OverNATS); ok { - if js, err := jetstream.New(bus.Conn); err == nil { - writing, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) - grantsIssued(writing, b.open.inventory, js, b.indent) - cancel() - } - } - } - fmt.Printf("%ssent %s %d resource(s)\n", b.indent, s.node, len(s.declared.Resources)) - return digest, nil -} - // brokerFirst is the machines to send in the order a grant needs (novox/hq issue 249): the machine // holding the bus first — its declaration carries the bus's user list (composeBusUsers), and a // module's new permissions are refused by the bus until that list says them. Among the machines it @@ -1316,24 +800,7 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er if err != nil { return nil, err } - added := "" - if behind && !slices.Contains(names, holder) { - added = holder - } names = brokerFirst(names, holder, behind) - // **No build moves on a machine without a gate** (novox/hq ADR 0236): a send outside its scope that - // would carry one is refused, said with what waits; the bus's machine added for its user list is left. - if names, err = ungatedIn(ctx, open, names, added); err != nil { - return nil, err - } - - // **A recorded build moves only by a person's push** (novox/hq issue 295, ADR 0245): this send — a - // plan's, a release plan's, a rollback's, a healer's, a rotation's — composes every recorded module at - // the build its machine runs. The bus step is a person's word for the bus alone. - if ctx, err = sendKeeps(ctx, inv); err != nil { - return nil, err - } - // Held from composing to sending (novox/hq ADR 0100); a caller that holds them already — // converge, which flips the node and then sends it — is not made to wait on itself. ctx, release, err := holdNodes(ctx, open, names) @@ -1366,17 +833,6 @@ func sendToEach(ctx context.Context, open *stores, names []string) ([]string, er continue } numbered.stamp(&declared) - // **The grants step sends the user list and nothing else** (novox/hq issue 490): judged on this very - // declaration, the one sent, so nothing composed between a check and the send can slip through. - if everyBuildKept(ctx, name) { - other, err := grantsOnlyIn(ctx, open, name, plan, declared) - if err != nil { - return nil, err - } - if other != "" { - return nil, fmt.Errorf("%w: %s", errNotGrantsOnly, other) - } - } reportLeftOut(name, declared) sending = append(sending, readyNode{name, declared}) } @@ -1933,3 +1389,59 @@ func withHearing(issued []byte, hears []broker.HeardIssued) ([]byte, bool, error body, err := json.Marshal(fields) return body, true, err } + +func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (string, error) { + // The contents the declaration names by digest are in the bucket before it is sent (novox/hq ADR 0486 §1, issue 1696): a node + // that applies it finds every one. A node reading no references has none. + // **Under the node's lock** from the put to the publication, so the sweep of its bucket cannot remove an object between them. + unlock := lockFilesOf(s.node) + err := b.putFiles(ctx, s) + if err != nil { + unlock() + return "", err + } + declared := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second) + unlock() + if declared != nil { + return "", declared + } + // After it is away, not before. A digest recorded for something that failed to send would make + // the machine look current for a declaration it never received. + digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch, + sentRecordOf(ctx, s.declared)) + if err != nil { + return "", err + } + if len(s.declared.Bindings) > 0 { + // And where it bound each consumer of a provision that keeps its data (novox/hq ADR 0232): + // what the next resolution keeps it at. On the same outliving context as the send's record. + kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + err := b.open.inventory.RecordBindings(kept, s.node, s.declared.Bindings) + cancel() + if err != nil { + return "", fmt.Errorf("%s was sent its declaration, and where its consumers are bound to their "+ + "data could not be recorded: %w", s.node, err) + } + } + if s.declared.BusUsers != "" { + // And the user list it carried, so the next send reads whether it must go first from the + // list alone (novox/hq issue 249). On the same outliving context as the send's record. + kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + err := b.open.inventory.RecordSentBusUsers(kept, s.node, digestOf([]byte(s.declared.BusUsers))) + cancel() + if err != nil { + return "", err + } + // The list sent issued the credential of every module it carried, at the fingerprint each is registered at: + // the grants the assigner's sixth rule reads (novox/hq ADR 0528 clause 2, issue 1394). + if bus, ok := b.server.Bus().(link.OverNATS); ok { + if js, err := jetstream.New(bus.Conn); err == nil { + writing, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + grantsIssued(writing, b.open.inventory, js, b.indent) + cancel() + } + } + } + fmt.Printf("%ssent %s %d resource(s)\n", b.indent, s.node, len(s.declared.Resources)) + return digest, nil +} diff --git a/cmd/mesh-controller/push_recreates_test.go b/cmd/mesh-controller/push_recreates_test.go deleted file mode 100644 index d8361730..00000000 --- a/cmd/mesh-controller/push_recreates_test.go +++ /dev/null @@ -1,78 +0,0 @@ -//go:build integration - -package main - -import ( - "bytes" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/link" -) - -// The live case of 2026-10-07, 20:27 UTC: a person's `push` of one machine moved mail to a new build whose -// automx container had a new image, and recreated it; the push's answer said nothing of it. The push now -// says, per machine, every module it moves and what that recreates — before it is sent. A module whose -// build did not change, and one new to the machine, are not moves. -func TestAPushSaysWhatItRecreates(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - start := time.Now().Add(-time.Hour) - image := func(c string) string { return "registry.invalid:5000/mailu/x@sha256:" + strings.Repeat(c, 64) } - for _, b := range []link.BuildResult{ - aContainerBuild(t, "mailu", "c74f407a", catalogue.PolicyRecord, start, - map[string][2]string{"smtp": {image("a"), ""}, "automx": {image("c"), ""}}), - aContainerBuild(t, "postgres", "c1111111", "", start.Add(time.Second), - map[string][2]string{"server": {image("e"), ""}}), - } { - if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"mailu", "postgres"} { - if _, err := inv.Assign(ctx, "anchor", m); err != nil { - t.Fatal(err) - } - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", - map[string]string{"mailu": "c74f407a", "postgres": "c1111111"}); err != nil { - t.Fatal(err) - } - // Mail's new build: automx's image changes, smtp's does not. - if _, _, err := takeIn(ctx, inv, aContainerBuild(t, "mailu", "1ab84e69", catalogue.PolicyRecord, - start.Add(time.Minute), map[string][2]string{"smtp": {image("a"), ""}, "automx": {image("d"), ""}})); err != nil { - t.Fatal(err) - } - - // Composed as a person's push composes it. - declared, err := composeSend(ctx, open, "anchor") - if err != nil { - t.Fatal(err) - } - if declared.Builds["mailu"] != "1ab84e69" { - t.Fatalf("a person's push carries mail at %q", declared.Builds["mailu"]) - } - - var said bytes.Buffer - sayWhatAPushRecreates(ctx, open, []readyNode{{node: "anchor", declared: declared}}, &said) - out := said.String() - for _, want := range []string{"anchor moves 1 module(s)", "mailu", "c74f407a → 1ab84e69", - "recreates 1 of mailu's 2 container(s) (with a new image: automx)"} { - if !strings.Contains(out, want) { - t.Fatalf("the push says %q; it does not say %q", out, want) - } - } - if strings.Contains(out, "postgres") { - t.Fatalf("a module whose build did not change is said to move: %q", out) - } - - // A machine never sent anything before has no moves to say. - said.Reset() - sayWhatAPushRecreates(ctx, open, []readyNode{{node: "laptop", declared: declared}}, &said) - if said.Len() != 0 { - t.Fatalf("a machine with no last send is said to move: %q", said.String()) - } -} diff --git a/cmd/mesh-controller/readable.go b/cmd/mesh-controller/readable.go index 3d037c68..df3179ea 100644 --- a/cmd/mesh-controller/readable.go +++ b/cmd/mesh-controller/readable.go @@ -42,9 +42,6 @@ type meshStatus struct { // whose is older is still working — and Waiting cannot tell those apart, because the sent // digest is recorded at send, not at apply. Reported []machineReported `json:"reported"` - // Plans is what the last merges produced and where each stands (novox/hq ADR 0162): the - // open ones first, each saying its tier, what it waits for, and whether it has waited too long. - Plans []planStatus `json:"plans"` // Unresolved is every machine that cannot be worked out at all, with what the mesh said when // it tried. **A machine here is in none of the lists above**: nothing was computed for it, so // there is nothing to compare it against and nothing it can be behind — which is why a @@ -252,7 +249,7 @@ func statusAsJSON(asked answers) ([]byte, error) { out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{}, Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{}, Reported: []machineReported{}, Unresolved: []machineUnresolved{}, - Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.tierBounds), + Network: asked.network, Adopted: adoptedNodes(nodes), NotAnnounced: asked.notAnnounced, NotAnnouncedWhy: asked.notAnnouncedWhy} // In a stated order, so two readings of an unchanged mesh are the same document. untakenNodes := make([]string, 0, len(asked.untaken)) diff --git a/cmd/mesh-controller/recorded_kept.go b/cmd/mesh-controller/recorded_kept.go deleted file mode 100644 index 7e16c74e..00000000 --- a/cmd/mesh-controller/recorded_kept.go +++ /dev/null @@ -1,219 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "sort" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A recorded build reaches a machine only by a person's push (novox/hq issue 295, ADR 0245). -// -// **A send carries the machine's whole declaration** (ADR 0221), composed from the build the mesh holds -// of every module on it. A module whose upgrade policy records — postgres, mongodb, keycloak, the -// network path — has its new build registered at its merge and sent nowhere, "until a person pushes". -// But every other send to its machine composed it too: on 2026-10-07 a catalogue merge adopting the -// images' health checks rebuilt postgres and mongodb with the rest, and the plan's gated send to the -// control node for mail — and to the anchor for the spreadsheet app — carried both, recreating the -// providers every consumer on those machines drops with. No gate judged them (the gate judges only what -// rolls out), and nobody had pushed. -// -// So **every send but a person's push composes a recorded module at the build its machine was last -// sent**: the manifest of that build, from the build records, in place of the one the mesh holds. The -// machine runs what it ran; the send records that it still carries that build; `status` keeps saying -// the machine is behind, and `push ` — a person's word — sends the new one. The bus step is a -// person's too, and carries the bus; any other recorded module waiting on the bus's machine stays. -// -// A recorded module the machine was never sent (a new assignment) is composed as the mesh holds it — -// there is nothing running to keep. One whose kept build is no longer in the records refuses the send, -// said: composing the new build would be the very move this exists to stop. - -type keepRecordedKey struct{} - -// sendKeeps is the context a send that is not a person's push composes under: every recorded module -// kept at the build its machine runs — except, on the bus step, the bus. -func sendKeeps(ctx context.Context, inv *inventory.Inventory) (context.Context, error) { - if !busStepSending(ctx) { - return keepingRecorded(ctx), nil - } - bus, err := pendingBus(ctx, inv) - if err != nil { - return nil, err - } - return keepingRecorded(ctx, bus.module), nil -} - -// keepingRecorded is a context whose sends compose every recorded module at the build its machine runs, -// except the modules named (the bus, on the bus step). -func keepingRecorded(ctx context.Context, except ...string) context.Context { - skip := map[string]bool{} - for _, m := range except { - skip[m] = true - } - return context.WithValue(ctx, keepRecordedKey{}, skip) -} - -// keptExcept is whether this context keeps recorded modules, and the modules it lets move. -func keptExcept(ctx context.Context) (map[string]bool, bool) { - skip, on := ctx.Value(keepRecordedKey{}).(map[string]bool) - return skip, on -} - -type keepEveryBuildKey struct{} - -// keepingEveryBuild is a context whose sends compose every module of one machine — recorded or rolling out — -// at the build that machine was last sent (novox/hq issue 490): the grants step, which sends the machine -// holding the bus its new user list and nothing of any module's new code. That code waits there for a gate -// like any other move; the list must not, or the first machine's gate is judged against a bus that refuses -// what the change newly grants. -// -// **That machine alone.** Composing one machine resolves the others too — who is on the private network, -// who answers a requirement — on the same context, and those are no part of the step's send: they are -// composed as any send composes them. Kept for every machine, a machine whose last send is not known -// refused the bus's machine's composition. -func keepingEveryBuild(ctx context.Context, node string) context.Context { - return context.WithValue(ctx, keepEveryBuildKey{}, node) -} - -// everyBuildKept is whether this context keeps every module of this machine at the build it runs. -func everyBuildKept(ctx context.Context, node string) bool { - on, _ := ctx.Value(keepEveryBuildKey{}).(string) - return on != "" && on == node -} - -type keepForSettingsKey struct{} - -// keepingBuildsForSettings is a context whose sends compose every module of one machine — recorded or rolling out — -// at the build that machine was last sent, as keepingEveryBuild does, for a settings walk's send (novox/hq ADR 0344): -// the layer the operator approved, and nothing of any module's new code. A build held for a person, or waiting -// for a gate, stays where it is; the walk that is its own carries it. -func keepingBuildsForSettings(ctx context.Context, node string) context.Context { - return context.WithValue(ctx, keepForSettingsKey{}, node) -} - -// errSettingsCannotKeep is a settings walk's send refused because a build of its machine cannot be kept: what the -// machine was last sent is not known, or a build it runs is gone from the records. It does not clear by itself, so -// the walk fails on it rather than trying again (ADR 0344). -var errSettingsCannotKeep = errors.New("the settings cannot be sent with every build kept") - -// settingsKeptOn is whether this context is a settings walk's send to this machine. -func settingsKeptOn(ctx context.Context, node string) bool { - on, _ := ctx.Value(keepForSettingsKey{}).(string) - return on != "" && on == node -} - -// recordedKept is, for a send under keepingRecorded, every recorded module the machine was last sent a -// build of that the mesh's build is not identical to: module → the commit it keeps. Nil when the context -// keeps nothing, or when what the machine was last sent is not known (it is then held whole elsewhere — -// ADR 0221). -// -// Under keepingEveryBuild, every module the machine was sent is kept, whatever its policy (novox/hq issue -// 490), and a machine whose last send is not known refuses the send: there is nothing to keep it at, and -// composing the mesh's builds would be the very move the grants step must not make. -func recordedKept(ctx context.Context, open *stores, node string) (map[string]string, error) { - every := everyBuildKept(ctx, node) || settingsKeptOn(ctx, node) - skip, on := keptExcept(ctx) - if !on && !every { - return nil, nil - } - if every { - skip = nil - } - inv := open.inventory - sent, known, err := inv.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - if !known { - if settingsKeptOn(ctx, node) { - return nil, fmt.Errorf("%w: what %s was last sent is not known (novox/hq ADR 0344)", errSettingsCannotKeep, node) - } - if every { - return nil, fmt.Errorf("what %s was last sent is not known, so the bus's user list cannot be sent "+ - "there alone with every build kept (novox/hq issue 490)", node) - } - return nil, nil - } - current, err := inv.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - var f *moveFacts - out := map[string]string{} - for m, was := range sent { - now, held := current[m] - if !held || (now.RollOut && !every) || skip[m] || was == "" || sameCommit(was, now.Commit) { - continue - } - if f == nil { - read, err := readMoveFacts(ctx, inv) - if err != nil { - return nil, err - } - f = &read - } - if f.identical(m, was, now.Commit) { - continue - } - // **Under its own `record`, a move that recreates nothing and moves no data is not kept** (novox/hq ADR 0377 - // §1): it goes as a module that rolls out goes. A person's `record`, the bus, and a move that cannot be - // judged are kept as before (§4, §5). - if !every && f.rolls(m, was) { - continue - } - out[m] = was - } - // **And the controller and the node-engine at the build the machine was last sent, over an older record** - // (novox/hq issue 201, core_downgrade.go). Under keepingEveryBuild they are kept already. - if !every { - core, err := coreKept(ctx, inv, node, sent, current) - if err != nil { - return nil, err - } - for m, was := range core { - out[m] = was - } - } - return out, nil -} - -// applyKept puts the manifest of each module kept at the build its machine runs into a shelf, and refuses, in the words of -// keepRecorded, a build the records no longer hold. The adapter to the composition's input (compose_input.go) calls it with -// a scratch shelf, to read the manifests it gives the node as its own Kept. -func applyKept(ctx context.Context, open *stores, node string, kept map[string]string, shelf map[string]catalogue.Manifest) error { - names := make([]string, 0, len(kept)) - for m := range kept { - names = append(names, m) - } - sort.Strings(names) - for _, m := range names { - ran, found, err := open.inventory.ManifestAt(ctx, m, kept[m]) - if err != nil { - return err - } - if !found && settingsKeptOn(ctx, node) { - return fmt.Errorf("%w: %s runs %s's build %s, which the build records no longer hold (novox/hq ADR 0344)", - errSettingsCannotKeep, node, m, short(kept[m])) - } - if !found && everyBuildKept(ctx, node) { - return fmt.Errorf("%s runs %s's build %s, which the build records no longer hold: the bus's user "+ - "list cannot be sent there alone with it kept (novox/hq issue 490)", node, m, short(kept[m])) - } - if !found && guarded(coreComponent(m)) { - return fmt.Errorf("%s was last sent %s's build %s, newer than the one the mesh holds, and the build records "+ - "no longer hold it: this send cannot keep it and does not move it back — `push %s` sends the older build on "+ - "a person's word, and a newer merge or `rebuild %s` moves the record forward (novox/hq issue 201)", - node, m, short(kept[m]), node, m) - } - if !found { - return fmt.Errorf("%s records rather than rolls out, and %s runs its build %s, which the build "+ - "records no longer hold: this send cannot keep it and does not move it — `push %s` sends the new "+ - "one on a person's word (novox/hq ADR 0245)", m, node, short(kept[m]), node) - } - shelf[m] = ran - } - return nil -} diff --git a/cmd/mesh-controller/recorded_kept_integration_test.go b/cmd/mesh-controller/recorded_kept_integration_test.go deleted file mode 100644 index 8f96201e..00000000 --- a/cmd/mesh-controller/recorded_kept_integration_test.go +++ /dev/null @@ -1,140 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -func TestARecordedBuildIsCarriedOnlyByAPersonsPush(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - start := time.Now().Add(-time.Hour) - pgImage := "registry.invalid:5000/postgres/server@sha256:" + strings.Repeat("a", 64) - mailImage := "registry.invalid:5000/mailu/smtp@sha256:" + strings.Repeat("c", 64) - for _, b := range []link.BuildResult{ - aContainerBuild(t, "postgres", "c1111111", catalogue.PolicyRecord, start, - map[string][2]string{"server": {pgImage, ""}}), - aContainerBuild(t, "mailu", "c1111111", "", start.Add(time.Second), - map[string][2]string{"smtp": {mailImage, ""}, "imap": {mailImage, ""}}), - } { - if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"postgres", "mailu"} { - if _, err := inv.Assign(ctx, "anchor", m); err != nil { - t.Fatal(err) - } - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", - map[string]string{"postgres": "c1111111", "mailu": "c1111111"}); err != nil { - t.Fatal(err) - } - // The merge: both adopt a health check; the images are the same. - for _, b := range []link.BuildResult{ - aContainerBuild(t, "postgres", "c2222222", catalogue.PolicyRecord, start.Add(time.Minute), - map[string][2]string{"server": {pgImage, "runtime"}}), - aContainerBuild(t, "mailu", "c2222222", "", start.Add(time.Minute+time.Second), - map[string][2]string{"smtp": {mailImage, "runtime"}, "imap": {mailImage, "runtime"}}), - } { - if _, _, err := takeIn(ctx, inv, b); err != nil { - t.Fatal(err) - } - } - - healthOf := func(plan catalogue.Resolution, module, id string) any { - for _, m := range plan.Modules { - for _, r := range m.Resources { - if m.Module == module && r["id"] == id { - return r["health"] - } - } - } - return nil - } - - // A plan's, a release plan's, a healer's send: the database is kept as it runs, mail moves. - kept := keepingRecorded(ctx) - plan, _, err := resolveNode(kept, open, "anchor") - if err != nil { - t.Fatal(err) - } - if healthOf(plan, "postgres", "server") != nil { - t.Fatal("a send that is not a person's push composed the recorded module's new build") - } - if healthOf(plan, "mailu", "smtp") == nil { - t.Fatal("the module that rolls out was not composed at its new build") - } - if imageOf(t, plan, "postgres", "server") != pgImage { - t.Fatal("the recorded module's container lost its image") - } - declared, err := composeSend(kept, open, "anchor") - if err != nil { - t.Fatal(err) - } - if declared.Builds["postgres"] != "c1111111" || declared.Builds["mailu"] != "c2222222" { - t.Fatalf("the send records it carries %v; want postgres still at c1111111 and mailu at c2222222", declared.Builds) - } - - // A person's push: the recorded module's new build. - plan, _, err = resolveNode(ctx, open, "anchor") - if err != nil { - t.Fatal(err) - } - if healthOf(plan, "postgres", "server") == nil { - t.Fatal("a person's push did not compose the recorded module's new build") - } - - // The bus step moves the bus alone: a recorded module it is told it may move moves, the others stay. - plan, _, err = resolveNode(keepingRecorded(ctx, "postgres"), open, "anchor") - if err != nil { - t.Fatal(err) - } - if healthOf(plan, "postgres", "server") == nil { - t.Fatal("the module a send is let move was kept") - } - - // What a send composes under (sendToEach): every recorded module kept; on the bus step, the bus moves. - if under, err := sendKeeps(ctx, inv); err != nil { - t.Fatal(err) - } else if skip, on := keptExcept(under); !on || len(skip) != 0 { - t.Fatalf("an ordinary send keeps %v %v", on, skip) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "nats", Version: "2", - Provides: []catalogue.Offer{{Name: "mesh-bus"}}}, inventory.Source{Repository: "novox/mesh-catalog", - Seat: "git", Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil { - t.Fatal(err) - } - if under, err := sendKeeps(withBusStep(ctx), inv); err != nil { - t.Fatal(err) - } else if skip, on := keptExcept(under); !on || !skip["nats"] || len(skip) != 1 { - t.Fatalf("the bus step keeps %v %v; want everything recorded but the bus", on, skip) - } - - // And a recorded module whose kept build the records no longer hold refuses the send, said. - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", - map[string]string{"postgres": "c0000000", "mailu": "c2222222"}); err != nil { - t.Fatal(err) - } - if _, _, err := resolveNode(kept, open, "anchor"); err == nil || !strings.Contains(err.Error(), "push anchor") { - t.Fatalf("a recorded build that cannot be kept did not refuse the send with its remedy: %v", err) - } - - // And the gated send for mail says what it recreates: both containers, no new image, at once. - moves := []inventory.CarriedMove{{Module: "mailu", Node: "anchor", From: "c1111111", To: "c2222222"}} - sayRecreations(ctx, open, moves) - if !strings.Contains(moves[0].Recreates, "recreates 2 of mailu's 2") || !strings.Contains(moves[0].Recreates, "no new image") { - t.Fatalf("the send says %q of mail's containers", moves[0].Recreates) - } - if said := recreationsSaid(moves); !strings.HasPrefix(said, "on anchor recreates") { - t.Fatalf("the plan's note says %q", said) - } -} diff --git a/cmd/mesh-controller/recorded_kept_test.go b/cmd/mesh-controller/recorded_kept_test.go deleted file mode 100644 index 27b6fab1..00000000 --- a/cmd/mesh-controller/recorded_kept_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package main - -import ( - "encoding/json" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/link" -) - -// A recorded build reaches a machine only by a person's push (novox/hq issue 295, ADR 0245). - -// aContainerBuild is a build outcome of a module of containers, each named by id with the image and the -// health it is given; a policy when one is said. -func aContainerBuild(t *testing.T, module, commit, policy string, asked time.Time, containers map[string][2]string) link.BuildResult { - t.Helper() - var resources []any - for id, c := range containers { - r := map[string]any{"id": id, "type": "container", "name": module + "-" + id, "image": c[0]} - if c[1] != "" { - r["health"] = map[string]any{"kind": c[1]} - } - resources = append(resources, r) - } - m := map[string]any{"module": module, "version": "1", "resources": resources} - if policy != "" { - m["upgrade"] = map[string]any{"policy": policy, "why": "a provider whose restart drops every consumer"} - } - manifest, _ := json.Marshal(m) - return link.BuildResult{ID: link.NewBuildID(asked), Repository: "novox/mesh-catalog", Path: "modules/" + module, - On: "anchor", Module: module, Commit: commit, Manifest: manifest, - Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}} -} - -// imageOf is the image the composed plan of a machine gives one of a module's containers. -func imageOf(t *testing.T, plan catalogue.Resolution, module, id string) string { - t.Helper() - for _, m := range plan.Modules { - if m.Module != module { - continue - } - for _, r := range m.Resources { - if r["id"] == id { - image, _ := r["image"].(string) - return image - } - } - } - t.Fatalf("%s has no container %s in the plan", module, id) - return "" -} - -// Tonight's case, 2026-10-07: a catalogue merge adopting the images' own health checks rebuilt the -// database (policy record) with mail (policy roll). The plan's gated send for mail carried the -// database's new build to the control node and recreated it, unjudged, with nobody's word. A send that -// is not a person's push now composes the database at the build the machine runs and records that it -// still carries it; a person's push composes the new one; the bus step moves the bus alone. - -// The send says what it recreates (ADR 0245): mail's health checks adopted recreate both its -// containers, with no new image, every one at once. -func TestASendSaysWhatItRecreates(t *testing.T) { - image := "registry.invalid:5000/mailu/smtp@sha256:" + strings.Repeat("c", 64) - from := catalogue.Manifest{Module: "mailu", Resources: []map[string]any{ - {"id": "smtp", "type": "container", "image": image}, - {"id": "imap", "type": "container", "image": image}, - {"id": "redis", "type": "container", "image": image}, - {"id": "config", "type": "file", "path": "/etc/x"}}} - to := catalogue.Manifest{Module: "mailu", Resources: []map[string]any{ - {"id": "smtp", "type": "container", "image": image, "health": map[string]any{"kind": "runtime"}}, - {"id": "imap", "type": "container", "image": image, "health": map[string]any{"kind": "runtime"}}, - {"id": "redis", "type": "container", "image": image}, - {"id": "config", "type": "file", "path": "/etc/y"}}} - r := catalogue.Recreates(from, to) - if !r.SpecOnly() || len(r.Recreated) != 2 || r.Containers != 3 { - t.Fatalf("recreation %+v", r) - } - said := r.Say("mailu") - for _, want := range []string{"recreates 2 of mailu's 3", "no new image", "imap, smtp", "interrupted"} { - if !strings.Contains(said, want) { - t.Fatalf("%q does not say %q", said, want) - } - } - if catalogue.Recreates(from, from).Say("mailu") != "" { - t.Fatal("a move that recreates nothing said something") - } - to.Resources[2]["image"] = strings.Replace(image, "c", "d", 1) - if r := catalogue.Recreates(from, to); r.SpecOnly() || len(r.Images) != 1 { - t.Fatalf("a new image read as a declaration only: %+v", r) - } -} diff --git a/cmd/mesh-controller/recorded_moves.go b/cmd/mesh-controller/recorded_moves.go deleted file mode 100644 index be6eb18f..00000000 --- a/cmd/mesh-controller/recorded_moves.go +++ /dev/null @@ -1,235 +0,0 @@ -package main - -import ( - "context" - "fmt" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// Under a module's own `record`, a move that recreates no container and moves no data rolls out (novox/hq ADR -// 0377, changing ADR 0242 §1 and ADR 0360 §1). -// -// A module records because its manifest says so, because it keeps irreplaceable data, because a person set it, or -// because it provides the bus. The first two hold for a reason the manifest shows — a change that recreates mail's -// containers interrupts everyone's mail, a new database image may upgrade its databases in place — and that hold -// also held the moves that interrupt nothing: on 2026-10-15 the database server moved with only its tools bundle -// changed, its walk said it "recreates none of its containers", and ace stayed behind until a person pushed. -// -// So **a move under a module's own `record` is judged per machine, against the build that machine was last sent** -// (§2): when it recreates none of the module's containers and moves none of its data (catalogue.Holds, §3) it -// goes as a module that rolls out goes — a walk sends it, and every other send treats it as a move waiting for its -// walk — and otherwise it is kept at the build the machine runs, as before. The module's policy stays `record`. A -// move that cannot be judged waits (§4); a person's `record` and the bus hold every move (§5); and each walk says -// per machine which way a recorded module went (§6). - -// recordedRollsWhy is what a walk says of a recorded module's move that went out (ADR 0377 §6). -const recordedRollsWhy = "went out: it recreates none of its containers and moves none of its data (novox/hq ADR 0377)" - -// recordVerdict is one machine's move of a recorded module: whether it rolls out, and what the walk says of it. -type recordVerdict struct { - rolls bool - // said is what the walk says: recordedRollsWhy, or "waits: " and why — the recreate words, the data change - // named, the person's or the bus's hold, or that it could not be judged. - said string -} - -// recordedMove judges a module's move on a machine from the build it was last sent (`was`) to the one it would be -// sent now. A module that rolls out rolls; one whose policy is a person's or the bus's waits, said; one under its -// own `record` rolls when the move recreates nothing and moves no data, and waits when it does or cannot be judged. -func (f moveFacts) recordedMove(module, was string) recordVerdict { - now := f.current[module] - switch { - case now.RollOut: - return recordVerdict{rolls: true} - case now.From == catalogue.FromPerson: - return recordVerdict{said: "waits: a person set it to record, and holds every move of it until they push (novox/hq ADR 0377 §5)"} - case now.From == catalogue.FromBus: - return recordVerdict{said: "waits: it provides the bus, which moves only by the bus step a person starts (novox/hq ADR 0377 §5)"} - case !now.OwnRecord(): - return recordVerdict{said: "waits: its policy records"} - case f.judge == nil: - return recordVerdict{said: "waits: it could not be judged — the build records were not read (novox/hq ADR 0377 §4)"} - } - return f.judge(module, was, now.Commit) -} - -// rolls is whether a module's move on a machine from `was` goes as a move that rolls out (ADR 0377 §1). -func (f moveFacts) rolls(module, was string) bool { - return f.recordedMove(module, was).rolls -} - -// recordJudge is the judging of moves under a module's own `record`, reading the build records once per move and -// the catalogue once (ADR 0377 §2–§4). -func recordJudge(ctx context.Context, inv *inventory.Inventory) func(module, was, now string) recordVerdict { - judged := map[string]recordVerdict{} - var shelf map[string]catalogue.Manifest - var shelfErr error - shelfRead := false - return func(module, was, now string) recordVerdict { - key := module + "\x00" + was + "\x00" + now - if v, seen := judged[key]; seen { - return v - } - v := func() recordVerdict { - cannot := func(why string) recordVerdict { - return recordVerdict{said: "waits: it could not be judged — " + why + " (novox/hq ADR 0377 §4)"} - } - if was == "" { - return cannot("what the machine was last sent of it is not known") - } - from, found, err := inv.ManifestAt(ctx, module, was) - if err != nil { - return cannot(fmt.Sprintf("its build %s, which the machine was last sent, cannot be read: %v", short(was), err)) - } - if !found { - return cannot(fmt.Sprintf("its build %s, which the machine was last sent, is not in the build records", short(was))) - } - if !shelfRead { - shelf, shelfErr = inv.Catalogue(ctx) - shelfRead = true - } - if shelfErr != nil { - return cannot(fmt.Sprintf("the build it would be sent cannot be read: %v", shelfErr)) - } - to, held := shelf[module] - if !held { - return cannot("the build it would be sent is not in the catalogue") - } - if why := catalogue.Holds(from, to); why != "" { - return recordVerdict{said: "waits: " + why} - } - return recordVerdict{rolls: true, said: recordedRollsWhy} - }() - judged[key] = v - return v - } -} - -// judgeRecordedInWalk judges, for a walk carrying a module whose policy records, each machine running it whose -// build would move (ADR 0377 §2), keeps on the plan which way each went (§6), says it on the controller's log, and -// answers the machines its move goes out to. Judged once per walk: a walk read again answers what it kept. -func judgeRecordedInWalk(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan, module string, - state *inventory.PlanModule) ([]string, error) { - if state.Recorded != nil { - return wentOut(state.Recorded), nil - } - running, err := inv.Running(ctx, module) - if err != nil { - return nil, err - } - f, err := readMoveFacts(ctx, inv) - if err != nil { - return nil, err - } - now := f.current[module] - said := map[string]string{} - for _, n := range running { - sent, known, err := inv.SentBuilds(ctx, n) - if err != nil { - return nil, err - } - was, carried := sent[module] - if known && carried && f.identical(module, was, now.Commit) { - continue // no move there - } - v := f.recordedMove(module, was) - if !known { - v = recordVerdict{said: "waits: it could not be judged — what " + n + " was last sent is not known (novox/hq ADR 0377 §4)"} - } - said[n] = v.said - fmt.Printf("%s: %s on %s %s\n", p.ID, module, n, v.said) - } - if len(said) > 0 { - state.Recorded = said - } - return wentOut(said), nil -} - -// wentOut is the machines a recorded module's move went out to, sorted. -func wentOut(recorded map[string]string) []string { - var out []string - for _, n := range sortedKeysOf(recorded) { - if recorded[n] == recordedRollsWhy { - out = append(out, n) - } - } - return out -} - -// recordedSaid is, for a walk's note at its end, which way each recorded module it carried went on each machine -// (ADR 0377 §6). -func recordedSaid(p inventory.Plan) []string { - var out []string - for _, m := range sortedKeysOf(planModuleNames(p)) { - s := p.Modules[m] - for _, n := range sortedKeysOf(s.Recorded) { - out = append(out, fmt.Sprintf("%s on %s %s", m, n, s.Recorded[n])) - } - } - return out -} - -func planModuleNames(p inventory.Plan) map[string]string { - out := map[string]string{} - for m, s := range p.Modules { - if s != nil && len(s.Recorded) > 0 { - out[m] = m - } - } - return out -} - -// heldMove is one move of a recorded module that waits for a person on one machine (ADR 0377 §6). -type heldMove struct { - Module string - From, To string - // Why is the walk's own word for it: "waits: " and the recreate words, the data change, or the hold. - Why string - // Chosen is whether a person's own choice holds it — a module they set to record, or the bus's step — which the - // operator is not asked about again: they chose that hold, and the bus has its own condition (S17). - Chosen bool -} - -// heldMovesOn is, on one machine, every move of a recorded module that waits for a person, in module order. -func heldMovesOn(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]heldMove, error) { - sent, known, err := inv.SentBuilds(ctx, node) - if err != nil || !known { - return nil, err - } - var out []heldMove - for _, m := range sortedKeysOf(sent) { - was := sent[m] - now, held := f.current[m] - if !held || now.RollOut || now.Commit == "" || was == "" || f.identical(m, was, now.Commit) { - continue - } - if v := f.recordedMove(m, was); !v.rolls { - out = append(out, heldMove{Module: m, From: was, To: now.Commit, Why: v.said, - Chosen: now.From == catalogue.FromPerson || now.From == catalogue.FromBus}) - } - } - return out, nil -} - -// recordedWaiting is, on one machine, every move of a recorded module that waits for a person, each said with why -// (ADR 0377 §6): what `status` names the machine behind for, with `push ` as the remedy. A move under a -// module's own `record` that recreates nothing and moves no data is not among them: it goes with its walk. -func recordedWaiting(ctx context.Context, inv *inventory.Inventory, f moveFacts, node string) ([]string, error) { - moves, err := heldMovesOn(ctx, inv, f, node) - var out []string - for _, h := range moves { - out = append(out, fmt.Sprintf("%s %s → %s %s", h.Module, short(h.From), short(h.To), h.Why)) - } - return out, err -} - -// stringsKeys is a map of lists as a map of its keys, for sortedKeysOf. -func stringsKeys(m map[string][]string) map[string]string { - out := map[string]string{} - for k := range m { - out[k] = k - } - return out -} diff --git a/cmd/mesh-controller/recorded_moves_test.go b/cmd/mesh-controller/recorded_moves_test.go deleted file mode 100644 index 64b705e3..00000000 --- a/cmd/mesh-controller/recorded_moves_test.go +++ /dev/null @@ -1,363 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "fmt" - "slices" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// Under a module's own `record`, a move that recreates no container and moves no data rolls out (novox/hq ADR -// 0377). Seen 2026-10-15: the database server moved with only its tools bundle changed, its walk said it -// "recreates none of its containers", and ace stayed behind until a person pushed. - -// aMailu is a mail module as the catalogue declares it, small: it records because its manifest says so, keeps its -// mailboxes as data, mounts its environment file, and serves its tools from a bundle. -func aMailu(policy bool) catalogue.Manifest { - raw := `{"module":"mailu","version":"1","tools":["mailu_status"], - "data":{"own":[{"id":"mail","path":"${dir:data-mail}","class":"valuable","why":"every mailbox"}]}, - "bundles":[{"name":"tools","source":"store/mailu-tools@sha256:aaa","digest":"sha256:aaa"}], - "resources":[ - {"id":"state","type":"directory"}, - {"id":"mesh-state","type":"directory"}, - {"id":"data-mail","type":"directory"}, - {"id":"config-env","type":"file","path":"${dir:state}/mailu.env","content":"A=1"}, - {"id":"runtime-config","type":"file","path":"${dir:mesh-state}/config.json","content":"{}"}, - {"id":"imap","type":"container","name":"mailu-imap","image":"registry.invalid/imap@sha256:1", - "volumes":["${dir:data-mail}:/mail"],"env-file":["${dir:state}/mailu.env"]}]}` - var m catalogue.Manifest - if err := json.Unmarshal([]byte(raw), &m); err != nil { - panic(err) - } - if policy { - m.Upgrade = &catalogue.UpgradePolicy{Policy: catalogue.PolicyRecord, Why: "people's mail: a person chooses the moment"} - } - return m -} - -func resourceOf(m *catalogue.Manifest, id string) map[string]any { - for _, r := range m.Resources { - if r["id"] == id { - return r - } - } - panic("no resource " + id) -} - -// recordedMesh is a mesh running mailu at build c1 on anchor and laptop, as each was last sent. -type recordedMesh struct { - open *stores - sent [][]string -} - -func aRecordedMesh(t *testing.T, c1 catalogue.Manifest) *recordedMesh { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - rm := &recordedMesh{open: open} - rm.register(t, c1, "c1", time.Now().Add(-2*time.Hour)) - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, c1.Module); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{c1.Module: "c1"}); err != nil { - t.Fatal(err) - } - } - return rm -} - -// register keeps a build of a module and makes it the mesh's. -func (rm *recordedMesh) register(t *testing.T, m catalogue.Manifest, commit string, at time.Time) { - t.Helper() - ctx := t.Context() - raw, _ := json.Marshal(m) - if err := rm.open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-" + m.Module + "-" + commit, Module: m.Module, - Commit: commit, Repository: "novox/mesh-catalog", Path: "modules/" + m.Module, Manifest: raw, Asked: at, At: at}); err != nil { - t.Fatal(err) - } - if err := rm.open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", - Path: "modules/" + m.Module, BuiltFrom: commit, Head: commit, Asked: at}); err != nil { - t.Fatal(err) - } -} - -// keptOn is what an ordinary send — a walk's, a healer's — keeps on a machine: module → the build it keeps. -func (rm *recordedMesh) keptOn(t *testing.T, node string) map[string]string { - t.Helper() - ctx := t.Context() - under, err := sendKeeps(ctx, rm.open.inventory) - if err != nil { - t.Fatal(err) - } - kept, err := recordedKept(under, rm.open, node) - if err != nil { - t.Fatal(err) - } - return kept -} - -// walk opens a merge's walk of mailu's c2, built, and walks it with every send composed as an ordinary send -// composes it — a recorded module kept where its move waits — and applied at once. -func (rm *recordedMesh) walk(t *testing.T) inventory.Plan { - t.Helper() - ctx := t.Context() - inv := rm.open.inventory - wasSend := sendRollout - n := 0 - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - rm.sent = append(rm.sent, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - kept := rm.keptOn(t, node) - builds := map[string]string{"mailu": current["mailu"].Commit} - if was, held := kept["mailu"]; held { - builds["mailu"] = was - } - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := inv.RecordSent(ctx, nodeID(t, open, node), digest, builds); err != nil { - return nil, err - } - if _, err := inv.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { - return nil, err - } - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - built := time.Now().UTC() - plan := inventory.Plan{ID: "plan-mailu", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", - Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"mailu"}}, - Modules: map[string]*inventory.PlanModule{"mailu": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-mailu-c2"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - for i := 0; i < 8; i++ { - advancePlans(ctx, rm.open) - } - p, err := inv.PlanByID(ctx, "plan-mailu") - if err != nil { - t.Fatal(err) - } - return p -} - -func (rm *recordedMesh) sentBuild(t *testing.T, node string) string { - t.Helper() - sent, _, err := rm.open.inventory.SentBuilds(t.Context(), node) - if err != nil { - t.Fatal(err) - } - return sent["mailu"] -} - -// recordedWaitsOn is what `status` names a machine behind for, waiting for a person (ADR 0377 §6). -func (rm *recordedMesh) recordedWaitsOn(t *testing.T, node string) []string { - t.Helper() - f, err := readMoveFacts(t.Context(), rm.open.inventory) - if err != nil { - t.Fatal(err) - } - held, err := recordedWaiting(t.Context(), rm.open.inventory, f, node) - if err != nil { - t.Fatal(err) - } - return held -} - -// §1, §6: a module that records because its manifest says so, whose move changes only its tools bundle, goes out -// with its walk to every machine, without a push; the walk says per machine that it went out and why; the policy -// still reads record; and status names no machine behind for it. -func TestARecordedModulesBundleOnlyMoveRollsOutInItsWalk(t *testing.T) { - c1 := aMailu(true) - rm := aRecordedMesh(t, c1) - c2 := aMailu(true) - c2.Bundles[0].Digest, c2.Bundles[0].Source = "sha256:bbb", "store/mailu-tools@sha256:bbb" - c2.Tools = append(c2.Tools, "mailu_queue") - rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) - - for _, n := range []string{"anchor", "laptop"} { - if _, kept := rm.keptOn(t, n)["mailu"]; kept { - t.Fatalf("an ordinary send to %s keeps mailu at c1 for a move that recreates nothing", n) - } - } - p := rm.walk(t) - if p.State != inventory.PlanDone { - t.Fatalf("the walk is %s: %s", p.State, p.Note) - } - for _, n := range []string{"anchor", "laptop"} { - if got := rm.sentBuild(t, n); got != "c2" { - t.Fatalf("%s was sent mailu %s; want c2, with its walk", n, got) - } - if said := p.Modules["mailu"].Recorded[n]; said != recordedRollsWhy { - t.Fatalf("the walk says of mailu on %s %q", n, said) - } - if held := rm.recordedWaitsOn(t, n); len(held) != 0 { - t.Fatalf("status names %s behind for %v", n, held) - } - } - if !strings.Contains(p.Note, "mailu on anchor went out") || !strings.Contains(p.Note, "mailu on laptop went out") { - t.Fatalf("the walk's note does not say which way mailu went: %q", p.Note) - } - if u, err := rm.open.inventory.UpgradeOf(t.Context(), "mailu"); err != nil || u.Policy() != catalogue.PolicyRecord { - t.Fatalf("mailu's policy is %q (%v); want record still", u.Policy(), err) - } -} - -// §1, §2, §6: a move that recreates a container — a new image, or a changed declaration — still waits for a push; -// the walk says per machine that it waits, with the recreate words; and status names the machine behind, with -// `push ` as the remedy. A bundle-only change on top of it stays held: it is judged against what the machine -// was last sent. -func TestARecordedModulesMoveThatRecreatesAContainerWaits(t *testing.T) { - for name, change := range map[string]func(m *catalogue.Manifest){ - "image": func(m *catalogue.Manifest) { resourceOf(m, "imap")["image"] = "registry.invalid/imap@sha256:2" }, - "spec": func(m *catalogue.Manifest) { resourceOf(m, "imap")["health"] = map[string]any{"kind": "runtime"} }, - } { - t.Run(name, func(t *testing.T) { - rm := aRecordedMesh(t, aMailu(true)) - c2 := aMailu(true) - change(&c2) - c2.Bundles[0].Digest = "sha256:bbb" - rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) - p := rm.walk(t) - for _, n := range []string{"anchor", "laptop"} { - if got := rm.sentBuild(t, n); got != "c1" { - t.Fatalf("%s was sent mailu %s; want c1 kept, waiting for a push", n, got) - } - if said := p.Modules["mailu"].Recorded[n]; !strings.HasPrefix(said, "waits: recreates 1 of mailu's 1") { - t.Fatalf("the walk says of mailu on %s %q", n, said) - } - if held := rm.recordedWaitsOn(t, n); len(held) != 1 || !strings.Contains(held[0], "recreates") { - t.Fatalf("status names %s behind for %v", n, held) - } - } - // §2: the next build changes only the bundle, and the machines were last sent c1: still held. - c3 := c2 - c3.Bundles = []catalogue.Bundle{{Name: "tools", Source: "store/mailu-tools@sha256:ccc", Digest: "sha256:ccc"}} - rm.register(t, c3, "c3", time.Now()) - if kept := rm.keptOn(t, "anchor"); kept["mailu"] != "c1" { - t.Fatalf("a bundle-only build on top of a held container change is not held: %v", kept) - } - }) - } -} - -// §3: a move touching a file a container mounts, a directory holding declared data, the data declaration or a -// provider waits. -func TestARecordedModulesMoveThatTouchesDataOrAProviderWaits(t *testing.T) { - for name, c := range map[string]struct { - change func(m *catalogue.Manifest) - says string - }{ - "a mounted file": {func(m *catalogue.Manifest) { resourceOf(m, "config-env")["content"] = "A=2" }, "which its container imap mounts"}, - "a data directory": {func(m *catalogue.Manifest) { resourceOf(m, "data-mail")["mode"] = "0700" }, "data-mail"}, - "the data declaration": {func(m *catalogue.Manifest) { - m.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Path: "${dir:data-mail}", Class: "irreplaceable", Why: "x"}}} - }, "data declaration"}, - "a provider": {func(m *catalogue.Manifest) { m.Requires = append(m.Requires, "postgres-database") }, "what it requires"}, - } { - t.Run(name, func(t *testing.T) { - rm := aRecordedMesh(t, aMailu(true)) - c2 := aMailu(true) - c.change(&c2) - rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) - kept := rm.keptOn(t, "anchor") - if kept["mailu"] != "c1" { - t.Fatalf("an ordinary send does not keep mailu at c1: %v", kept) - } - held := rm.recordedWaitsOn(t, "anchor") - if len(held) != 1 || !strings.Contains(held[0], c.says) { - t.Fatalf("status says %v; want it to name %q", held, c.says) - } - }) - } -} - -// §5: a module a person set to record, and the bus, wait on a bundle-only move. -func TestAPersonsRecordAndTheBusHoldEveryMove(t *testing.T) { - t.Run("a person sets record", func(t *testing.T) { - rm := aRecordedMesh(t, aMailu(false)) - c2 := aMailu(false) - c2.Bundles[0].Digest = "sha256:bbb" - rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) - if err := rm.open.inventory.SetUpgradeOf(t.Context(), "mailu", inventory.Upgrade{Why: "the migration week", By: "jochen"}); err != nil { - t.Fatal(err) - } - p := rm.walk(t) - if got := rm.sentBuild(t, "anchor"); got != "c1" { - t.Fatalf("a person's record was walked past: anchor was sent %s", got) - } - if said := p.Modules["mailu"].Recorded["anchor"]; !strings.Contains(said, "a person set it to record") { - t.Fatalf("the walk says %q", said) - } - if held := rm.recordedWaitsOn(t, "anchor"); len(held) != 1 { - t.Fatalf("status names anchor behind for %v", held) - } - }) - t.Run("the bus", func(t *testing.T) { - bus := aMailu(false) - bus.Module = "nats" - bus.Provides = []catalogue.Offer{{Name: "mesh-bus"}} - bus.Data = nil - rm := aRecordedMesh(t, bus) - c2 := bus - c2.Bundles = []catalogue.Bundle{{Name: "tools", Source: "store/nats-tools@sha256:bbb", Digest: "sha256:bbb"}} - rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) - if kept := rm.keptOn(t, "anchor"); kept["nats"] != "c1" { - t.Fatalf("the bus moved by an ordinary send: %v", kept) - } - f, err := readMoveFacts(t.Context(), rm.open.inventory) - if err != nil { - t.Fatal(err) - } - if v := f.recordedMove("nats", "c1"); v.rolls || !strings.Contains(v.said, "bus") { - t.Fatalf("the bus's move: %+v", v) - } - }) -} - -// §4: a move whose earlier build is not in the build records cannot be judged, waits, and is said so. -func TestARecordedMoveThatCannotBeJudgedWaits(t *testing.T) { - rm := aRecordedMesh(t, aMailu(true)) - c2 := aMailu(true) - c2.Bundles[0].Digest = "sha256:bbb" - rm.register(t, c2, "c2", time.Now().Add(-time.Minute)) - // anchor was last sent a build the records do not hold. - if err := rm.open.inventory.RecordSent(t.Context(), nodeID(t, rm.open, "anchor"), "d-anchor-c0", - map[string]string{"mailu": "c0"}); err != nil { - t.Fatal(err) - } - f, err := readMoveFacts(t.Context(), rm.open.inventory) - if err != nil { - t.Fatal(err) - } - if v := f.recordedMove("mailu", "c0"); v.rolls || !strings.Contains(v.said, "could not be judged") || - !strings.Contains(v.said, "not in the build records") { - t.Fatalf("an unjudgeable move: %+v", v) - } - if _, _, err := resolveNode(keepingRecorded(t.Context()), rm.open, "anchor"); err == nil || !strings.Contains(err.Error(), "push anchor") { - t.Fatalf("a send that cannot keep the unjudged build did not refuse with its remedy: %v", err) - } - // laptop, judged from c1, goes; anchor waits. - p := rm.walk(t) - if said := p.Modules["mailu"].Recorded["anchor"]; !strings.Contains(said, "could not be judged") { - t.Fatalf("the walk says of anchor %q", said) - } - if !slices.Contains(wentOut(p.Modules["mailu"].Recorded), "laptop") || rm.sentBuild(t, "laptop") != "c2" { - t.Fatalf("laptop, judged from c1, did not go: %+v", p.Modules["mailu"].Recorded) - } -} diff --git a/cmd/mesh-controller/recorded_push.go b/cmd/mesh-controller/recorded_push.go deleted file mode 100644 index c6ca5de3..00000000 --- a/cmd/mesh-controller/recorded_push.go +++ /dev/null @@ -1,153 +0,0 @@ -package main - -import ( - "context" - "fmt" - "sort" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A push of recorded builds is no repair (novox/hq issue 301, ADR 0242, to-be 45 §7). -// -// **A recorded build moves only by a person's push** (ADR 0242): a module whose upgrade policy is -// `record` — the network path, the providers whose restart costs, mail — has its new build registered -// at its merge and sent by nothing but `push `. That push is the person's word the policy asks -// for: the mesh working as decided. The hand-act log counted it as a repair all the same, because a push -// by hand is the repair roll-out by default exists to end. On 2026-10-07 the operator approved, node by -// node, new builds of the resolver, the FortiClient adapter, the network managers and the packet filter; -// ten pushes later S15 wanted a healer for `split-dns`, `words` and `uplink-verbs`. -// -// **The push says what it was, from what it carried.** Never from a word the person gives — that is -// how a repair could pass for anything (issue 292). Before it is recorded, a push that names one machine -// reads what that machine was last sent against what the mesh holds. It is a **push of recorded builds** -// when: -// -// - what the machine was last sent is known; -// - its last report is no failure or refusal, so the push does not re-send to mend one; -// - it moves at least one module's build; and -// - every module it moves records rather than rolls out: a held new build of one the machine runs, or -// the first build of one newly assigned there (whose `assign` says "run `push ` to send it"). -// -// Anything else — a build that rolls out, a module taken off, nothing moved at all — is a push by hand -// as before, and counts. The act is written with `kind: recorded-builds` and what it carried; the table -// of verbs (handActVerbs) reads a push of that kind as a person's decision, and S15 never counts it. -// -// What is not compared: settings and grants. A push of a recorded build carries whatever else -// changed in the machine's declaration with it, as any push does. - -// recordedPush answers what a push to one machine carries, one "module from → to" each, when every -// build it moves is a recorded one held for a person's word; otherwise nil and why it is not. -// -// `modules` is the machine's set; `sent` and `known` what it was last sent (Inventory.SentBuilds); -// `current` what the mesh holds; `identical` whether two builds of a module put the same thing on a -// machine (moveFacts.identical); `failing` the machine's last report when it was a failure or a refusal. -func recordedPush(modules []string, sent map[string]string, known bool, current map[string]inventory.CurrentBuild, - identical func(module, a, b string) bool, failing string) ([]string, string) { - if !known { - return nil, "what the machine was last sent is not known" - } - if failing != "" { - return nil, "its last report was " + failing + ": the push sends again what failed" - } - in := map[string]bool{} - var carried []string - for _, m := range modules { - in[m] = true - // A module the mesh holds no build of, or holds without a source, has no build to move. - now, held := current[m] - if !held || now.Commit == "" { - continue - } - was, ran := sent[m] - if ran && identical(m, was, now.Commit) { - continue - } - if now.RollOut { - return nil, fmt.Sprintf("%s would move %sto %s, and it rolls out: its build is a plan's to send", m, - fromBuild(was, ran), buildName(now.Commit)) - } - carried = append(carried, fmt.Sprintf("%s %s→ %s", m, fromBuild(was, ran), buildName(now.Commit))) - } - for m := range sent { - if !in[m] { - return nil, m + " is taken off the machine" - } - } - if len(carried) == 0 { - return nil, "it moves no build: a send again" - } - sort.Strings(carried) - return carried, "" -} - -// recordedPushOf reads, for a push about to name one machine, whether it is a push of recorded builds: what -// it carries, -// or nil and why not. An error is that it could not be read; the push is then recorded as a push by hand, -// as every push was before, and says why. -func recordedPushOf(ctx context.Context, node string) ([]string, string, error) { - open, err := openStores(ctx) - if err != nil { - return nil, "", err - } - defer open.Close() - inv := open.inventory - plan, _, err := resolveNode(ctx, open, node) - if err != nil { - return nil, "", fmt.Errorf("its set cannot be worked out: %w", err) - } - modules := make([]string, 0, len(plan.Modules)) - for _, m := range plan.Modules { - modules = append(modules, m.Module) - } - sent, known, err := inv.SentBuilds(ctx, node) - if err != nil { - return nil, "", err - } - f, err := readMoveFacts(ctx, inv) - if err != nil { - return nil, "", err - } - doing, said, err := inv.DoingOf(ctx, node) - if err != nil { - return nil, "", err - } - failing := "" - if said && (doing.Outcome == inventory.OutcomeFailed || doing.Outcome == inventory.OutcomeRefused) { - failing = doing.Outcome - } - carried, why := recordedPush(modules, sent, known, f.current, f.identical, failing) - return carried, why, nil -} - -// recordedBefore are the pushes recorded before a push said its kind, each a push of recorded builds by -// what it carried (novox/hq issue 301): the operator's word, machine by machine, for new builds of modules -// whose upgrade policy records — systemd-resolved, forticlient, networkmanager, systemd-networkd, -// nftables, mailu. The log did not keep what a push carried then, so the record names them; no push -// recorded since needs it, because every push now says its kind. S15 reads these as it reads a push of -// that kind, and the three `healer-wanted` they opened clear on the next tick. -var recordedBefore = map[string]string{ - "act-1791404241010309198-1": "systemd-resolved, first build on its machine (ADR 0247)", - "act-1791404587689907257-1": "systemd-resolved, first build; nftables, held build (catalogue #111)", - "act-1791404809925218992-1": "nftables, held build (catalogue #111)", - "act-1791404844504887009-1": "mailu and nftables, held builds (catalogue #106, #111)", - "act-1791408072745552019-1": "forticlient, held build (catalogue #114)", - "act-1791408101286318350-1": "forticlient, held build (catalogue #114)", - "act-1791409209593713522-1": "networkmanager, held build (catalogue #107)", - "act-1791409280125022264-1": "networkmanager, held build (catalogue #107)", - "act-1791409336586475835-1": "networkmanager, held build (catalogue #107)", - "act-1791409393494198352-1": "systemd-networkd, held build (catalogue #107)", -} - -// pushedRecorded is whether an act in the log is a push of recorded builds. -func pushedRecorded(a link.HandAct) bool { - if a.Verb != "push" { - return false - } - if a.Kind == link.KindRecordedBuilds { - return true - } - _, before := recordedBefore[a.ID] - return before && a.Kind == "" -} diff --git a/cmd/mesh-controller/recorded_push_test.go b/cmd/mesh-controller/recorded_push_test.go deleted file mode 100644 index 599cf43d..00000000 --- a/cmd/mesh-controller/recorded_push_test.go +++ /dev/null @@ -1,124 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// **A push says whether it pushed recorded builds only, from what it carries** (novox/hq issue 301): -// every module it moves records rather than rolls out, the machine's last send is known and its last -// report no failure. Anything else is a push by hand, as before. -func TestAPushIsOfRecordedBuildsOnlyWhenEveryBuildItMovesWasHeldForAPersonsWord(t *testing.T) { - current := map[string]inventory.CurrentBuild{ - "resolver": {Commit: "r2222222"}, "nftables": {Commit: "n2222222"}, "mail": {Commit: "m2222222"}, - "web": {Commit: "w2222222", RollOut: true}, "proxy": {Commit: "p1111111", RollOut: true}, - } - same := func(_, a, b string) bool { return a == b } - for _, c := range []struct { - name string - modules []string - sent map[string]string - known bool - failing string - carried string // what it carried, joined; empty when it is not a push of recorded builds - why string - }{ - {name: "one held build", modules: []string{"resolver", "proxy"}, - sent: map[string]string{"resolver": "r1111111", "proxy": "p1111111"}, known: true, - carried: "resolver from r1111111 → r2222222"}, - {name: "two held builds", modules: []string{"mail", "nftables", "proxy"}, - sent: map[string]string{"mail": "m1111111", "nftables": "n1111111", "proxy": "p1111111"}, known: true, - carried: "mail from m1111111 → m2222222; nftables from n1111111 → n2222222"}, - {name: "a recorded module's first build, newly assigned", modules: []string{"resolver", "proxy"}, - sent: map[string]string{"proxy": "p1111111"}, known: true, - carried: "resolver (never sent it) → r2222222"}, - {name: "a build that rolls out moves too", modules: []string{"resolver", "web"}, - sent: map[string]string{"resolver": "r1111111", "web": "w1111111"}, known: true, why: "rolls out"}, - {name: "nothing moves", modules: []string{"resolver", "proxy"}, - sent: map[string]string{"resolver": "r2222222", "proxy": "p1111111"}, known: true, why: "a send again"}, - {name: "a module taken off", modules: []string{"resolver"}, - sent: map[string]string{"resolver": "r1111111", "proxy": "p1111111"}, known: true, why: "taken off"}, - {name: "what it was last sent is not known", modules: []string{"resolver"}, why: "not known"}, - {name: "its last report failed", modules: []string{"resolver"}, - sent: map[string]string{"resolver": "r1111111"}, known: true, failing: "failed", why: "failed"}, - } { - carried, why := recordedPush(c.modules, c.sent, c.known, current, same, c.failing) - if got := strings.Join(carried, "; "); got != c.carried { - t.Errorf("%s: carried %q, want %q (%s)", c.name, got, c.carried, why) - } - if c.why != "" && !strings.Contains(why, c.why) { - t.Errorf("%s: not of recorded builds because %q, want it to say %q", c.name, why, c.why) - } - } -} - -// **The live case, as the log holds it** (novox/hq issue 301). The ten pushes of 2026-10-07 recorded -// before a push said its kind pushed recorded builds only; the three `healer-wanted` they opened clear. -// The five pushes for the settings beside them, and any push that says no kind, still count; and a push -// recorded as one of recorded builds since, repeated, never does. -func TestThePushesOfRecordedBuildsOf20261007WantNoHealer(t *testing.T) { - now := time.Date(2026, 10, 8, 9, 0, 0, 0, time.UTC) - pushed := func(id, cause string, at time.Time) link.HandAct { - return link.HandAct{ID: id, At: at, By: "operator", Verb: "push", Args: []string{"laptop"}, - Why: "operator's yes", Cause: cause} - } - var acts []link.HandAct - at := now.Add(-12 * time.Hour) - causes := map[string]string{ - "act-1791404241010309198-1": "split-dns", "act-1791404587689907257-1": "split-dns", - "act-1791408072745552019-1": "split-dns", "act-1791408101286318350-1": "split-dns", - "act-1791404809925218992-1": "words", "act-1791404844504887009-1": "words", - "act-1791409209593713522-1": "uplink-verbs", "act-1791409280125022264-1": "uplink-verbs", - "act-1791409336586475835-1": "uplink-verbs", "act-1791409393494198352-1": "uplink-verbs", - } - if len(causes) != len(recordedBefore) { - t.Fatalf("the live case has %d acts and the record names %d", len(causes), len(recordedBefore)) - } - for id, cause := range causes { - if _, named := recordedBefore[id]; !named { - t.Fatalf("%s is not named a push of recorded builds", id) - } - acts = append(acts, pushed(id, cause, at)) - at = at.Add(time.Minute) - } - for i := range 2 { - acts = append(acts, pushed("act-settings-"+string(rune('a'+i)), "push", at)) - at = at.Add(time.Minute) - } - f := calm(now) - f.handActs = acts - got := watchHandActs(f) - if len(got) != 1 || got[0].ID != "hand-acts.push" { - t.Fatalf("S15 saw %+v; want only the settings pushes, counted as before", got) - } - - // A push that says it carried recorded builds, repeated, wants no healer; one recorded with the same word and no - // kind still counts, as does an act of another verb whose ID such a push once had. - ofRecorded := func(at time.Time) link.HandAct { - a := pushed("", "split-dns", at) - a.Kind, a.Carried = link.KindRecordedBuilds, []string{"resolver from r1111111 → r2222222"} - return a - } - f = calm(now) - f.handActs = []link.HandAct{ofRecorded(now.Add(-26 * time.Hour)), ofRecorded(now.Add(-time.Hour)), - ofRecorded(now.Add(-time.Minute))} - if got := watchHandActs(f); len(got) != 0 { - t.Errorf("pushes of recorded builds repeated asked for a healer: %+v", got) - } - f = calm(now) - f.handActs = []link.HandAct{pushed("a", "split-dns", now.Add(-time.Hour)), pushed("b", "split-dns", now.Add(-time.Minute))} - if got := watchHandActs(f); len(got) != 1 { - t.Errorf("pushes that said no kind were not counted: %+v", got) - } - other := actOf(now.Add(-time.Hour), "plans close", "split-dns") - other.ID = "act-1791404241010309198-1" - f = calm(now) - f.handActs = []link.HandAct{other, actOf(now.Add(-time.Minute), "plans close", "split-dns")} - if got := watchHandActs(f); len(got) != 1 { - t.Errorf("another verb under such a push's ID passed for one: %+v", got) - } -} diff --git a/cmd/mesh-controller/related.go b/cmd/mesh-controller/related.go deleted file mode 100644 index 7c507af1..00000000 --- a/cmd/mesh-controller/related.go +++ /dev/null @@ -1,195 +0,0 @@ -package main - -import ( - "context" - "sort" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// What relates two changes (novox/hq ADR 0304 decision 1, as ADR 0306 decision 3 reads it): two changes are -// related when they share a repository, a module, a delivery group's head, or an edge of the module graph -// between their modules. A module is every one the walk builds or sends: the modules its merges moved and the -// modules that depend on them, not only the ones the merges touched. Unrelated changes may walk beside each -// other (ADR 0304 decision 2, ADR 0282 decision 3); a closed batch is cut beside an open walk it is -// not related to, and a walk about to ask its first tier waits only for a started walk it is related to (issue 973, -// ADR 0407 decision 4). - -// footprint is what one change reaches: its repositories (lower case), its modules, and its delivery group -// heads (the head branch names of its merges). unknown says a merge of it was kept without the modules it -// moves, so what it reaches is not known: it is related to every change, never wrongly to none. -type footprint struct { - repos, modules, heads map[string]bool - unknown string -} - -func newFootprint() footprint { - return footprint{repos: map[string]bool{}, modules: map[string]bool{}, heads: map[string]bool{}} -} - -// footprintOfWalk is a walk's footprint: the repositories it carries and the modules it plans. A walk keeps no -// head of its merges; kept are the merges the record holds for it, whose announcements say their heads, and -// may be nil. -func footprintOfWalk(p inventory.Plan, kept []inventory.BatchedMerge) footprint { - f := newFootprint() - for _, c := range p.Carried() { - f.repos[strings.ToLower(c.Repository)] = true - } - for name := range p.Modules { - f.modules[name] = true - } - for _, m := range kept { - f.repos[strings.ToLower(m.Repository)] = true - if h := announcedOf(m).Head; h != "" { - f.heads[h] = true - } - } - return f -} - -// footprintOfBatch is an assembling batch's footprint: each merge's repository and head, and the modules its -// merges moved when heard, widened to their dependents (planOfMoves). A merge kept with no moves (one owed a -// later look, or kept for its packaging) is judged again at the cut and may move anything then: its batch is -// unknown, related to everything. -func footprintOfBatch(merges []inventory.BatchedMerge, edges []inventory.Edge) footprint { - f := newFootprint() - var moved []string - for _, m := range merges { - k := announcedOf(m) - f.repos[strings.ToLower(m.Repository)] = true - if k.Head != "" { - f.heads[k.Head] = true - } - if len(k.Moves) == 0 && f.unknown == "" { - f.unknown = "the modules a merge of " + m.Repository + " moves are not known until its batch is cut" - } - moved = append(moved, k.Moves...) - } - for _, name := range reachableFrom(moved, edges) { - f.modules[name] = true - } - return f -} - -// related says whether two changes are related, and the first reason in plain words; false and empty when -// they are not. -func related(a, b footprint, edges []inventory.Edge) (bool, string) { - if a.unknown != "" { - return true, a.unknown - } - if b.unknown != "" { - return true, b.unknown - } - if r := firstShared(a.repos, b.repos); r != "" { - return true, "both change " + r - } - if m := firstShared(a.modules, b.modules); m != "" { - return true, "both move " + m - } - if h := firstShared(a.heads, b.heads); h != "" { - return true, "both are members of group " + h - } - for _, e := range edges { - if a.modules[e.From] && b.modules[e.To] || b.modules[e.From] && a.modules[e.To] { - return true, e.From + " depends on " + e.To + " (" + e.Kind + ")" - } - } - return false, "" -} - -// firstShared is the first key, in order, both sets hold; empty when none. -func firstShared(a, b map[string]bool) string { - var both []string - for k := range a { - if b[k] { - both = append(both, k) - } - } - if len(both) == 0 { - return "" - } - sort.Strings(both) - return both[0] -} - -// openWalk is a walk open beside a change: its id, what it reaches, and whether it is on the controller's own path. -type openWalk struct { - id string - fp footprint - own bool -} - -// relatedOpen is the first open walk a change is related to, with the reason; empty when it is related to none -// and may walk beside them all (issue 973, ADR 0407 decision 4, ADR 0306 decision 3). -// -// **The controller's own path is related to everything** (decided for issue 973): a walk or batch that replaces -// the controller, the node-engine, the tool runner, the bus or mesh-delivery never runs beside another, and -// nothing runs beside it, whatever the footprints say. -func relatedOpen(cand footprint, candOwn bool, open []openWalk, edges []inventory.Edge) (behind, why string) { - for _, w := range open { - if candOwn { - return w.id, "it is on the controller's own path" - } - if w.own { - return w.id, w.id + " is on the controller's own path" - } - if ok, why := related(cand, w.fp, edges); ok { - return w.id, why - } - } - return "", "" -} - -// ownWalk says a walk is on the controller's own path: its batch says so, or it plans a module of that path -// (issue 973). -func ownWalk(p inventory.Plan, onPath map[string]string) bool { - if p.OwnPath() { - return true - } - for name := range p.Modules { - if _, own := onPath[name]; own { - return true - } - } - return false -} - -// startedWalks are the open walks started beside the walk id, made at created: a merge's walk past its wait, not -// the backlog's, with a tier asked, or on its way (let go, or waiting for nobody) before this one. -func startedWalks(plans []inventory.Plan, id string, created time.Time) []inventory.Plan { - var out []inventory.Plan - for _, q := range plans { - if q.ID == id || q.Release != nil || q.Waiting() { - continue - } - if q.Tier > 0 || askedAny(q) || q.Created.Before(created) { - out = append(out, q) - } - } - return out -} - -// openWalkOf is an open walk as a change is judged against it: what it reaches, from its planned modules and the -// merges the record holds for it. -func openWalkOf(ctx context.Context, inv *inventory.Inventory, p inventory.Plan, onPath map[string]string) (openWalk, error) { - kept, err := inv.MergesOf(ctx, p.ID) - if err != nil { - return openWalk{}, err - } - return openWalk{id: p.ID, fp: footprintOfWalk(p, kept), own: ownWalk(p, onPath)}, nil -} - -// openWalksOf are openWalkOf for each of plans. -func openWalksOf(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan, onPath map[string]string) ([]openWalk, error) { - var out []openWalk - for _, p := range plans { - w, err := openWalkOf(ctx, inv, p, onPath) - if err != nil { - return nil, err - } - out = append(out, w) - } - return out, nil -} diff --git a/cmd/mesh-controller/related_integration_test.go b/cmd/mesh-controller/related_integration_test.go deleted file mode 100644 index b29fb967..00000000 --- a/cmd/mesh-controller/related_integration_test.go +++ /dev/null @@ -1,52 +0,0 @@ -//go:build integration - -package main - -import ( - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestRelatedOpenNamesTheFirstOpenWalkAChangeIsRelatedTo(t *testing.T) { - edges := []inventory.Edge{{From: "nzbget", To: "base-image", Kind: inventory.EdgeStandsOn}} - walk := func(id, repo string, own bool, modules ...string) openWalk { - return openWalk{id: id, fp: footprintOfWalk(walkOf(repo, modules...), nil), own: own} - } - one, two := walk("plan-1", "novox/one", false, "notes"), walk("plan-2", "novox/two", false, "nzbget") - cand := func(repo, head string, moves ...string) footprint { - return footprintOfBatch([]inventory.BatchedMerge{heardMerge(repo, head, moves...)}, edges) - } - cases := []struct { - name string - fp footprint - own bool - open []openWalk - behind string - why string - }{ - {"nothing open", cand("novox/x", "fix/a", "app"), false, nil, "", ""}, - {"unrelated to every open walk", cand("novox/x", "fix/a", "app"), false, []openWalk{one, two}, "", ""}, - {"a repository", cand("novox/ONE", "fix/a", "app"), false, []openWalk{two, one}, "plan-1", "both change novox/one"}, - {"a module through its dependents", cand("novox/x", "fix/a", "base-image"), false, []openWalk{one, two}, - "plan-2", "both move nzbget"}, - {"a graph edge", cand("novox/x", "fix/a", "nzbget"), - false, []openWalk{one, walk("plan-3", "novox/y", false, "base-image")}, "plan-3", - "nzbget depends on base-image (stands-on)"}, - {"moves not known", cand("novox/x", "fix/a"), false, []openWalk{one, two}, "plan-1", - "the modules a merge of novox/x moves are not known until its batch is cut"}, - {"the candidate on the own path", cand("novox/x", "fix/a", "app"), true, []openWalk{one}, "plan-1", - "it is on the controller's own path"}, - {"an open walk on the own path", cand("novox/x", "fix/a", "app"), false, - []openWalk{one, walk("plan-4", "novox/mesh-controller", true, "mesh-controller")}, "plan-4", - "plan-4 is on the controller's own path"}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - behind, why := relatedOpen(c.fp, c.own, c.open, edges) - if behind != c.behind || why != c.why { - t.Errorf("relatedOpen = %q, %q; want %q, %q", behind, why, c.behind, c.why) - } - }) - } -} diff --git a/cmd/mesh-controller/related_test.go b/cmd/mesh-controller/related_test.go deleted file mode 100644 index d67c0cb9..00000000 --- a/cmd/mesh-controller/related_test.go +++ /dev/null @@ -1,193 +0,0 @@ -package main - -import ( - "encoding/json" - "errors" - "fmt" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// heardMerge is a kept merge of a repository from a head branch that moved modules. -func heardMerge(repository, head string, moves ...string) inventory.BatchedMerge { - m := inventory.BatchedMerge{Repository: repository, Branch: "main", Commit: "c-" + repository} - m.Event, _ = json.Marshal(keptMerge{SourceMoved: link.SourceMoved{Head: head}, Moves: moves}) - return m -} - -// walkOf is a walk carrying one repository's commit and planning modules. -func walkOf(repository string, modules ...string) inventory.Plan { - p := inventory.Plan{Repository: repository, Commit: "w-" + repository, Modules: map[string]*inventory.PlanModule{}} - for _, name := range modules { - p.Modules[name] = &inventory.PlanModule{} - } - return p -} - -// TestRelatedChanges: two changes are related by a repository, a module, a group head or a graph edge between -// their modules, and by a module reached only through dependents (novox/hq ADR 0304 decision 1, ADR 0306 -// decision 3); otherwise they are not. -func TestRelatedChanges(t *testing.T) { - edges := []inventory.Edge{ - {From: "nzbget", To: "base-image", Kind: inventory.EdgeStandsOn}, - {From: "app", To: "build-machine", Kind: inventory.EdgeBuiltBy}, - } - cases := []struct { - name string - a, b footprint - why string - }{ - {"repository, case aside", - footprintOfWalk(walkOf("novox/Mesh-Catalog", "notes"), nil), - footprintOfBatch([]inventory.BatchedMerge{heardMerge("NOVOX/mesh-catalog", "fix/a", "app")}, nil), - "both change novox/mesh-catalog"}, - {"module", - footprintOfWalk(walkOf("novox/one", "nzbget"), nil), - footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "fix/b", "nzbget")}, nil), - "both move nzbget"}, - {"edge, walk depends on batch", - footprintOfWalk(walkOf("novox/one", "nzbget"), nil), - footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "fix/b", "base-image")}, nil), - "nzbget depends on base-image (stands-on)"}, - {"edge, batch depends on walk", - footprintOfWalk(walkOf("novox/one", "build-machine"), nil), - footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "fix/b", "app")}, nil), - "app depends on build-machine (built-by)"}, - {"group head", - footprintOfWalk(walkOf("novox/one", "notes"), []inventory.BatchedMerge{heardMerge("novox/one", "feat/x")}), - footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "feat/x", "app")}, nil), - "both are members of group feat/x"}, - {"module reached through dependents", - footprintOfWalk(walkOf("novox/one", "nzbget"), nil), - footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "fix/b", "base-image")}, edges), - "both move nzbget"}, - {"unrelated", - footprintOfWalk(walkOf("novox/one", "notes"), []inventory.BatchedMerge{heardMerge("novox/one", "feat/x")}), - footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "feat/y", "app")}, edges), - ""}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - for _, pair := range [][2]footprint{{c.a, c.b}, {c.b, c.a}} { - is, why := related(pair[0], pair[1], edges) - if is != (c.why != "") || why != c.why { - t.Errorf("related = %v, %q; want %q", is, why, c.why) - } - } - }) - } -} - -// TestFootprintOfBatchWidensToDependents: a batch's modules are what its merges moved and every module that -// depends on them, along the edges that widen a walk. -func TestFootprintOfBatchWidensToDependents(t *testing.T) { - edges := []inventory.Edge{ - {From: "nzbget", To: "base-image", Kind: inventory.EdgeStandsOn}, - {From: "base-image", To: "build-machine", Kind: inventory.EdgeBuiltBy}, - } - f := footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/two", "", "base-image")}, edges) - if !f.modules["base-image"] || !f.modules["nzbget"] || f.modules["build-machine"] || len(f.heads) != 0 { - t.Errorf("footprint = %+v; want base-image and nzbget, no head", f) - } -} - -// A batch holding a merge kept with no moves (one owed a later look, judged again at the cut) is related to -// every change, in both orders: what it will move is not known, and an unknown change is never unrelated -// (issue 525 review). -func TestABatchWhoseMovesAreNotKnownIsRelatedToEverything(t *testing.T) { - unknown := footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/one", "fix/a", "app"), heardMerge("novox/two", "fix/b")}, nil) - other := footprintOfWalk(walkOf("novox/three", "nzbget"), nil) - for _, pair := range [][2]footprint{{unknown, other}, {other, unknown}} { - if ok, why := related(pair[0], pair[1], nil); !ok || !strings.Contains(why, "not known") { - t.Fatalf("an unknown batch read as %v (%q)", ok, why) - } - } - known := footprintOfBatch([]inventory.BatchedMerge{heardMerge("novox/one", "fix/a", "app")}, nil) - if ok, why := related(known, other, nil); ok { - t.Fatalf("a known, unrelated batch read as related: %q", why) - } -} - -// TestRelatedOpenNamesTheFirstOpenWalkAChangeIsRelatedTo: a change that shares nothing with any open walk is -// held behind none (issue 973, ADR 0407 decision 4); one that shares a repository, a module, a graph edge or a -// group head is held behind the first walk it shares it with, with the reason; one whose moves are not known -// is held behind the first; a walk or change on the controller's own path is related to every other. - -// A walk is on the controller's own path when its batch says so or it plans a module of that path (issue 973). -func TestAWalkOnTheControllersPathIsKnownByItsFlagOrItsModules(t *testing.T) { - if ownWalk(walkOf("novox/one", "notes"), ownPathOf(nil)) { - t.Error("an ordinary walk read as on the controller's own path") - } - flagged := walkOf("novox/one", "notes") - flagged.Delivery = &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: true}} - if !ownWalk(flagged, ownPathOf(nil)) { - t.Error("a walk whose batch is own-path read as ordinary") - } - var module string - for m := range onTheControllersPath { - module = m - } - if !ownWalk(walkOf("novox/one", "notes", module), ownPathOf(nil)) { - t.Errorf("a walk planning %s read as ordinary", module) - } -} - -// A walk about to ask its first tier waits only for a started walk it is related to (issue 973): started is a -// tier asked, or a walk made before it; a waiting walk, the backlog's and itself are not. -func TestWalksStartedBesideAreTheOpenOnesBeforeIt(t *testing.T) { - at := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC) - mk := func(id string, created time.Time, tier int) inventory.Plan { - p := walkOf("novox/"+id, id) - p.ID, p.Created, p.Tier, p.State = id, created, tier, inventory.PlanBuilding - return p - } - older, newer, ticking := mk("older", at, 0), mk("newer", at.Add(time.Minute), 0), mk("ticking", at.Add(2*time.Minute), 1) - backlog := mk("backlog", at, 1) - backlog.Release = &inventory.PlanRelease{} - got := startedWalks([]inventory.Plan{older, newer, ticking, backlog, mk("me", at.Add(time.Minute), 0)}, "me", at.Add(time.Minute)) - var ids []string - for _, p := range got { - ids = append(ids, p.ID) - } - if strings.Join(ids, ",") != "older,ticking" { - t.Errorf("started beside me = %v; want older, ticking", ids) - } -} - -// A retry is refused beside an open walk it is related to and allowed beside one it shares nothing with -// (issue 973); a retry on the controller's own path is refused beside any open walk. -func TestARetryWaitsOnlyForAnOpenWalkItIsRelatedTo(t *testing.T) { - failed := walkOf("novox/one", "notes") - failed.ID, failed.State = "plan-1", inventory.PlanFailed - open := func(id, repo string, modules ...string) inventory.Plan { - p := walkOf(repo, modules...) - p.ID, p.State = id, inventory.PlanBuilding - return p - } - if err := oneWalkAtATime(failed, []inventory.Plan{open("plan-2", "novox/two", "app")}, ownPathOf(nil)); err != nil { - t.Errorf("refused beside an unrelated walk: %v", err) - } - err := oneWalkAtATime(failed, []inventory.Plan{open("plan-3", "novox/three", "notes")}, ownPathOf(nil)) - if err == nil || !strings.Contains(err.Error(), "plan-3") || !strings.Contains(err.Error(), "both move notes") { - t.Errorf("a retry beside a walk moving the same module: %v", err) - } - failed.Delivery = &inventory.PlanDelivery{Batch: &inventory.PlanBatch{Own: true}} - if err := oneWalkAtATime(failed, []inventory.Plan{open("plan-2", "novox/two", "app")}, ownPathOf(nil)); err == nil { - t.Error("an own-path retry went beside an open walk") - } -} - -// Plans another controller is working are "not now", not a failure: the merge is held and handed over again -// (issue 921). -func TestPlansHeldByAnotherControllerAreNotNow(t *testing.T) { - if err := notNow(fmt.Errorf("x: %w", inventory.ErrPlansBusy)); !errors.Is(err, link.ErrTryAgain) { - t.Errorf("busy plans read as %v; want ErrTryAgain", err) - } - if err := notNow(errors.New("no such module")); errors.Is(err, link.ErrTryAgain) { - t.Errorf("another failure read as ErrTryAgain: %v", err) - } -} diff --git a/cmd/mesh-controller/release.go b/cmd/mesh-controller/release.go deleted file mode 100644 index 348468c8..00000000 --- a/cmd/mesh-controller/release.go +++ /dev/null @@ -1,1689 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "flag" - "fmt" - "io" - "slices" - "sort" - "strings" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// No build reaches a machine without a gate (novox/hq ADR 0236). -// -// **A send carries the machine's whole declaration.** A plan sending one module to its first machine -// sends every other module whose build moved there too; a cascade, a healer's resend and a whole-mesh -// push do the same. Under the old default (`record`) builds were registered and sent nowhere, so on the -// day the default became `roll` every machine was behind on dozens of builds no gate had seen — and the -// next send of anything would have restarted all of them at once, on every machine. -// -// So **a build moves on a machine only through a send that judges it there**, or a person's: -// -// - a gated send — a plan's first machine, a release plan's machine — carries every move waiting on that -// machine, and its gate judges each of them; a pass is each build's verdict, a failure puts back what -// failed; -// - a module a send exists for may move anywhere it is sent: a policy of *together*, a rollback, a build -// whose gate already passed; -// - a send naming a machine by a person (`push `, the bus step) carries what it carries; -// - every other send — a plan's "rest", a cascade, a healer's, the bus's user list carried — is refused, -// or leaves the machine, while a move there waits for a gate. A rebuild that made the same artifacts -// from the same manifest is no move. -// -// **The release plan** is what walks the waiting moves through: whenever moves wait for a gate that no -// started plan is walking, the mesh opens one — every such machine, the first sent and judged, then the rest -// at once, each judged by its own gate (novox/hq ADR 0353, in place of one at a time). A release plan that fails stops the next one opening on its own: -// a person releases it again (`upgrade release-backlog --why`), after the condition says why. - -// sendScope is what a send may carry that no gate has seen. -type sendScope struct { - // judged are the machines whose every move this send's gate judges. - judged map[string]bool - // modules are those a send exists for, which may move wherever it goes. - modules map[string]bool - // person is a person's act: it carries what it carries. - person bool - // puttingBack is a gate's own put-back of its modules: the one send that may carry a core module to a build - // older than the one its machine runs (novox/hq issue 201). - puttingBack bool -} - -type sendScopeKey struct{} - -func withScope(ctx context.Context, s sendScope) context.Context { - return context.WithValue(ctx, sendScopeKey{}, s) -} - -func scopeOf(ctx context.Context) sendScope { - s, _ := ctx.Value(sendScopeKey{}).(sendScope) - return s -} - -// errUngated is a send refused because it would carry a build no gate has seen. -var errUngated = errors.New("a build waits there for its gate") - -// errWalkedElsewhere is a gated send refused because a plan that has started walks a move there. -var errWalkedElsewhere = errors.New("a plan already walking a build there sends it") - -// moveFacts is what tells a move from a rebuild, and a gated build from one no gate has seen. -type moveFacts struct { - current map[string]inventory.CurrentBuild - fps map[string]map[string]string - // srcs is, per module, per commit, its build's source fingerprint (novox/hq issue 280). - srcs map[string]map[string]string - passed map[string]map[string]bool - plans []inventory.Plan - // self is the release walking these moves, whose own gates are no other walk's (novox/hq ADR 0353). - self string - // judge judges a move under a module's own `record` (novox/hq ADR 0377): nil when the facts were not read, - // and every such move then waits. - judge func(module, was, now string) recordVerdict -} - -type walkingKey struct{} - -// withWalking says a release plan is the one sending: its own gates never hold its own sends back. -func withWalking(ctx context.Context, id string) context.Context { - return context.WithValue(ctx, walkingKey{}, id) -} - -func readMoveFacts(ctx context.Context, inv *inventory.Inventory) (moveFacts, error) { - var f moveFacts - var err error - if f.current, err = inv.CurrentBuilds(ctx); err != nil { - return f, err - } - if f.fps, err = inv.Fingerprints(ctx); err != nil { - return f, err - } - if f.srcs, err = inv.SourceFingerprints(ctx); err != nil { - return f, err - } - if f.passed, err = inv.PassedCommits(ctx); err != nil { - return f, err - } - f.plans, err = inv.OpenPlans(ctx) - f.self, _ = ctx.Value(walkingKey{}).(string) - f.judge = recordJudge(ctx, inv) - return f, err -} - -// identical is whether two builds of a module put the same thing on a machine: the same commit, -// builds made from the same source (novox/hq issue 280) — the module's tree, the contexts it read, its -// bases and toolchains, whatever digests an image rebuild made of them — or builds that made the same -// artifacts from the same manifest. -func (f moveFacts) identical(module, a, b string) bool { - if a == b || sameCommit(a, b) { - return true - } - if sa := f.srcs[module][a]; sa != "" && sa == f.srcs[module][b] { - return true - } - fa := f.fps[module][a] - return fa != "" && fa == f.fps[module][b] -} - -// gated is whether a build of a module from this commit — or one identical to it — passed a gate. -func (f moveFacts) gated(module, commit string) bool { - for c := range f.passed[module] { - if f.identical(module, c, commit) { - return true - } - } - return false -} - -// unchangedOnEvery is whether a plan's build of a module was made from the source of the build every -// machine running it was last sent (novox/hq issue 280): no move on any of them. False when no machine -// runs it, when what one was sent is not known, or when the build stands for itself. -func unchangedOnEvery(ctx context.Context, inv *inventory.Inventory, module, build string, running []string) (bool, error) { - if build == "" || len(running) == 0 { - return false, nil - } - same, err := inv.SameSourceCommits(ctx, module, build) - if err != nil || len(same) == 0 { - return false, err - } - for _, n := range running { - sent, known, err := inv.SentBuilds(ctx, n) - if err != nil { - return false, err - } - was, carried := sent[module] - if !known || !carried || !inRun(same, was) { - return false, nil - } - } - return true, nil -} - -// inRun is whether a commit is one of a run's, however either is abbreviated. -func inRun(run map[string]bool, commit string) bool { - for c := range run { - if sameCommit(c, commit) { - return true - } - } - return false -} - -// moves is what a machine's next send would move that no gate has seen: modules whose policy rolls out -// (a recorded one is a person's push, unless its move under its own record recreates nothing and moves no data — -// novox/hq ADR 0377), that the machine was sent before, moving to a build not identical -// to the one it runs and that has passed no gate. A machine whose last send's builds are not known names -// none: the cascade holds it whole (ADR 0221). -// -// With all, a move to a build that passed a gate elsewhere counts too: what a gated send carries. -func (f moveFacts) moves(node string, modules []string, sent map[string]string, known, all bool) []inventory.CarriedMove { - if !known { - return nil - } - var out []inventory.CarriedMove - for _, m := range modules { - now := f.current[m] - was, carried := sent[m] - if !carried || f.identical(m, was, now.Commit) || (!all && f.gated(m, now.Commit)) || !f.rolls(m, was) { - continue - } - out = append(out, inventory.CarriedMove{Module: m, Node: node, From: was, To: now.Commit}) - } - sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) - return out -} - -// walkedBy is the open plan that has started walking a module's build — sent it to a first machine, not -// yet passed — and whose walk this move would cross; empty when none does. A move of the same build to a -// machine that plan already sent it is not a crossing: the build is there. A move of **another** build of -// the module to that machine is (novox/hq issue 352): on 2026-10-09 a merge's plan sent the control node a -// newer controller while a release's gate was judging the controller there, the release's gate read the -// machine's report against the newer send, failed three builds and put them back under the new plan's -// feet. A release keeps no module records: its open gate's carried moves are its walk. -func (f moveFacts) walkedBy(module, node, to string) string { - for _, p := range f.plans { - if !p.Open() || (p.Release != nil && p.ID == f.self) { - continue - } - if p.Release != nil { - // The first machine's gate, and every gate of the rest still judging (novox/hq ADR 0353). - gates := []*inventory.PlanGate{p.Release.Gate} - for _, g := range p.Release.Gates { - gates = append(gates, g) - } - for _, g := range gates { - // A send kept and not yet made walks nothing yet (ADR 0353 §8). - if g == nil || g.Verdict != "" || g.Silent || g.Pending { - continue - } - for _, c := range g.Carried { - if c.Module == module && !(slices.Contains(g.Machines, node) && sameCommit(c.To, to)) { - return p.ID - } - } - } - continue - } - s, holds := p.Modules[module] - if !holds || s == nil || s.FirstAt == nil || s.SentAt != nil { - continue - } - if slices.Contains(s.First, node) && sameCommit(s.Commit, to) { - continue - } - if s.Gate != nil && s.Gate.Verdict == inventory.GatePassed { - continue - } - return p.ID - } - return "" -} - -// machineMoves is the moves no gate has seen on one machine, read from what it would be sent now. -var machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { - plan, _, err := resolveNode(ctx, open, node) - if err != nil { - return nil, nil // it cannot be worked out: the send says why - } - modules := make([]string, 0, len(plan.Modules)) - for _, m := range plan.Modules { - modules = append(modules, m.Module) - } - sent, known, err := open.inventory.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - return f.moves(node, modules, sent, known, all), nil -} - -// withoutKept is a machine's moves less those its send keeps at the build the machine was last sent, and so does -// not make: the grants step keeps every module (novox/hq issue 490), and every send keeps the controller and the -// node-engine over an older record (issue 201). Read as the send composes it (sendKeeps, set after the checks that -// call this), so a move it could not keep is still counted. -func withoutKept(ctx context.Context, open *stores, node string, moves []inventory.CarriedMove) ([]inventory.CarriedMove, error) { - if len(moves) == 0 { - return moves, nil - } - keeping := ctx - if _, on := keptExcept(ctx); !on { - keeping = keepingRecorded(ctx) - } - keeps, err := recordedKept(keeping, open, node) - if err != nil || len(keeps) == 0 { - return moves, err - } - var out []inventory.CarriedMove - for _, mv := range moves { - if was, kept := keeps[mv.Module]; kept && sameCommit(was, mv.From) { - continue - } - out = append(out, mv) - } - return out, nil -} - -// ungatedIn refuses a send whose machines would move a build no gate has seen, outside its scope: the -// machines named, and why; the machine holding the bus, added only for its user list, is left instead. -func ungatedIn(ctx context.Context, open *stores, names []string, addedHolder string) ([]string, error) { - scope := scopeOf(ctx) - if scope.person { - return names, nil - } - f, err := readMoveFacts(ctx, open.inventory) - if err != nil { - return nil, err - } - var kept, refused []string - for _, n := range names { - moves, err := machineMoves(ctx, open, f, n, false) - if err != nil { - return nil, err - } - if moves, err = withoutKept(ctx, open, n, moves); err != nil { - return nil, err - } - var waiting []string - for _, mv := range moves { - if !scope.judged[n] && !scope.modules[mv.Module] { - waiting = append(waiting, fmt.Sprintf("%s %s → %s", mv.Module, short(mv.From), short(mv.To))) - } - } - switch { - case len(waiting) == 0: - kept = append(kept, n) - case n == addedHolder: - fmt.Printf("%s holds the bus and its user list changed, and it is not sent now: %s wait there for a gate "+ - "— the bus may refuse what was newly granted until it is\n", n, strings.Join(waiting, ", ")) - default: - refused = append(refused, fmt.Sprintf("%s (%s)", n, strings.Join(waiting, ", "))) - } - } - if len(refused) > 0 { - return nil, fmt.Errorf("%w: %s — a walk sends them, one machine first and then the rest at once, each judged (novox/hq ADR "+ - "0236, ADR 0353); `upgrade backlog` lists them", errUngated, strings.Join(refused, "; ")) - } - return kept, nil -} - -// gatedSend sends one machine everything waiting there, under a gate that judges it all: what moved is -// answered, with the build each moved to, for the gate to judge and to put back. Refused while a plan that -// has started walks one of those builds elsewhere: that plan sends it here once its gate passed. -// -// owns are the moves the send exists for — every module of a plan's tier whose first machine this is, in -// one send (novox/hq issue 281): a send carries the machine's whole declaration (ADR 0221), so a send per -// module was the same declaration sent again and again, each one setting aside the one before. -// -// And the grants step first, when the send changes what the bus's user list must say (novox/hq issue 490): -// answered for the gate to keep, nil when there was none. -func gatedSend(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, []string, *inventory.GrantsStep, error) { - moves, err := gatedMoves(ctx, open, node, owns) - if err != nil || len(moves) == 0 { - return nil, nil, nil, err - } - sayRecreations(ctx, open, moves) - sent, grants, err := sendJudged(ctx, open, node) - if err != nil { - return nil, nil, nil, err - } - return moves, sent, grants, nil -} - -// gatedMoves is what a gated send to one machine would carry, with the build each moves to, made of nothing: -// gatedSend's reading, which a walk also keeps before it sends (novox/hq ADR 0353 §8). Refused the same way -// while a plan that has started walks one of those builds elsewhere. -func gatedMoves(ctx context.Context, open *stores, node string, owns []inventory.CarriedMove) ([]inventory.CarriedMove, error) { - inv := open.inventory - f, err := readMoveFacts(ctx, inv) - if err != nil { - return nil, err - } - moves, err := machineMoves(ctx, open, f, node, true) - if err != nil { - return nil, err - } - // A move the send keeps is not made, and is no move for the gate to judge (novox/hq issue 201): a core module - // kept over an older record judged as moving back to it would read the gate as superseded by its own send. - if moves, err = withoutKept(ctx, open, node, moves); err != nil { - return nil, err - } - own := func(module string) bool { - return slices.ContainsFunc(owns, func(o inventory.CarriedMove) bool { return o.Module == module }) - } - for _, mv := range moves { - if own(mv.Module) { - continue - } - if id := f.walkedBy(mv.Module, node, mv.To); id != "" { - return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere, - mv.Module, short(mv.To), node, id) - } - } - // **And the plan's own modules wait too** (novox/hq issue 352): a walk of the same module by another - // plan, or a release, on this machine is not crossed with a newer build; this send waits for its gate. - for _, o := range owns { - if id := f.walkedBy(o.Module, node, o.To); id != "" { - return nil, fmt.Errorf("%w: %s's build %s waits on %s, which %s is walking", errWalkedElsewhere, - o.Module, short(o.To), node, id) - } - } - for _, o := range owns { - i := slices.IndexFunc(moves, func(mv inventory.CarriedMove) bool { return mv.Module == o.Module }) - switch { - case i < 0: - moves = append(moves, o) - case moves[i].Build == "": - moves[i].Build = o.Build - } - } - if len(moves) == 0 { - return nil, nil - } - for i := range moves { - if moves[i].Build == "" { - if moves[i].Build, err = inv.BuildOf(ctx, moves[i].Module, moves[i].To); err != nil { - return nil, err - } - } - } - sort.Slice(moves, func(i, j int) bool { return moves[i].Module < moves[j].Module }) - return moves, nil -} - -// passCarried keeps a pass as the verdict of every build the gate judged beside its own module. -func passCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) { - seen := map[string]bool{} - for _, c := range g.Carried { - // A build put back with another machine's failure is not this gate's to pass (novox/hq ADR 0353 §3). - if c.Module == except || c.Build == "" || seen[c.Build] || slices.Contains(g.Returned, c.Module) { - continue - } - seen[c.Build] = true - if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To, - Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: whyFor(g, c.Module), - Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil { - fmt.Printf("%s: %s passed its gate on %s, and the verdict could not be kept: %v\n", p.ID, c.Module, c.Node, err) - } - } -} - -// failCarried puts back every build the gate carried that it found wanting, on the machines that were -// sent it, once each. -func failCarried(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) { - var notes []string - if p.Note != "" { - notes = append(notes, p.Note) - } - done := map[string]bool{except: true} - for _, m := range g.Returned { - done[m] = true // put back at once when it broke - } - // **What passed on its own keeps its pass** (novox/hq ADR 0254, issue 318): healthy for the passes the - // gate asks while another module of the send failed, it is neither put back nor left unjudged — a build - // left without a verdict is one more move every other walk would wait for. - if kept := keepPassing(ctx, open, p, g, except); len(kept) > 0 { - notes = append(notes, "passed on their own and kept: "+strings.Join(kept, ", ")) - for _, m := range kept { - done[m] = true - } - } - for _, c := range g.Carried { - if done[c.Module] || (len(g.Failing) > 0 && !slices.Contains(g.Failing, c.Module)) { - continue - } - done[c.Module] = true - machines, err := sentTheBuild(ctx, open, c.Module, c.To) - if err != nil || len(machines) == 0 { - machines = []string{c.Node} - } - state := &inventory.PlanModule{Build: c.Build, Previous: c.From, Commit: c.To} - // A module of the plan's tier sent in the same send (issue 281) keeps its own record of it. - if s := p.Modules[c.Module]; s != nil && except != "" && s.GatedBy == except && s.Build == c.Build { - state = s - } - p.Note = "" - gateFailed(ctx, open, p, c.Module, state, machines, g.Why) - notes = append(notes, p.Note) - } - p.State = inventory.PlanFailed - p.Note = strings.Join(notes, "; ") -} - -// keepPassing keeps a pass as the verdict of every build the gate carried that passed on its own when the -// send failed (ADR 0254), except the plan's own module; answers the modules kept. -func keepPassing(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, except string) []string { - var kept []string - seen := map[string]bool{} - for _, c := range g.Carried { - if c.Module == except || c.Build == "" || seen[c.Build] || !slices.Contains(g.Passing, c.Module) { - continue - } - seen[c.Build] = true - why := passedAloneWhy(g, c.Module) - if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: c.Build, Module: c.Module, Commit: c.To, - Previous: c.From, Plan: p.ID, Machines: []string{c.Node}, Verdict: inventory.GatePassed, Why: why, - Component: coreComponent(c.Module), JudgingFrom: g.Since}); err != nil { - fmt.Printf("%s: %s passed its gate on %s on its own, and the verdict could not be kept: %v\n", p.ID, c.Module, - c.Node, err) - continue - } - if !slices.Contains(kept, c.Module) { - kept = append(kept, c.Module) - } - } - return kept -} - -// putBackBroken puts back, at once, every module a gate found broken and has not put back yet (issue 318 -// review): a witness that reverted a core component, or a machine that refused or failed its send, is not -// left registered at the build that broke for as long as the modules beside it take to be judged — a resend -// meanwhile would declare it again. The send goes on judging the others to their own verdicts; its own -// verdict is still failed. lead and leadState are the module a plan's gate is kept on, when it is a plan's. -// Answers whether anything was put back. -func putBackBroken(ctx context.Context, open *stores, p *inventory.Plan, g *inventory.PlanGate, lead string, - leadState *inventory.PlanModule) bool { - var todo []string - for _, m := range g.Broken { - if !slices.Contains(g.Returned, m) { - todo = append(todo, m) - } - } - if len(todo) == 0 || g.Verdict != "" { - return false - } - stateWas, noteWas := p.State, p.Note - batched, back := batchingRollbacks(ctx) - var said []string - for _, m := range todo { - machines := g.Machines - var state *inventory.PlanModule - if m == lead && leadState != nil { - judged := *leadState - judged.Gate = nil // its own record of the failure; the send's gate goes on judging - state = &judged - } else { - i := slices.IndexFunc(g.Carried, func(c inventory.CarriedMove) bool { return c.Module == m }) - if i < 0 { - continue - } - c := g.Carried[i] - state = &inventory.PlanModule{Build: c.Build, Previous: c.From, Commit: c.To} - if sent, err := sentTheBuild(ctx, open, m, c.To); err == nil && len(sent) > 0 { - machines = sent - } else { - machines = []string{c.Node} - } - } - // Counted as put back only once there is something to put back (a carried move or the plan's own). - g.Returned = append(g.Returned, m) - p.Note = "" - gateFailed(batched, open, p, m, state, machines, g.BrokenWhy) - if m == lead && leadState != nil { - leadState.Why = "put back at once, its send still judged: " + g.BrokenWhy - } - said = append(said, m) - } - sendRollbacks(ctx, open, p, back) - p.State = stateWas - p.Note = noteWas - if len(said) > 0 { - p.Note = fmt.Sprintf("put back at once: %s (%s); the rest of the send judged to their own verdicts", - strings.Join(said, ", "), g.BrokenWhy) - fmt.Printf("%s: %s\n", p.ID, p.Note) - } - return true -} - -// passedAloneWhy is the verdict of a module that passed on its own while its send failed, with its own wait. -func passedAloneWhy(g *inventory.PlanGate, module string) string { - why := fmt.Sprintf("healthy %d times on its own while the send failed: %s", g.Healthy[module], g.Why) - if w := g.Waits[module]; w != "" { - why += waitsPrefix + w - } - return why -} - -// sentTheBuild is every machine running a module that was last sent this build of it. -func sentTheBuild(ctx context.Context, open *stores, module, commit string) ([]string, error) { - running, err := open.inventory.Running(ctx, module) - if err != nil { - return nil, err - } - var out []string - for _, n := range running { - sent, known, err := open.inventory.SentBuilds(ctx, n) - if err != nil { - return nil, err - } - if known && sameCommit(sent[module], commit) { - out = append(out, n) - } - } - return out, nil -} - -// releaseRepository is what a release plan says it is for, where a merge's says its repository. -const releaseRepository = "the builds waiting for a gate" - -// releaseHeard is the machines a release plan may send: heard within their heartbeat's bound. A machine -// away is left, not judged against a bound it cannot meet. A variable so a test says who is heard. -var releaseHeard = func(ctx context.Context, open *stores) (map[string]bool, error) { - if d := doctorFrom; d != nil && d.watchdogs != nil { - return heardMachines(d), nil - } - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return nil, err - } - out := map[string]bool{} - for _, r := range reports { - if r.At != nil && time.Since(*r.At) < 15*time.Minute { - out[r.Node] = true - } - } - return out, nil -} - -// backlogNow is what the newest look found waiting, for `upgrade backlog` and the gate's probe. -var backlogNow struct { - held string - waiting map[string][]inventory.CarriedMove -} - -// waitingMoves is every machine's moves no gate has seen and no started plan walks. -func waitingMoves(ctx context.Context, open *stores, all bool) (map[string][]inventory.CarriedMove, error) { - f, err := readMoveFacts(ctx, open.inventory) - if err != nil { - return nil, err - } - nodes, err := open.inventory.Nodes(ctx) - if err != nil { - return nil, err - } - out := map[string][]inventory.CarriedMove{} - for _, n := range nodes { - moves, err := machineMoves(ctx, open, f, n.Name, all) - if err != nil { - return nil, err - } - for _, mv := range moves { - if f.walkedBy(mv.Module, n.Name, mv.To) == "" { - out[n.Name] = append(out[n.Name], mv) - } - } - } - return out, nil -} - -// releaseBacklog opens a release plan when builds wait for a gate and none is open; not after a release -// plan failed, until a person releases one (by). Called with the plans held. -func releaseBacklog(ctx context.Context, open *stores, by string) (*inventory.Plan, error) { - inv := open.inventory - plans, err := inv.OpenPlans(ctx) - if err != nil { - return nil, err - } - for _, p := range plans { - if p.Release != nil { - if by != "" && p.Release.Settings != nil { - return nil, fmt.Errorf("%s is sending %s; what waits is released once it ends — `plans %s` says where it is", - p.ID, p.Repository, p.ID) - } - if by != "" { - return nil, fmt.Errorf("%s is already releasing what waits; `plans %s` says where it is", p.ID, p.ID) - } - return nil, nil - } - } - waiting, err := waitingMoves(ctx, open, false) - if err != nil { - return nil, err - } - backlogNow.waiting, backlogNow.held = waiting, "" - if len(waiting) == 0 { - return nil, nil - } - // Opened by what no gate has seen; it walks every machine where anything of the release waits — a - // build that passed on the first machine still goes to the next one by this plan, judged there too. - if waiting, err = waitingMoves(ctx, open, true); err != nil { - return nil, err - } - if by == "" { - recent, err := inv.RecentPlans(ctx, 50) - if err != nil { - return nil, err - } - for _, p := range recent { - // A settings walk that failed holds nothing back (novox/hq ADR 0344 §6): its failure is its module's - // settings on one machine, said on its own, not a build no gate passed. - if p.Release == nil || p.Release.Settings != nil { - continue - } - if p.State == inventory.PlanFailed { - backlogNow.held = fmt.Sprintf("%s failed (%s); what waits is released again by a person", p.ID, p.Note) - return nil, nil - } - break - } - } - heard, err := releaseHeard(ctx, open) - if err != nil { - return nil, err - } - controllers, err := inv.Running(ctx, catalogue.ControllerSeatName) - if err != nil { - return nil, err - } - var order, last []string - modules := map[string]bool{} - for node, moves := range waiting { - if !heard[node] { - continue - } - for _, mv := range moves { - modules[mv.Module] = true - } - if slices.Contains(controllers, node) { - last = append(last, node) - } else { - order = append(order, node) - } - } - if len(order)+len(last) == 0 { - return nil, nil - } - sort.Strings(order) - sort.Strings(last) - order = append(order, last...) - names := make([]string, 0, len(modules)) - for m := range modules { - names = append(names, m) - } - sort.Strings(names) - now := time.Now().UTC() - p := inventory.Plan{ID: fmt.Sprintf("release-%d", now.UnixNano()), Repository: releaseRepository, - Created: now, State: inventory.PlanRolling, Tiers: [][]string{names}, Modules: map[string]*inventory.PlanModule{}, - Release: &inventory.PlanRelease{Order: order, By: by}, - Note: fmt.Sprintf("%d build(s) wait for a gate on %s; one machine first, then the rest at once, each judged", len(names), - strings.Join(order, ", "))} - if err := inv.SavePlan(ctx, &p); err != nil { - return nil, err - } - fmt.Printf("%s: %s\n", p.ID, p.Note) - return &p, nil -} - -// advanceRelease takes one step of a release plan: the first machine sent everything waiting there under a -// gate and judged; once it passed, every other machine sent at once, each under its own gate (novox/hq ADR -// 0353 §1), and judged; the plan done when every machine is. -// -// **A send is kept before it is made** (ADR 0353 §8): the step that chooses a machine keeps its gate pending with -// what it will carry, and the next step makes the send. A controller replaced between the send and keeping it -// finds the gate pending, reads the machine's last send, and neither sends again nor leaves the machine unjudged. -func advanceRelease(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) { - r := p.Release - if r.Settings != nil { - return advanceSettingsWalk(ctx, open, p) - } - ctx = withWalking(ctx, p.ID) - now := time.Now().UTC() - if r.Rest { - return advanceRest(ctx, open, p, now) - } - if r.Gate == nil && !gateFirstMachine { - // **No first machine** (novox/hq ADR 0360 §7): every machine heard from is sent at once, in one step. - r.Rest = true - return advanceRest(ctx, open, p, now) - } - if r.Gate == nil { - heard, err := releaseHeard(ctx, open) - if err != nil { - return false, err - } - for r.Next < len(r.Order) { - node := r.Order[r.Next] - if !heard[node] { - r.Skipped = append(r.Skipped, node) - r.Next++ - continue - } - moves, err := gatedMoves(ctx, open, node, nil) - if errors.Is(err, errWalkedElsewhere) { - note := fmt.Sprintf("waiting before %s: %v", node, err) - changed := p.Note != note - p.Note = note - return changed, nil - } - if err != nil { - return false, fmt.Errorf("reading what waits on %s: %w", node, err) - } - if len(moves) == 0 { - r.Done = append(r.Done, node) - r.Next++ - continue - } - r.Gate = &inventory.PlanGate{Machines: []string{node}, Carried: moves, Pending: true} - p.Note = fmt.Sprintf("sending %s %d build(s) that wait for a gate, first", node, len(moves)) - return true, nil - } - p.State, p.Tier = inventory.PlanDone, len(p.Tiers) - p.Note = fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", "))) - if len(r.Skipped) > 0 { - p.Note += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ") - } - return true, nil - } - g := r.Gate - if g.Pending { - node := firstOf(g.Machines) - out, err := sendKept(ctx, open, p, node, g, now) - if err != nil || out == keptWaiting { - return out == keptWaiting, err - } - if out == keptNothing { - r.Gate = nil - r.Done = append(r.Done, node) - r.Next++ - } - return true, nil - } - verdict, err := judgeMoves(ctx, open, g, nil, now) - if err != nil { - return false, err - } - switch verdict { - case "": - if putBackBroken(ctx, open, p, g, "", nil) { - return true, nil - } - note := fmt.Sprintf("judging %s: %s", strings.Join(g.Machines, ", "), gateLine(g)) - changed := p.Note != note - p.Note = note - return changed, nil - case inventory.GatePassed: - passCarried(ctx, open, p, g, "") - r.Done = append(r.Done, firstOf(g.Machines)) - r.Next++ - r.Gate = nil - // **Then the rest at once** (novox/hq ADR 0353 §1): every other machine heard from, each judged by its own - // gate, in place of one machine at a time (ADR 0236 §22). - r.Rest = true - p.Note = fmt.Sprintf("passed on %s; the rest are sent at once, each judged", firstOf(g.Machines)) - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil - case inventory.GateSuperseded: - // Another send moved a judged module on the judged machine (novox/hq issue 352): no verdict on what - // was carried, nothing put back, and this release ends; the builds still waiting are released again - // by the next pass, judged afresh. - p.State = inventory.PlanSuperseded - p.Note = fmt.Sprintf("superseded on %s: %s — nothing judged, nothing put back; what still waits is released again", - strings.Join(g.Machines, ", "), g.Why) - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil - } - p.Note = "" - batched, back := batchingRollbacks(ctx) - failCarried(batched, open, p, g, "") - sendRollbacks(ctx, open, p, back) - p.Note = fmt.Sprintf("failed its gate on %s: %s — %s", strings.Join(g.Machines, ", "), g.Why, p.Note) - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil -} - -// What sendKept did with a kept send. -const ( - keptSent = iota // sent now - keptAlready // the machine was already sent it: judged from that send, nothing sent - keptNothing // nothing waits there any more: no gate - keptWaiting // another walk judges a build it would carry: it waits -) - -// sendKept makes a send a gate kept before it was made (novox/hq ADR 0353 §8) — unless the machine's last send -// already carries what the gate is for, made before a controller was replaced and before it was kept: then -// nothing is sent, and the gate judges from that send (§6). -func sendKept(ctx context.Context, open *stores, p *inventory.Plan, node string, g *inventory.PlanGate, - now time.Time) (int, error) { - inv := open.inventory - moves, sent, grants, err := gatedSend(ctx, open, node, nil) - if errors.Is(err, errWalkedElsewhere) { - p.Note = fmt.Sprintf("waiting before %s: %v", node, err) - return keptWaiting, nil - } - if err != nil { - return keptSent, fmt.Errorf("sending %s what waits there: %w", node, err) - } - if len(moves) > 0 { - g.Machines, g.Carried, g.Since, g.Grants, g.Pending = sent, moves, &now, grants, false - g.Sent = sentNow(ctx, inv, sent) - p.Note = fmt.Sprintf("sent %s %d build(s) that waited for a gate; judging them there", node, len(moves)) - if said := grantsSaid(grants); said != "" { - p.Note += "; " + said - } - if said := recreationsSaid(moves); said != "" { - p.Note += "; " + said - } - fmt.Printf("%s: %s\n", p.ID, p.Note) - return keptSent, nil - } - if !alreadySent(ctx, inv, node, g.Carried) { - return keptNothing, nil - } - since := now - if s, found, err := inv.SentTo(ctx, node); err == nil && found { - if !s.At.IsZero() { - since = s.At - } - g.Sent = map[string]inventory.SentDeclaration{node: s} - } - g.Since, g.Pending = &since, false - p.Note = fmt.Sprintf("%s was already sent the %d build(s) kept for it, at %s: judged from that send, nothing sent again", - node, len(g.Carried), since.Local().Format("15:04:05")) - fmt.Printf("%s: %s\n", p.ID, p.Note) - return keptAlready, nil -} - -// alreadySent says a machine's last send carries every build of the moves: a send made and not kept. -func alreadySent(ctx context.Context, inv *inventory.Inventory, node string, moves []inventory.CarriedMove) bool { - if len(moves) == 0 { - return false - } - sent, known, err := inv.SentBuilds(ctx, node) - if err != nil || !known { - return false - } - for _, mv := range moves { - if !sameCommit(sent[mv.Module], mv.To) { - return false - } - } - return true -} - -// advanceRest takes one step of a release plan's rest (novox/hq ADR 0353): the sends kept for a step made, every -// machine of the step judged by its own gate, one that fails put back by itself while the others go on to their -// verdicts, and the next step — or the end — once every gate of the step has its verdict or is silent. -func advanceRest(ctx context.Context, open *stores, p *inventory.Plan, now time.Time) (bool, error) { - r := p.Release - // The sends kept for this step, in the plan's order: the control node, last in it, last (§1). - var pending []string - for _, n := range r.Order { - if g := r.Gates[n]; g != nil && g.Pending { - pending = append(pending, n) - } - } - if len(pending) > 0 { - // **A kept send carries only what was kept** (ADR 0353 §2): a build that passed no gate and came to wait on - // a machine since its step was chosen could reach a second machine in it; the step is chosen again. - if grew, err := keptStepGrew(ctx, open, r, pending); err != nil { - return false, err - } else if grew != "" { - for _, n := range pending { - delete(r.Gates, n) - } - opened, waiting, err := openRestStep(ctx, open, p) - if err != nil { - return true, err - } - p.Note = "chosen again: " + grew - if !opened && waiting != "" { - p.Note += "; " + waiting - } - return true, nil - } - var sent, already, waiting []string - for _, node := range pending { - out, err := sendKept(ctx, open, p, node, r.Gates[node], now) - if err != nil { - return true, err - } - switch out { - case keptSent: - sent = append(sent, node) - case keptAlready: - already = append(already, node) - case keptNothing: - delete(r.Gates, node) - r.Done = append(r.Done, node) - case keptWaiting: - waiting = append(waiting, node) - } - } - var said []string - if len(sent) > 0 { - said = append(said, "sent "+strings.Join(sent, ", ")+" at once, each judged there") - } - if len(already) > 0 { - said = append(said, strings.Join(already, ", ")+" already sent it: judged from that send") - } - if len(waiting) > 0 { - said = append(said, "waiting before "+strings.Join(waiting, ", ")+": another walk judges a build it carries") - } - if len(said) > 0 { - p.Note = strings.Join(said, "; ") - } - return len(sent)+len(already) > 0 || len(waiting) < len(pending), nil - } - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return false, err - } - byNode := map[string]inventory.Reported{} - for _, rep := range reports { - byNode[rep.Node] = rep - } - changed, noteWas := false, p.Note - var notes, judging []string - batched, back := batchingRollbacks(ctx) - for _, node := range r.Order { - g := r.Gates[node] - if g == nil || g.Verdict != "" || g.Silent { - continue - } - // What another machine's failure already put back is not this gate's to judge (ADR 0353 §3). - if len(g.Carried) > 0 && len(g.Returned) >= len(g.Carried) { - decide(g, inventory.GateFailed, "what it was sent was put back with another machine's failure", now) - r.Failed = append(r.Failed, node) - changed = true - continue - } - // **A machine that says nothing on its send is left silent, not failed** (ADR 0353 §4): it is not judged - // until it reports, and past ApplySilentAfter it is listed and left. - outcome, reported := byNode[node].Outcome, reportedOnSend(byNode[node], node, g) - if sent, kept := g.Sent[node]; kept && !gateFirstMachine { - // A module handed to the provisioner is applied only once its client said so too (ADR 0362, issue 840). - carried := make([]string, 0, len(g.Carried)) - for _, c := range g.Carried { - carried = append(carried, c.Module) - } - outcome, reported = provisionedOutcome(node, sent.Only(carried...), byNode[node], reported, provLookup(ctx)) - } - if !reported { - if now.Sub(*g.Since) > applySilentAfter { - g.Silent = true - r.Silent = append(r.Silent, node) - changed = true - notes = append(notes, fmt.Sprintf("%s said nothing on its send within %s: left silent", node, - applySilentAfter)) - continue - } - judging = append(judging, node+": not yet reported on its send") - continue - } - if !gateFirstMachine { - // **Judged by its report alone** (novox/hq ADR 0360 §7): applied passes; failed or refused is named and - // fails the walk, with nothing put back — apply-failing says it. - switch outcome { - case inventory.OutcomeApplied: - decide(g, inventory.GatePassed, node+" reported it applied its send (novox/hq ADR 0360)", now) - passCarried(ctx, open, p, g, "") - r.Done = append(r.Done, node) - changed = true - case inventory.OutcomeFailed, inventory.OutcomeRefused: - decide(g, inventory.GateFailed, fmt.Sprintf("%s %s its send", node, outcome), now) - r.Failed = append(r.Failed, node) - changed = true - notes = append(notes, fmt.Sprintf("%s %s its send: nothing put back, apply-failing says it", node, outcome)) - default: - if strings.HasPrefix(outcome, inventory.OutcomeProvisioningFailed) { - decide(g, inventory.GateFailed, fmt.Sprintf("%s: %s", node, outcome), now) - r.Failed = append(r.Failed, node) - changed = true - notes = append(notes, fmt.Sprintf("%s: %s: nothing put back, apply-failing says it", node, outcome)) - continue - } - judging = append(judging, node+": not yet reported on its send") - } - continue - } - verdict, err := judgeMoves(ctx, open, g, nil, now) - if err != nil { - return changed, err - } - switch verdict { - case "": - if putBackBroken(ctx, open, p, g, "", nil) { - changed = true - notes = append(notes, p.Note) - } - judging = append(judging, node+": "+gateLine(g)) - case inventory.GatePassed: - passCarried(ctx, open, p, g, "") - r.Done = append(r.Done, node) - changed = true - case inventory.GateSuperseded: - // Another send moved a judged module there (novox/hq issue 352): nothing judged, nothing put back on - // it; what still waits is released again. - r.Done = append(r.Done, node) - changed = true - notes = append(notes, fmt.Sprintf("superseded on %s: %s", node, g.Why)) - default: - // **Put back by itself** (ADR 0353 §3): what was found wanting goes back at once, and every other - // machine of the step goes on to its own verdict. - stateWas := p.State - p.Note = "" - failCarried(batched, open, p, g, "") - p.State = stateWas - r.Failed = append(r.Failed, node) - changed = true - notes = append(notes, fmt.Sprintf("failed its gate on %s: %s — %s", node, g.Why, p.Note)) - returnedElsewhere(ctx, open, r, node, g) - } - } - p.Note = "" - sendRollbacks(ctx, open, p, back) - if p.Note != "" { - notes = append(notes, strings.TrimPrefix(p.Note, "; ")) - } - if len(judging) > 0 { - note := strings.Join(append(notes, "judging "+strings.Join(judging, "; ")), "; ") - p.Note = note - return changed || noteWas != note, nil - } - // Every gate of the step has its verdict or is silent: the next step, unless a machine failed (§3). - if len(r.Failed) == 0 { - opened, waiting, err := openRestStep(ctx, open, p) - if err != nil { - return changed, err - } - if opened { - return true, nil - } - if waiting != "" { - note := strings.Join(append(notes, waiting), "; ") - p.Note = note - return changed || noteWas != note, nil - } - } - p.Tier = len(p.Tiers) - end := fmt.Sprintf("released on %s", orNone(strings.Join(r.Done, ", "))) - if len(r.Silent) > 0 { - end += "; silent on their send, left: " + strings.Join(r.Silent, ", ") - } - if len(r.Skipped) > 0 { - end += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ") - } - p.State = inventory.PlanDone - if len(r.Failed) > 0 { - p.State = inventory.PlanFailed - end = "failed on " + strings.Join(r.Failed, ", ") + "; " + end - } else if len(r.Silent) > 0 && len(r.Gates) == len(r.Silent) { - // **A rest that said nothing is no success** (ADR 0353 §4): failure is loud; the condition says it. - p.State = inventory.PlanFailed - end = "no machine of the rest reported on its send; " + end - } - p.Note = strings.Join(append(notes, end), "; ") - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil -} - -// keptStepGrew says why a step's kept sends would now carry a build no gate passed that they were not kept with; -// empty when none would. -func keptStepGrew(ctx context.Context, open *stores, r *inventory.PlanRelease, pending []string) (string, error) { - if !gateFirstMachine { - return "", nil // every machine goes at once (novox/hq ADR 0360 §7): a send carries what waits when it is made - } - f, err := readMoveFacts(ctx, open.inventory) - if err != nil { - return "", err - } - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - return "", err - } - for _, node := range pending { - moves, err := gatedMoves(ctx, open, node, nil) - if errors.Is(err, errWalkedElsewhere) { - continue // sendKept says it waits - } - if err != nil { - return "", err - } - kept := r.Gates[node].Carried - for _, mv := range moves { - // The node-engine and the resolver keep one machine at a time, passed or not (ADR 0353 §9). - if (f.gated(mv.Module, mv.To) && !oneAtATime(mv.Module, shelf[mv.Module])) || slices.ContainsFunc(kept, func(c inventory.CarriedMove) bool { - return c.Module == mv.Module && sameCommit(c.To, mv.To) - }) { - continue - } - return fmt.Sprintf("%s's build %s, which this step did not keep for it, came to wait on %s", mv.Module, short(mv.To), node), nil - } - } - return "", nil -} - -// reportedOnSend says a machine's last report is about the send its gate judges (novox/hq issue 352): the -// declaration kept on the gate, or one sequenced after it; a gate that kept none reads the send made last. -func reportedOnSend(r inventory.Reported, node string, g *inventory.PlanGate) bool { - if r.At == nil { - return false - } - if s, kept := g.Sent[node]; kept { - return s.ReportsOn(r) - } - return r.Current -} - -// returnedElsewhere marks, on every other gate of the rest still judging, the builds a failed machine's -// rollback put back wherever they were sent (ADR 0236 §5): no longer theirs to judge (ADR 0353 §3). -func returnedElsewhere(ctx context.Context, open *stores, r *inventory.PlanRelease, failed string, g *inventory.PlanGate) { - for _, c := range g.Carried { - if c.Build == "" { - continue - } - if marked, err := open.inventory.GateFailed(ctx, c.Build); err != nil || !marked { - continue - } - for node, other := range r.Gates { - if node == failed || other.Verdict != "" || slices.Contains(other.Returned, c.Module) { - continue - } - for _, oc := range other.Carried { - if oc.Module == c.Module && sameCommit(oc.To, c.To) { - other.Returned = append(other.Returned, c.Module) - } - } - } - } -} - -// openRestStep chooses the next step of a release plan's rest and keeps its sends (novox/hq ADR 0353 §1, §2): -// every machine left that is heard from and has something waiting, except that a build no gate has passed goes -// to one machine of the step only. Answers whether a step was kept, or why the walk waits. -func openRestStep(ctx context.Context, open *stores, p *inventory.Plan) (bool, string, error) { - r := p.Release - heard, err := releaseHeard(ctx, open) - if err != nil { - return false, "", err - } - f, err := readMoveFacts(ctx, open.inventory) - if err != nil { - return false, "", err - } - waiting := map[string][]inventory.CarriedMove{} - var nodes, walked []string - for _, node := range r.Order[min(r.Next, len(r.Order)):] { - if r.Gates[node] != nil || slices.Contains(r.Done, node) || slices.Contains(r.Skipped, node) || - slices.Contains(r.Failed, node) || slices.Contains(r.Silent, node) { - continue - } - if !heard[node] { - r.Skipped = append(r.Skipped, node) - continue - } - moves, err := gatedMoves(ctx, open, node, nil) - if errors.Is(err, errWalkedElsewhere) { - walked = append(walked, fmt.Sprintf("%s (%v)", node, err)) - continue - } - if err != nil { - return false, "", fmt.Errorf("reading what waits on %s: %w", node, err) - } - if len(moves) == 0 { - r.Done = append(r.Done, node) - continue - } - waiting[node] = moves - nodes = append(nodes, node) - } - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - return false, "", err - } - // **Every machine at once** (novox/hq ADR 0360 §2, §7): no build waits for one machine to pass first. - step := nodes - if gateFirstMachine { - step = restStep(nodes, waiting, f.gated, func(module string) bool { return oneAtATime(module, shelf[module]) }) - } - if len(step) == 0 { - if len(walked) > 0 { - return false, "waiting before " + strings.Join(walked, "; "), nil - } - return false, "", nil - } - if r.Gates == nil { - r.Gates = map[string]*inventory.PlanGate{} - } - for _, node := range step { - r.Gates[node] = &inventory.PlanGate{Machines: []string{node}, Carried: waiting[node], Pending: true} - } - p.Note = fmt.Sprintf("sending %s at once, each judged there", strings.Join(step, ", ")) - if left := len(nodes) - len(step); left > 0 { - p.Note += fmt.Sprintf("; %d machine(s) wait for a build that passed no gate yet to pass on one", left) - } - return true, "", nil -} - -// oneAtATime is a module whose builds go to the rest one machine at a time (novox/hq ADR 0353 §9): the -// node-engine, which applies every send, and a module holding the mesh's resolver, which names the bus. -func oneAtATime(module string, m catalogue.Manifest) bool { - if module == hostModule { - return true - } - return slices.ContainsFunc(m.Claims, func(c catalogue.Claim) bool { return c.Name == resolverSeat }) -} - -// restStep is the machines of one step of the rest (novox/hq ADR 0353 §2), in the order given: each machine, unless -// a build it would carry has passed no gate and an earlier machine of the step already carries it — that build's -// first machine. Pure. -// -// **The node-engine and the mesh's resolver still go one machine at a time** (ADR 0353 §9): a step carries a -// build of either to one machine only, judged before the next, since a fault that cuts a machine off the bus -// shows only as its silence and no rollback reaches it. -func restStep(nodes []string, waiting map[string][]inventory.CarriedMove, passed func(module, commit string) bool, - alone func(module string) bool) []string { - first := map[string]string{} - var out []string - for _, n := range nodes { - var unpassed []string - for _, mv := range waiting[n] { - if !passed(mv.Module, mv.To) || alone(mv.Module) { - unpassed = append(unpassed, mv.Module+"@"+mv.To) - } - } - if slices.ContainsFunc(unpassed, func(k string) bool { _, taken := first[k]; return taken }) { - continue - } - for _, k := range unpassed { - first[k] = n - } - out = append(out, n) - } - return out -} - -// silentObservation is what the gate's probe says of the newest release whose rest went silent (novox/hq ADR 0353 -// §4): machines that never reported on their send, left as they were — loud, never a success said by nothing. -func silentObservation(ctx context.Context, inv *inventory.Inventory) ([]conditions.Observation, error) { - recent, err := inv.RecentPlans(ctx, 20) - if err != nil { - return nil, err - } - for _, p := range recent { - if p.Release == nil || p.Release.Settings != nil || p.Open() { - continue - } - if len(p.Release.Silent) == 0 { - return nil, nil - } - machines := append([]string(nil), p.Release.Silent...) - sort.Strings(machines) - o := conditions.Observation{Scope: conditions.ScopeMesh, ID: "release", Token: "silent", Kind: "release-silent", - Severity: conditions.Warning, Resolver: conditions.ResolverOperator, Also: machines, - Summary: fmt.Sprintf("%s: %s said nothing on the send of the builds that waited for a gate within %s, and were "+ - "left as they were", p.ID, strings.Join(machines, ", "), inventory.ApplySilentAfter)} - w := releaseSilentWords(machines) - o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs - return []conditions.Observation{o}, nil - } - return nil, nil -} - -// backlogObservation is what the gate's probe says of a release held after a failure. -func backlogObservation() []conditions.Observation { - if backlogNow.held == "" || len(backlogNow.waiting) == 0 { - return nil - } - n := 0 - var machines []string - seen := map[string]bool{} - var modules []string - for node, moves := range backlogNow.waiting { - n += len(moves) - machines = append(machines, node) - for _, mv := range moves { - if !seen[mv.Module] { - seen[mv.Module] = true - modules = append(modules, mv.Module) - } - } - } - sort.Strings(machines) - sort.Strings(modules) - o := conditions.Observation{Scope: conditions.ScopeMesh, ID: "release", Token: "held", Kind: "release-held", - Severity: conditions.Warning, Resolver: conditions.ResolverOperator, Also: machines, - Summary: fmt.Sprintf("%d build move(s) on %s wait for a gate and are not released: %s — `upgrade backlog` lists "+ - "them, `upgrade release-backlog --why …` releases them", n, strings.Join(machines, ", "), backlogNow.held)} - w := releaseHeldWords(modules, machines) - o.Headline, o.Explanation, o.Resolved, o.Needs = w.Headline, w.Explanation, w.Resolved, w.Needs - // Asked of the operator, who releases them with an Approve (novox/hq issue 1732): the controller's own - // `upgrade release-backlog`, which opens the release plan that walks them through the machines, each judged. - o.Actions = []conditions.Action{{Label: "Release", Verb: "mesh-controller.upgrade", Level: conditions.LevelApprove, - Arguments: map[string]string{"release-backlog": "true", "why": "", "cause": conditions.CauseOperatorAnswer}}} - return []conditions.Observation{o} -} - -// backlogCommand is `upgrade backlog`, read-only, and `upgrade release-backlog --why`. -func backlogCommand(ctx context.Context, sub string, args []string) error { - set := flag.NewFlagSet("upgrade "+sub, flag.ContinueOnError) - why := addHandActFlags(set) - if rest, err := parseAround(set, args); err != nil { - return err - } else if len(rest) > 0 { - return errors.New("upgrade backlog | upgrade release-backlog --why ") - } - if sub == "release-backlog" { - if err := why.require("upgrade release-backlog"); err != nil { - return err - } - } - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - if sub == "release-backlog" { - // Said on the bus (novox/hq ADR 0308 decision 1): the release plan starts walking here, and its start is - // said by the verb that opens it. - var p *inventory.Plan - if err := sayingOnTheBus(ctx, open.inventory, func() error { - release, err := open.inventory.HoldPlans(ctx, true) - if err != nil { - return err - } - defer release() - why.record(ctx, "upgrade release-backlog", nil) - p, err = releaseBacklog(ctx, open, link.Caller()) - return err - }); err != nil { - return err - } - if p == nil { - fmt.Println("nothing waits for a gate on any machine heard from: nothing to release") - return nil - } - fmt.Printf("%s releases it; `plans %s` says where it is\n", p.ID, p.ID) - return nil - } - waiting, err := waitingMoves(ctx, open, false) - if err != nil { - return err - } - if len(waiting) == 0 { - fmt.Println("no build waits for a gate on any machine") - return nil - } - nodes := make([]string, 0, len(waiting)) - for n := range waiting { - nodes = append(nodes, n) - } - sort.Strings(nodes) - for _, n := range nodes { - fmt.Printf("%s: %d build(s) wait for a gate\n", n, len(waiting[n])) - for _, mv := range waiting[n] { - fmt.Printf(" %-28s %s → %s\n", mv.Module, short(mv.From), short(mv.To)) - } - } - return nil -} - -// sayRecreations says, on each move a send carries, what it does to the module's containers (novox/hq -// ADR 0245): the build the machine ran against the one it is sent, container by container. Left unsaid -// for a move whose earlier build is not in the records. -func sayRecreations(ctx context.Context, open *stores, moves []inventory.CarriedMove) { - if len(moves) == 0 { - return - } - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - return - } - for i, mv := range moves { - to, held := shelf[mv.Module] - if !held || mv.From == "" { - continue - } - from, found, err := open.inventory.ManifestAt(ctx, mv.Module, mv.From) - if err != nil || !found { - continue - } - moves[i].Recreates = catalogue.Recreates(from, to).Say(mv.Module) - } -} - -// sayWhatAPushRecreates says, per machine a push is about to send, every module the send moves and what -// the move recreates (novox/hq ADR 0245) — the same words a gated send keeps on its plan. A person's push -// carries every module's new build, whatever its policy and whether or not a gate saw it, so it is the -// send that most needs to say so: on 2026-10-07 a `push` of one machine moved mail to a new build and -// recreated one of its containers with a new image, and the answer said nothing about it. A machine whose -// last send's builds are not known is not said, and neither is a failure to read: the push goes on. -func sayWhatAPushRecreates(ctx context.Context, open *stores, sending []readyNode, w io.Writer) { - if len(sending) == 0 { - return - } - f, err := readMoveFacts(ctx, open.inventory) - if err != nil { - return - } - for _, r := range sending { - sent, known, err := open.inventory.SentBuilds(ctx, r.node) - if err != nil || !known { - continue - } - moves := pushMoves(f, r.node, r.declared.Builds, sent) - if len(moves) == 0 { - continue - } - sayRecreations(ctx, open, moves) - fmt.Fprintf(w, "%s moves %d module(s):\n", r.node, len(moves)) - for _, mv := range moves { - said := mv.Recreates - if said == "" { - said = "recreates none of its containers, or what it ran is not in the records" - } - fmt.Fprintf(w, " %-28s %s → %s: %s\n", mv.Module, short(mv.From), short(mv.To), said) - } - } -} - -// pushMoves is what a send carrying these builds moves on a machine last sent `sent`: every module it ran -// before at a build not identical to the one it is now sent. A module new to the machine is not a move. -func pushMoves(f moveFacts, node string, carries, sent map[string]string) []inventory.CarriedMove { - var out []inventory.CarriedMove - for m, to := range carries { - was, ran := sent[m] - if !ran || was == "" || to == "" || f.identical(m, was, to) { - continue - } - out = append(out, inventory.CarriedMove{Module: m, Node: node, From: was, To: to}) - } - sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) - return out -} - -// recreationsSaid is every recreation a send's moves say, joined: what a plan's note carries. -func recreationsSaid(moves []inventory.CarriedMove) string { - var said []string - for _, mv := range moves { - if mv.Recreates != "" { - said = append(said, fmt.Sprintf("on %s %s", mv.Node, mv.Recreates)) - } - } - return strings.Join(said, "; ") -} - -// The grants step (novox/hq issue 490). -// -// The bus's user list — every module's grants, composed mesh-wide from the catalogue — travels only in the -// declaration of the machine holding the bus. A send to any other machine that changed it put that machine -// first (issue 249), unless a build waited there for a gate: then ungatedIn left it out, and said so. On -// 2026-10-11 a merge gave restic a new health tool; its walk sent restic to its first machine and judged it -// there, the machine's node-engine asked the tool and the bus refused it ("this host's grant does not name -// …"), because the machine holding the bus also runs restic, whose new build waited for that very gate. The -// gate could not pass; a person pushed the bus's machine by hand, which carried restic's new build there -// unjudged. -// -// So before a gated send, when the user list composed now is not the one the machine holding the bus was -// last sent, that machine is sent its declaration with **every build kept at the one it runs**: the new -// list, and nothing of any module's new code. Said on the gate (`plans`), and, when it cannot be sent, said -// with why and passed over: the send goes on as it did before, and its gate says what it finds. - -// brokerBehindOf is brokerBehind: a variable so a test of the walk needs no store. -var brokerBehindOf = brokerBehind - -// grantsStep sends the machine holding the bus its user list alone, ahead of a gated send to node, when the -// list changed; answers what it did, or nil when there was nothing to send. -func grantsStep(ctx context.Context, open *stores, node string) *inventory.GrantsStep { - holder, behind, err := brokerBehindOf(ctx, open, []string{node}) - if err != nil { - fmt.Printf("grants step before %s: whether the bus's user list changed could not be read: %v\n", node, err) - return &inventory.GrantsStep{Node: "the machine holding the bus", Failed: err.Error()} - } - if holder == "" || holder == node || !behind { - // No bus with a user list, the gate's own machine holds it (its send carries the list first), or the - // list is the one already sent. - return nil - } - if _, err := sendRollout(keepingEveryBuild(withScope(ctx, sendScope{}), holder), open, []string{holder}); err != nil { - fmt.Printf("grants step: the bus's user list could not be sent to %s ahead of %s, which is sent without "+ - "it — the bus may refuse what the send newly grants: %v\n", holder, node, err) - return &inventory.GrantsStep{Node: holder, Failed: err.Error()} - } - now := time.Now().UTC() - fmt.Printf("grants step: %s sent the bus's user list alone, every build there kept, before %s is judged\n", - holder, node) - return &inventory.GrantsStep{Node: holder, At: &now} -} - -// errNotGrantsOnly is a grants step refused unsent: its declaration would change more than the user list. -var errNotGrantsOnly = errors.New("the grants step would change more than the bus's user list, and is not sent") - -// grantsOnlyIn is what a grants step's declaration, composed for sending, would change on the machine holding -// the bus besides its user list, against what it was last sent (novox/hq issue 490); empty when nothing else. -// Judged by sendToEach on the very declaration it then sends — one composition, judged, then sent or refused -// unsent. Kept builds are not the whole of a declaration: a new assignment, a settings change, an assignment -// taken away, a provision's logins are composed as the mesh holds them now, and a step that carried any of -// it would be the unjudged change this step exists to avoid. Compared resource by resource with the summary -// the last send kept (ADR 0217): exact, rather than a list of the cases that can differ. A module there that -// was never sent is a new assignment, refused whatever its resources. -func grantsOnlyIn(ctx context.Context, open *stores, holder string, plan catalogue.Resolution, declared sendable) (string, error) { - inv := open.inventory - sent, known, err := inv.SentBuilds(ctx, holder) - if err != nil { - return "", err - } - if !known { - return fmt.Sprintf("what %s was last sent is not known", holder), nil - } - var other []string - listID := "" - for _, m := range plan.Modules { - // A module no build is recorded for is carried with no commit and names none on its key (issue 1109), so it is "sent" - // only by the comparison of the summaries below; a module with a commit that was never sent is a new assignment. - if _, was := sent[m.Module]; !was && declared.Builds[m.Module] != "" { - other = append(other, m.Module+" newly assigned") - } - if m.BusUsers != "" && m.ClaimsSeat(catalogue.BrokerSeat) { - listID = m.Module + "." + catalogue.BusUsersID() - } - } - var facts *moveFacts - for m, was := range sent { - now, carried := declared.Builds[m] - if !carried || sameCommit(now, was) { - continue - } - if facts == nil { - f, err := readMoveFacts(ctx, inv) - if err != nil { - return "", err - } - facts = &f - } - if !facts.identical(m, was, now) { - other = append(other, fmt.Sprintf("%s %s → %s", m, short(was), short(now))) - } - } - for _, f := range declared.foreseen { - other = append(other, f+" would be minted") - } - body, err := declared.Body() - if err != nil { - return "", err - } - after, err := summarize(body) - if err != nil { - return "", err - } - prev, err := inv.SentSummary(ctx, holder) - if err != nil { - return "", err - } - if len(prev) == 0 { - return fmt.Sprintf("what %s was last sent is not kept for comparison", holder), nil - } - var before []sentResource - if err := json.Unmarshal(prev, &before); err != nil { - return "", fmt.Errorf("what %s was last sent is kept in a form this version does not read: %w", holder, err) - } - other = append(other, otherThanTheList(diffSent(before, after), listID)...) - sort.Strings(other) - if len(other) == 0 { - return "", nil - } - return "its send would change more than the bus's user list: " + strings.Join(other, "; "), nil -} - -// otherThanTheList is what a diff against the last send changes besides the user list's content: every -// resource added, removed or changed, but the list's own resource when its content is all that changed — -// its path, mode and every other field must be as last sent (novox/hq issue 490). -func otherThanTheList(d sentDiff, listID string) []string { - var other []string - for _, id := range d.Added { - other = append(other, "+"+id) - } - for _, id := range d.Removed { - other = append(other, "-"+id) - } - for _, c := range d.Changed { - if c.ID == listID && len(c.Fields) == 1 && c.Fields[0] == "content" { - continue - } - other = append(other, fmt.Sprintf("~%s (%s)", c.ID, strings.Join(c.Fields, ", "))) - } - return other -} - -// sendJudged is a gated send's sends: the grants step when the user list changed, then the machine judged. -func sendJudged(ctx context.Context, open *stores, node string) ([]string, *inventory.GrantsStep, error) { - grants := grantsStep(ctx, open, node) - sent, err := sendRollout(withScope(ctx, sendScope{judged: map[string]bool{node: true}}), open, []string{node}) - return sent, grants, err -} - -// grantsSaid is a grants step as `plans` and a walk's note say it. -func grantsSaid(g *inventory.GrantsStep) string { - switch { - case g == nil: - return "" - case g.Failed != "": - return fmt.Sprintf(grantsStepWord+" to %s FAILED, sent without it: %s", g.Node, g.Failed) - default: - return fmt.Sprintf(grantsStepWord+": %s sent the bus's user list first, its builds kept", g.Node) - } -} diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go deleted file mode 100644 index 9ed917a6..00000000 --- a/cmd/mesh-controller/release_plan.go +++ /dev/null @@ -1,2474 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "flag" - "fmt" - "os" - "slices" - "sort" - "strings" - "sync" - "time" - - "github.com/nats-io/nats.go" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/chain" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A merge produces a tiered plan the mesh keeps (novox/hq ADR 0162). -// -// The handler that hears the merge computes the plan from the catalogue's one dependency relation, -// writes it to the store, asks the first tier and returns — the receive loop is never held by a -// build. Every outcome taken in advances the plan it belongs to; a look at the plans, asked for by each event that can -// move one (plan_looks.go, ADR 0406 §5), advances what outcomes alone cannot (a tier waiting for machines to report); -// a controller replaced mid-plan finds the plan where it left it and looks first. - -// gateFirstMachine sends a module that rolls out to one machine first and judges it there (ADR 0236). Off since -// novox/hq ADR 0360: the gate's code is kept for the walks a controller before it started, and its tests turn it on. -var gateFirstMachine = false - -// planWaitBound is how long a plan may wait on one thing before `status` names it red. -const planWaitBound = 30 * time.Minute - -// tiersOf sorts a set of modules into tiers along the ordering edges among them: tier 0 depends -// on nothing else in the set, tier 1 only on tier 0, and so on. An edge to a module outside the set says -// nothing about the order inside it. A cycle — which the catalogue should never produce — puts -// what remains in one last tier rather than losing it, and is said by the caller. -func tiersOf(set []string, edges []inventory.Edge) [][]string { - in := map[string]bool{} - for _, m := range set { - in[m] = true - } - deps := map[string]map[string]bool{} - for _, m := range set { - deps[m] = map[string]bool{} - } - for _, e := range edges { - // A code dependency — B packages A's source — rebuilds B with A, in the same tier: B's - // build needs nothing of A's first. Stands-on and declared order after the base is built, and the - // build seat's holder follows the controller that defines its worker (EdgeWorkerOf, issue 206). - if !in[e.From] || !in[e.To] || e.From == e.To || e.Kind == inventory.EdgePackages { - continue - } - // **Being built by the builder orders nothing** (novox/hq ADR 0360 §3): whichever builder runs builds it. - if e.Kind == inventory.EdgeBuiltBy { - continue - } - deps[e.From][e.To] = true - } - placed := map[string]bool{} - var tiers [][]string - for len(placed) < len(set) { - var tier []string - for _, m := range set { - if placed[m] { - continue - } - free := true - for d := range deps[m] { - if !placed[d] { - free = false - break - } - } - if free { - tier = append(tier, m) - } - } - if len(tier) == 0 { - // A cycle: everything left, together, and the caller says so. - for _, m := range set { - if !placed[m] { - tier = append(tier, m) - } - } - } - sort.Strings(tier) - for _, m := range tier { - placed[m] = true - } - tiers = append(tiers, tier) - } - return tiers -} - -// reachableFrom is the moved modules plus everything that depends on them, through every layer: -// what a merge rebuilds. Along the code and build edges only: a module *built by* the build machine -// is not changed by a new build machine, so a built-by edge orders and gates a plan and never -// widens it — the first plan of 2026-10-01 took the whole catalogue along for a controller change. -func reachableFrom(moved []string, edges []inventory.Edge) []string { - in := map[string]bool{} - for _, m := range moved { - in[m] = true - } - for grew := true; grew; { - grew = false - for _, e := range edges { - // Built-by and worker-of order a plan; neither widens it. A new build machine changes - // nothing it builds, and a new controller changes nothing about the holder it orders. A - // packages edge, read from a record made before novox/hq ADR 0267, widens nothing either: - // a shared file moves each module whose build source holds it, directly. - if e.Kind == inventory.EdgeBuiltBy || e.Kind == inventory.EdgeWorkerOf || e.Kind == inventory.EdgePackages || - e.Kind == edgeChangeOrder { - continue - } - if in[e.To] && !in[e.From] { - in[e.From] = true - grew = true - } - } - } - out := make([]string, 0, len(in)) - for m := range in { - out = append(out, m) - } - sort.Strings(out) - return out -} - -// hasCycle says whether the tiers' last tier holds modules that still depend on each other. A packages -// or built-by edge orders nothing (tiersOf), so a module and what packages its source share a tier by rule: that is -// no cycle, and saying one was is a false report in every such plan's log. -func hasCycle(tiers [][]string, edges []inventory.Edge) bool { - if len(tiers) == 0 { - return false - } - last := map[string]bool{} - for _, m := range tiers[len(tiers)-1] { - last[m] = true - } - for _, e := range edges { - if e.Kind == inventory.EdgePackages || e.Kind == inventory.EdgeBuiltBy { - continue - } - if last[e.From] && last[e.To] { - return true - } - } - return false -} - -// planOfMerge is the plan one merge produces: the moved modules and everything reachable from them, -// tiered, with the merge it answers. -func planOfMerge(m link.SourceMoved, moved []string, edges []inventory.Edge) inventory.Plan { - p := planOfMoves(moved, edges) - merged, _ := time.Parse(time.RFC3339Nano, m.MergedAt) - p.Repository, p.Branch, p.Commit, p.Merged = m.Owner+"/"+m.Repo, m.Base, m.Commit, merged.UTC() - return p -} - -// planOfMoves is the walk of a set of moved modules (novox/hq ADR 0162, 0276): they and everything reachable -// from them, tiered along the graph, with no merge named yet. -func planOfMoves(moved []string, edges []inventory.Edge) inventory.Plan { - set := reachableFrom(moved, edges) - tiers := tiersOf(set, edges) - modules := map[string]*inventory.PlanModule{} - for _, name := range set { - modules[name] = &inventory.PlanModule{} - } - return inventory.Plan{ - ID: fmt.Sprintf("plan-%d", time.Now().UnixNano()), - Created: time.Now().UTC(), - State: inventory.PlanBuilding, - Tiers: tiers, - Modules: modules, - } -} - -// gates is what the next tier needs running from this one: a module of the tier that a later -// tier is built by — the runtime dependency — and whose policy rolls it out, must be applied by -// the machines running it before the next tier is asked. A base an image stands on need only be -// built; a source another module packages need not even be that. -func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool) []string { - if p.Tier >= len(p.Tiers) { - return nil - } - inTier := map[string]bool{} - for _, m := range p.Tiers[p.Tier] { - inTier[m] = true - } - later := map[string]bool{} - for _, tier := range p.Tiers[p.Tier+1:] { - for _, m := range tier { - later[m] = true - } - } - seen := map[string]bool{} - var out []string - // The plan's own order (ADR 0419) is not in the catalogue's graph: it is kept on the plan. - if p.Delivery != nil { - edges = append(append([]inventory.Edge{}, edges...), p.Delivery.Order...) - } - for _, e := range edges { - // A delivery group's order inside a walk (novox/hq ADR 0276 decision 5) gates as built-by does: the - // member before is running on its machines before the member after is asked. - // A built-by edge waits for nothing since novox/hq ADR 0360 §3. - ordered := e.Kind == edgeChangeOrder - if later[e.From] && inTier[e.To] && ordered && !seen[e.To] && rollsOut(e.To) { - seen[e.To] = true - out = append(out, e.To) - } - } - sort.Strings(out) - return out -} - -// noMoveWhy is what a plan says of a module of its tier whose build was made from the source every machine -// running it runs (novox/hq issue 280): nothing is sent for it. -// noMovePrefix opens it, and is what noMove reads: one prefix for the writer and the reader. -const ( - noMovePrefix = "no move" - noMoveWhy = noMovePrefix + ": made from the source every machine running it runs" -) - -// noMove is whether the plan marked the module no move and sent it nothing. Read from what the plan says, -// so a plan kept before the mark had a name of its own is read the same (novox/hq issue 648). -func noMove(s *inventory.PlanModule) bool { - return s != nil && s.FirstAt == nil && s.Gate == nil && strings.HasPrefix(s.Why, noMovePrefix) -} - -// applied says whether every machine running the module has reported since the module was built. -// appliedEach is applied with a moment of its own for each machine: the reports that count are the ones -// after that machine was sent the build (novox/hq issue 256). -func appliedEach(module string, since func(node string) time.Time, running []string, reports []inventory.Reported) (bool, []string) { - at := map[string]*time.Time{} - for _, r := range reports { - at[r.Node] = r.At - } - var waiting []string - for _, n := range running { - if t := at[n]; t == nil || t.Before(since(n)) { - waiting = append(waiting, n) - } - } - return len(waiting) == 0, waiting -} - -// gateNeed is a module the next tier waits for: the machines running it, and from when each must have reported. -type gateNeed struct { - module string - running []string - since func(node string) time.Time -} - -// silentNodes is the machines a walk sent and heard nothing from past ApplySilentAfter, on whichever tier it sent -// them (novox/hq ADR 0360 §7, issue 1160): the one rule both the gates between tiers and the walk's end read, through -// walkReports. A machine that reports on its send is not silent from then on. -func silentNodes(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) map[string]bool { - _, _, silent := walkReports(p, reports, now, prov) - out := map[string]bool{} - for _, n := range silent { - out[n] = true - } - return out -} - -// withoutSilent is the machines of running that are not silent: a later tier is not sent to a machine the walk heard -// nothing from, which keeps its want of the plan's builds for the send it gets when it is heard again (ADR 0360 §7). -func withoutSilent(running []string, silent map[string]bool) []string { - return slices.DeleteFunc(slices.Clone(running), func(n string) bool { return silent[n] }) -} - -// settleTierGate is the wait between tiers on the reports of the machines running what the next tier needs: what is -// still waited for ("module on a, b"), the silent machines left out of the wait, sorted, and whether every machine the -// gate waits on is silent, so that nothing it needs was delivered anywhere (issue 1160). -func settleTierGate(needs []gateNeed, reports []inventory.Reported, silent map[string]bool) (waiting, quiet []string, lost bool) { - all, silentAll := 0, 0 - seenQuiet := map[string]bool{} - seenNode := map[string]bool{} - for _, need := range needs { - for _, n := range need.running { - if !seenNode[n] { - seenNode[n] = true - all++ - if silent[n] { - silentAll++ - } - } - } - _, on := appliedEach(need.module, need.since, need.running, reports) - var still []string - for _, n := range on { - if silent[n] { - if !seenQuiet[n] { - seenQuiet[n] = true - quiet = append(quiet, n) - } - continue - } - still = append(still, n) - } - if len(still) > 0 { - waiting = append(waiting, fmt.Sprintf("%s on %s", need.module, strings.Join(still, ", "))) - } - } - sort.Strings(quiet) - return waiting, quiet, all > 0 && silentAll == all -} - -func applied(module string, builtAt time.Time, running []string, reports []inventory.Reported) (bool, []string) { - at := map[string]*time.Time{} - for _, r := range reports { - at[r.Node] = r.At - } - var waiting []string - for _, n := range running { - if t := at[n]; t == nil || t.Before(builtAt) { - waiting = append(waiting, n) - } - } - return len(waiting) == 0, waiting -} - -// askTier asks the build machine for the named modules of the plan's tier — all of it, or what of it is still -// unasked (novox/hq issue 754) — and marks each asked. A module -// the catalogue no longer holds ends as deleted (issue 1922); one whose ask could not be made is a failure of the -// plan: a tier half asked is a tier that will never complete. -func askTier(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan, names []string) error { - entries, err := inv.Catalogued(ctx) - if err != nil { - return err - } - byName := map[string]inventory.Entry{} - for _, e := range entries { - byName[e.Manifest.Module] = e - } - for _, name := range names { - askModule(ctx, p, name, byName) - } - return nil -} - -// unaskedOf are the modules of a plan's current tier it has still to ask: those it never asked, and one it holds -// built that it never asked for (novox/hq issue 754) — a build taken in by a controller from before a walk took -// only its own asks, somebody else's, which the walk would otherwise send as its own (ADR 0219). A tier partly asked is asked the rest, not -// waited on for ever: a walk that waits for every build of a tier it never asked whole never moves again. -func unaskedOf(p inventory.Plan) []string { - if p.Tier >= len(p.Tiers) { - return nil - } - var out []string - for _, m := range p.Tiers[p.Tier] { - s := p.Modules[m] - switch { - case s == nil || s.State == "": - out = append(out, m) - case s.State == "built" && s.Build == "" && s.AskedAt == nil: - out = append(out, m) - } - } - return out -} - -// askedInTier says the plan asked a module of its current tier itself: under an id, or at a time it kept. -func askedInTier(p inventory.Plan) bool { - if p.Tier >= len(p.Tiers) { - return false - } - for _, m := range p.Tiers[p.Tier] { - if s := p.Modules[m]; s != nil && (s.Build != "" || s.AskedAt != nil) { - return true - } - } - return false -} - -// askModule asks the build machine for one module of a plan and marks it asked, with the id it was -// asked under (novox/hq ADR 0219) — or failed, with the plan, when it could not be asked. -func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[string]inventory.Entry) { - now := time.Now().UTC() - state := p.Modules[name] - if state == nil { - state = &inventory.PlanModule{} - p.Modules[name] = state - } - e, known := byName[name] - if !known { - // **A module the catalogue no longer holds ends as deleted** (issue 1922): the plan's own merge removed it, so - // there is nothing to build or send; it is no failure, and a retry of a plan failed that way resumes. - state.State = planDeleted - state.Why = "no longer in the catalogue" - return - } - // **Every module is taken from the builder's done key, asking no build** (novox/hq ADR 0503 §6, ADR 0501, ADR 0453 §5; - // issue 1851): the look settles it (settleFromKeys). A key not done yet waits, named in the plan, and never fails it. - takeFromKey(p, state, e, now) -} - -// refusalsKeptFor is the refusals kept in this process for these builds (keepRefusalFor), by id. -func refusalsKeptFor(ids []string) map[string]string { - out := map[string]string{} - refusedLeftMu.Lock() - defer refusedLeftMu.Unlock() - for _, id := range ids { - if k, ok := refusedLeft[id]; ok { - out[id] = k.made - } - } - return out -} - -// refusedLeft is why each outcome left to the records was refused, by the build's id: kept in the process beside the -// store's record (RecordRefused), so a store that did not take it still fails the plan while this controller serves. -var ( - refusedLeftMu sync.Mutex - refusedLeft = map[string]madeKept{} -) - -// keepRefusalFor keeps why an outcome left to the records was refused, forgetting refusals older than madeLeftFor. -func keepRefusalFor(id, why string, now time.Time) { - if id == "" || why == "" { - return - } - refusedLeftMu.Lock() - defer refusedLeftMu.Unlock() - for k, v := range refusedLeft { - if now.Sub(v.at) > madeLeftFor { - delete(refusedLeft, k) - } - } - refusedLeft[id] = madeKept{made: why, at: now} -} - -// madeLeft is what each build left to the records did, by the build's id, until a settle takes it: kept in the -// process, so a controller replaced before the settle loses only these words, never the outcome. -var ( - madeLeftMu sync.Mutex - madeLeft = map[string]madeKept{} -) - -type madeKept struct { - made string - at time.Time -} - -// madeLeftFor is how long words nobody settled are kept: a plan that ended or moved on never asks for them. -const madeLeftFor = time.Hour - -// takeMadeOf is what the build of this id did, if its outcome was left to the records with words; taken once. -func takeMadeOf(id string) string { - madeLeftMu.Lock() - defer madeLeftMu.Unlock() - k, ok := madeLeft[id] - if !ok { - return "" - } - delete(madeLeft, id) - return k.made -} - -// startedBeside is the id of a started walk open beside this one that it is related to — a merge's walk past -// its wait, not the backlog's — with why, or empty: a walk waits only for a started walk it shares something with -// (novox/hq ADR 0276 decision 4, ADR 0407 decision 4, issue 973). -func startedBeside(ctx context.Context, inv *inventory.Inventory, p inventory.Plan) (behind, why string, err error) { - plans, err := inv.OpenPlans(ctx) - if err != nil { - return "", "", err - } - started := startedWalks(plans, p.ID, p.Created) - if len(started) == 0 { - return "", "", nil - } - edges, err := inv.Dependencies(ctx) - if err != nil { - return "", "", err - } - onPath, err := ownPathNow(ctx, inv) - if err != nil { - return "", "", err - } - open, err := openWalksOf(ctx, inv, started, onPath) - if err != nil { - return "", "", err - } - mine, err := openWalkOf(ctx, inv, p, onPath) - if err != nil { - return "", "", err - } - behind, why = relatedOpen(mine.fp, mine.own, open, edges) - return behind, why, nil -} - -// refusalNotePrefix and refusalNoteSuffix frame the note of a step the walk was refused: "tier N: — tried -// again". Only a note so framed is a hold (novox/hq issue 920); a tier's summary left in the note is not. -const ( - refusalNotePrefix = "tier " - refusalNoteSuffix = " — tried again" -) - -// refusalNote is the note of a plan whose step at tier was refused with err. -func refusalNote(tier int, err error) string { - return refusalNotePrefix + fmt.Sprint(tier) + ": " + err.Error() + refusalNoteSuffix -} - -// refused says a plan's note is a refusal the walk wrote. -func refused(p inventory.Plan) bool { - return strings.HasPrefix(p.Note, refusalNotePrefix) && strings.HasSuffix(p.Note, refusalNoteSuffix) -} - -// deferredNote begins the note of a walk deferred behind another. -const deferredNote = "let go; starts once " - -// deferred says a walk has its word and has not started: let go while another walk was started, it waits for -// that one to end (startedBeside), and its note says so. Read as a wait, not as a tier running late: `plans` -// and `status` say its note, and S3 leaves it out. A let-go walk whose first ask failed carries that error as -// its note instead, and is watched as before. -func deferred(p inventory.Plan) bool { - return p.Open() && p.Release == nil && p.Tier == 0 && !askedAny(p) && p.Delivery != nil && - p.Delivery.Awaits != "" && p.Delivery.Go != nil && strings.HasPrefix(p.Note, deferredNote) -} - -// currentTierBuilt says every module of a plan's current tier is built (or deleted), so no build is what it waits on. -func currentTierBuilt(p inventory.Plan) bool { - if p.Tier < 0 || p.Tier >= len(p.Tiers) || len(p.Tiers[p.Tier]) == 0 { - return false - } - for _, m := range p.Tiers[p.Tier] { - s := p.Modules[m] - if s == nil || (s.State != "built" && s.State != planDeleted) { - return false - } - } - return true -} - -// askedAny says a plan asked any module. -func askedAny(p inventory.Plan) bool { - for _, s := range p.Modules { - if s != nil && s.State != "" { - return true - } - } - return false -} - -// keepWalkPhases keeps where the walks that ended lately spent their time (novox/hq ADR 0282), outside the hold -// on the plans so it never lengthens it: measured, never acted on, and an error only said. -func keepWalkPhases(ctx context.Context, open *stores) { - if err := recordWalkPhases(ctx, open.inventory, time.Now()); err != nil { - fmt.Printf("plans: the phases of the walks ended lately could not be kept: %v\n", err) - } -} - -// advanceHeld moves every open plan as far as the facts allow, for a caller holding the plans: a tier whose modules -// are all built and whose gates are applied gives way to the next; the last tier done is the plan done. It says -// whether a step failed, which the look at the plans (lookAtPlans) retries. -// -// **One controller at a time** (novox/hq issue 213). A plan is read, changed and saved whole; two controllers — the -// old and the new while a machine hands its controller over — would each ask a tier the other had just asked. -func advanceHeld(ctx context.Context, open *stores) (failed bool) { - inv := open.inventory - // A walk that ended lets the next batch be cut at once, not at a window's next expiry (novox/hq ADR 0276, ADR 0407). - defer func() { - if err := cutBatchesHeld(ctx, open, time.Now().UTC()); err != nil { - fmt.Printf("batches: %v\n", err) - } - }() - plans, err := inv.OpenPlans(ctx) - if err != nil { - fmt.Printf("plans: cannot read them: %v\n", err) - return true - } - if len(plans) == 0 { - return false - } - edges, err := inv.Dependencies(ctx) - if err != nil { - fmt.Printf("plans: cannot read the dependencies: %v\n", err) - return true - } - rollsOut := func(module string) bool { - u, err := inv.UpgradeOf(ctx, module) - return err == nil && u.RollOut - } - for i := range plans { - p := &plans[i] - for p.Open() { - // **Saved only when the step changed it** (novox/hq issue 296). Every step was saved, and - // the steps run on every tick and after every build outcome on the mesh: a plan whose tier - // stood still for half an hour was written every few seconds, its revision moved, `plan-moved` - // said on the bus, and its last save read as when it began building. - before := planSnapshot(*p) - sends := &sendsMade{} - moved, err := advanceOnce(withSendsMade(ctx, sends), open, p, edges, rollsOut) - if err != nil { - failed = true - // Kept in the plan, so `plans` says why it has not moved rather than the log alone; - // the state is left as it was and the step is tried again by the next look. Said and - // kept when it is new: the same refusal on every tick is one fact, not one per tick. - p.Note = refusalNote(p.Tier, err) - // A refusal for the bus's planned step is a person's to end: S17 says it from its first moment. - if refusedForTheBus(err) { - busRefusedFirst.mark(p.ID, time.Now()) - } - if planSnapshot(*p) != before { - fmt.Printf("%s: %v\n", p.ID, err) - if err := inv.SavePlan(ctx, p); err != nil { - fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err) - } - } - break - } - if p.State == inventory.PlanFailed { - sayUnsent(p, rollsOut) - } - // A module the bus did not answer an ask of is asked again by a look, and no event may bring one (a tier all - // refused has no build in flight): the look fails, so the looker retries it from five seconds (issue 1334). - if busWaiting(*p) { - failed = true - } - if planSnapshot(*p) != before { - if err := inv.SavePlan(ctx, p); err != nil { - fmt.Printf("%s: cannot keep the plan: %v\n", p.ID, err) - failed = true - break - } - // Said once the send is kept (ADR 0406 §5, issue 974): what an outsider hears of a tier sent. - saySends(ctx, *p, sends) - } - if !moved { - break - } - } - } - return failed -} - -// planSnapshot is a plan as a save would write it, to compare before and after a step: what the store -// stamps itself (when it was saved, its revision and epoch, when its tier was entered) is left out. -func planSnapshot(p inventory.Plan) string { - p.Updated, p.Revision, p.Epoch, p.TierEntered = time.Time{}, 0, 0, time.Time{} - b, err := json.Marshal(p) - if err != nil { - // Unreadable is never the same: the plan is saved, as every step was before. - return fmt.Sprintf("unmarshallable %p %v", &p, err) - } - return string(b) -} - -// advanceOnce takes one step of one plan and says whether anything changed. -func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, - edges []inventory.Edge, rollsOut func(string) bool) (bool, error) { - inv := open.inventory - if p.Release != nil { - return advanceRelease(ctx, open, p) - } - // **A walk that waits for its delivery's word asks nothing** (novox/hq ADR 0239): nothing of it is - // registered before its turn, so no other send carries it to a machine. - if p.Waiting() { - note := waitingNote(*p) - changed := p.Note != note - p.Note = note - return changed, nil - } - if p.Tier >= len(p.Tiers) { - p.State = inventory.PlanDone - fmt.Printf("%s: done — %s at %s, %d tier(s)\n", p.ID, p.Repository, short(p.Commit), len(p.Tiers)) - return true, nil - } - tier := p.Tiers[p.Tier] - // Not yet asked: ask — the whole tier, or what of it is still unasked (novox/hq issue 754). - if unasked := unaskedOf(*p); len(unasked) > 0 { - // **One walk at a time beside a related one** (novox/hq ADR 0276, ADR 0407 decision 4, issue 973): a walk - // about to ask its first tier while a started walk it is related to is open waits for that one to end, - // let go or not, and says so and why. A started walk it shares nothing with does not hold it. - if p.Tier == 0 && !askedInTier(*p) { - if behind, why, err := startedBeside(ctx, inv, *p); err != nil { - return false, err - } else if behind != "" { - note := fmt.Sprintf("%s%s ended — %s", deferredNote, behind, why) - changed := p.Note != note - p.Note = note - return changed, nil - } - } - // **A backup before the send** (novox/hq ADR 0404, issue 1065): a walk that starts asks the backup holders of - // the machines running a moved module that keeps data, and asks its first tier only when every one is done. - if p.Tier == 0 && !askedInTier(*p) { - if proceed, err := backupsFirst(ctx, inv, p); err != nil { - return false, err - } else if !proceed { - return false, nil - } - } - if len(unasked) < len(tier) { - fmt.Printf("%s: tier %d was asked in part; asking the rest: %s\n", p.ID, p.Tier, strings.Join(unasked, ", ")) - } - note := p.Note - if err := askTier(ctx, inv, p, unasked); err != nil { - return false, err - } - // Nothing asked and nothing new to say (the bus still silent, issue 1334): not saved again at every look. - return len(unaskedOf(*p)) < len(unasked) || p.Note != note || p.State != inventory.PlanBuilding, nil - } - // **Asked: settle from the build records first** (novox/hq 04-ISSUES/214). An outcome is taken - // in by whichever controller hears it, and a merge to the controller's own repository replaces - // the controller in its first tier: the build that produced the new one is recorded, and the - // plan never hears it. The record is the fact; a build recorded after the ask is that tier's - // outcome, whoever was listening. - // **Taken from the builder's keys first** (ADR 0503 §6, issue 1851): every module of the tier but the builder's own. - if keyed(p, tier) { - keys, closeKeys, err := openPlanKeys() - if err != nil { - return false, err - } - was := p.State - changed, err := settleFromKeys(ctx, open, keys, p, tier) - closeKeys() - if err != nil { - return false, err - } - if p.State == inventory.PlanFailed && was != inventory.PlanFailed { - sayUnsent(p, rollsOut) - } - if changed { - return true, nil - } - } - recorded, byID, err := recordsOfAsked(ctx, inv, p, tier) - if err != nil { - return false, err - } - before, was := statesOf(p), p.State - if settleFromRecords(p, tier, recorded, byID) { - // As taking the outcome directly does (planBuiltSaying): a module deleted at its source is forgotten, and a - // failed plan says what it built and never sent (issue 1708). - deleted, failed := settledEffects(before, p, was) - for _, m := range deleted { - forgetDeleted(ctx, inv, m, p) - } - if failed { - sayUnsent(p, rollsOut) - } - return true, nil - } - // Asked: wait for every build. - var latest time.Time - for _, m := range tier { - s := p.Modules[m] - if s == nil || (s.State != "built" && s.State != planDeleted) { - return false, nil - } - if s.BuiltAt != nil && s.BuiltAt.After(latest) { - latest = *s.BuiltAt - } - } - // Built: send every module of the tier whose policy rolls out, once, to the machines running - // it — whether or not its source commit moved. A dependent rebuilt because its base moved, or - // a module that packages another repository's source, keeps its commit; the catalogue announces - // no move for it and its machines would keep the old image until somebody pushed (novox/hq - // issue 189). A module whose policy records is built and left, as its policy says. - // - // **One machine first, unless the module's policy says together** (novox/hq issue 249, ADR - // 0218). The plan sent every machine running the module at once, and the operator's policy — - // one at a time, stopping at the first that fails, which an announced upgrade honours — was not - // read here at all: a module whose new declarations broke it broke everywhere in the same - // minute. Now the first machine is sent, the plan records it and waits for that machine's report - // after the send to say it applied what it was sent; only then are the rest sent. A first machine - // that fails or refuses stops the module's rollout and the plan with it, the rest untouched. - // - // **One send per machine for the whole tier** (novox/hq issue 281). A send carries the machine's - // whole declaration (ADR 0221), and the plan sent the first machine once per module: on 2026-10-06 a - // tier of seventy modules sent one laptop a new declaration every few seconds, the node-engine set - // aside one for the next, the node tools never settled, and the gate blamed a module for the machine - // being behind. Now every module of the tier whose first machine is the same is sent there in one - // send, judged there by one gate — kept on the first of them, the others gated by it — and the rest - // of the tier's machines are sent once, together, when the gates have passed. - // **A machine silent past the bound is left as it was, on every tier** (novox/hq ADR 0360 §7, issue 1160): the - // walk's end names it, and the gates between tiers and the sends of the tiers after do not wait for it or go to it. - allReports, err := inv.LastReports(ctx) - if err != nil { - return false, err - } - silent := silentNodes(*p, allReports, time.Now().UTC(), provLookup(ctx)) - // **A machine away when a tier is sent is not sent to and not waited for** (ADR 0435 §1, issue 1161): read once, - // for a tier that has something left to send. - var away map[string]bool - for _, m := range tier { - if s := p.Modules[m]; s != nil && s.SentAt == nil && s.FirstAt == nil && s.State != planDeleted { - away = judgedAway(ctx, open, time.Now().UTC()) - break - } - } - var pending []string - firstOn := map[string][]string{} - runningOf := map[string][]string{} - var rest []string - restTo := map[string][]string{} - together := map[string]bool{} - for _, m := range tier { - state := p.Modules[m] - if state == nil || state.SentAt != nil || state.State == planDeleted { - continue - } - // **A recorded module goes where its move recreates nothing and moves no data** (novox/hq ADR 0377): judged - // per machine against the build it was last sent, said per machine, and sent only there. - var goes map[string]bool - if !rollsOut(m) { - out, err := judgeRecordedInWalk(ctx, inv, p, m, state) - if err != nil { - return false, err - } - if len(out) == 0 { - continue - } - goes = map[string]bool{} - for _, n := range out { - goes[n] = true - } - } - running, err := inv.Running(ctx, m) - if err != nil { - return false, err - } - if goes != nil { - running = slices.DeleteFunc(running, func(n string) bool { return !goes[n] }) - } - if state.FirstAt == nil { - running = withoutSilent(running, silent) - passed := withoutSilent(running, away) - if len(passed) < len(running) { - for _, n := range running { - if away[n] && !slices.Contains(p.Away, n) { - fmt.Printf("%s: %s is away; %s is not sent to it and not waited for\n", p.ID, n, m) - } - } - rememberAway(p, running, away) - running = passed - } - } - runningOf[m] = running - // **A build whose source is unchanged is never a move** (novox/hq issue 280): made from the source - // of the build every machine running it was sent, it is registered with that build's artifacts — - // nothing to send, and nothing for a gate to judge. - if state.FirstAt == nil { - if same, err := unchangedOnEvery(ctx, inv, m, state.Build, running); err != nil { - return false, err - } else if same { - now := time.Now().UTC() - state.SentAt = &now - state.Why = noMoveWhy - fmt.Printf("%s: %s built from the source every machine running it runs: no move, nothing sent\n", p.ID, m) - continue - } - } - policy, err := inv.UpgradeOf(ctx, m) - if err != nil { - return false, err - } - // **Every machine at once** (novox/hq ADR 0360 §1, §2): a module that rolls out is sent to every machine - // running it in one step, with no first machine and no gate, whatever module it is. A module a controller - // before ADR 0360 already sent to its first machine finishes under its gate. - if !gateFirstMachine && state.FirstAt == nil { - policy.Together = true - } - var reports []inventory.Reported - if !policy.Together { - // Read for the choice of the first machine as well as for its report. - if reports, err = inv.LastReports(ctx); err != nil { - return false, err - } - } - now := time.Now().UTC() - // **Sent with others, judged with them** (issue 281): the gate of the send that carried it is its - // verdict on its first machine. A failure there stopped the plan already. - if state.GatedBy != "" { - lead := p.Modules[state.GatedBy] - if lead == nil || lead.Gate == nil || lead.Gate.Verdict == "" { - pending = append(pending, fmt.Sprintf("%s judged with %s on %s", m, state.GatedBy, strings.Join(state.First, ", "))) - continue - } - if lead.Gate.Verdict != inventory.GatePassed { - continue - } - passedWith(m, state, state.GatedBy, lead.Gate) - } - // Read after its pass is taken over from the send that carried it, so a passed gate is never judged - // again from the first machine's later reports (novox/hq issue 335). - step := nextRollout(*state, running, policy.Together, reports, now, planWaitBound) - switch { - case step.failed != "": - // The first machine refused or failed what it was sent, or never said: the gate failed, and - // what it carried is put back there (novox/hq ADR 0236); the rest are left as they were. - failFirstSend(ctx, open, p, m, state, firstRunning(state.First, running), step.failed, step.rest) - return true, nil - case step.waiting != "": - at := "" - if state.FirstAt != nil { - at = state.FirstAt.Local().Format("15:04") - } - pending = append(pending, fmt.Sprintf("%s on %s, sent first at %s", m, step.waiting, at)) - continue - } - // **The gate** (novox/hq ADR 0236, to-be 45 §8): the first machine reported the build applied; - // it is judged by its health before anything else is sent — the rest, or, where it is the only - // machine, the plan's next step. - if state.GatedBy == "" && !policy.Together && state.FirstAt != nil && len(state.First) > 0 { - verdict, err := judgeGate(ctx, open, p, m, state, running, now) - if err != nil { - return false, err - } - switch verdict { - case "": - if putBackBroken(ctx, open, p, state.Gate, m, state) { - return true, nil - } - pending = append(pending, fmt.Sprintf("%s judged on %s: %s", m, - strings.Join(state.Gate.Machines, ", "), gateLine(state.Gate))) - continue - case inventory.GateFailed: - failFirstSend(ctx, open, p, m, state, state.Gate.Machines, state.Gate.Why, step.rest) - return true, nil - case inventory.GateSuperseded: - // Another send moved this module on its first machine (novox/hq issue 352): the build is not - // judged, not marked, not put back; the plan ends here, said, and a newer plan carries on. - state.Why = "superseded: " + state.Gate.Why - p.State = inventory.PlanSuperseded - p.Note = fmt.Sprintf("%s's judging on %s was superseded: %s", m, strings.Join(state.Gate.Machines, ", "), - state.Gate.Why) - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil - case inventory.GatePassed: - if !state.Gate.Kept { - gatePassed(ctx, open, p, m, state) - state.Gate.Kept = true - } - } - } - switch { - case len(step.send) == 0: - // No machine runs it: nothing to send, and nothing to wait for. - state.SentAt = &now - continue - case step.first: - firstOn[step.send[0]] = append(firstOn[step.send[0]], m) - continue - } - rest = append(rest, m) - restTo[m] = step.send - together[m] = policy.Together - } - // The first machines: one send each for everything of the tier that goes there first, one machine - // per step — the plan is kept between them, so the next machine's send sees what this one walks. - if len(firstOn) > 0 { - machines := make([]string, 0, len(firstOn)) - for n := range firstOn { - machines = append(machines, n) - } - sort.Strings(machines) - for _, node := range machines { - err := firstSend(ctx, open, p, node, firstOn[node], runningOf) - if errors.Is(err, errWalkedElsewhere) { - // A build this machine would carry is being judged elsewhere: it goes once that gate passed. - pending = append(pending, fmt.Sprintf("%s for %s: %v", node, strings.Join(firstOn[node], ", "), err)) - continue - } - if err != nil { - // Not marked sent, so the next step tries again (issue 249): a grant that could not be - // issued is a send that did not happen. - return false, fmt.Errorf("sending %s to %s first after tier %d: %w", strings.Join(firstOn[node], ", "), - node, p.Tier, err) - } - return true, nil - } - } - // The rest, after their gates passed, and what rolls out together: every machine once. - if len(rest) > 0 { - var machines []string - scope := sendScope{modules: map[string]bool{}} - for _, m := range rest { - for _, n := range restTo[m] { - if !slices.Contains(machines, n) { - machines = append(machines, n) - } - } - if together[m] { - // What its policy rolls out together may move wherever it goes; anything else no gate has - // seen still stops the send. - scope.modules[m] = true - } - } - sort.Strings(machines) - was := map[string]map[string]chain.Ref{} - for _, n := range machines { - was[n] = ranBefore(ctx, inv, n) - } - sent, err := sendRollout(withScope(sendingAs(ctx, p), scope), open, machines) - if err != nil { - return false, fmt.Errorf("sending %s to %s after tier %d: %w", strings.Join(rest, ", "), - strings.Join(machines, ", "), p.Tier, err) - } - now := time.Now().UTC() - // What each machine of the rest was sent, kept for when it reports it applied (novox/hq ADR 0282 decision - // 6): measured, never acted on. - sentWhat := sentNow(ctx, open.inventory, sent) - for _, m := range rest { - p.Modules[m].SentAt = &now - p.Modules[m].Rest = restOf(restTo[m], sent, sentWhat) - } - // One `sent` for each machine, with what it was sent (ADR 0406 §5, issue 974). - for _, n := range sent { - var carried []string - for _, m := range rest { - if slices.Contains(restTo[m], n) { - carried = append(carried, m) - } - } - noteSends(ctx, p.Tier, []string{n}, carried, now) - noteChanges(p, n, carried, was[n], now) - } - fmt.Printf("%s: tier %d built; sent %s to %s, one send each\n", p.ID, p.Tier, strings.Join(rest, ", "), - strings.Join(sent, ", ")) - return true, nil - } - if len(pending) > 0 { - note := "tier " + fmt.Sprint(p.Tier) + " built; waiting for " + strings.Join(pending, "; ") + - " to report it applied before the rest are sent" - changed := p.State != inventory.PlanRolling || p.Note != note - p.State = inventory.PlanRolling - p.Note = note - return changed, nil - } - // And wait for what the next tier needs running. - leftAsWas := "" - needed := gates(*p, edges, rollsOut) - if len(needed) > 0 { - reports, err := inv.LastReports(ctx) - if err != nil { - return false, err - } - var needs []gateNeed - for _, m := range needed { - running, err := inv.Running(ctx, m) - if err != nil { - return false, err - } - state := p.Modules[m] - if state == nil { - state = &inventory.PlanModule{} - p.Modules[m] = state - } - if state.State == planDeleted { - continue - } - // A machine passed as away was sent nothing to wait on (ADR 0435 §1, issue 1161). - running = runningHeard(*p, running, state) - // The plan sends what it waits for. A rebuild from the same source commit is not a - // move the catalogue announces — the build machine rebuilt for a controller change - // is one — so the roll-out that opens this gate is the plan's to make, once, and - // the reports that open it are the ones after the send. - since := latest - if state.BuiltAt != nil { - since = *state.BuiltAt - } - // **A build that is no move is waited on by the send that last carried it** (novox/hq issue 648). - // Nothing was sent for it, so no report follows its marking or its build: a machine that does not - // happen to report again held the walk for ever. Each machine runs it once it answered its last - // send — the one whose builds made it no move — whenever that was. - if noMove(state) { - byNode := map[string]inventory.Reported{} - for _, r := range reports { - byNode[r.Node] = r - } - needs = append(needs, gateNeed{m, running, func(node string) time.Time { - if s := byNode[node].Sent; s != nil { - return *s - } - return time.Time{} - }}) - continue - } - // **Each machine from its own send** (novox/hq issue 256). With one machine first (ADR 0218) - // a module is sent twice — the first machine, then the rest — and SentAt is the second. - // Asked of every machine, the first machine's report, made between the two sends, read as - // older than the build, and the gate waited for a report it already had, for ever. - sinceFor := func(node string) time.Time { - at := since - sent := state.SentAt - for _, n := range state.First { - if n == node { - sent = state.FirstAt - } - } - if sent != nil && sent.After(at) { - at = *sent - } - return at - } - needs = append(needs, gateNeed{m, running, sinceFor}) - } - waiting, quiet, lost := settleTierGate(needs, reports, silent) - if lost { - // Every machine the next tier waits on is silent: nothing it needs was delivered anywhere, so the plan does - // not go on as if it had been (issue 1160). - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("tier %d built; %s %s: %s", p.Tier, silentGateWords, applySilentAfter, strings.Join(quiet, ", ")) - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil - } - if len(quiet) > 0 { - fmt.Printf("%s: tier %d: %s said nothing on its send within %s and is left as it was; the plan goes on without it\n", - p.ID, p.Tier, strings.Join(quiet, ", "), applySilentAfter) - leftAsWas = fmt.Sprintf("%s said nothing on its send within %s and is left as it was", strings.Join(quiet, ", "), - applySilentAfter) - } - if len(waiting) > 0 { - note := "tier " + fmt.Sprint(p.Tier) + " built; waiting for " + strings.Join(waiting, "; ") + " to be applied" - changed := p.State != inventory.PlanRolling || p.Note != note - p.State = inventory.PlanRolling - p.Note = note - return changed, nil - } - } - // **A walk ends on its machines' reports** (novox/hq ADR 0360 §4, §7): after its last tier, every machine it sent - // has reported on that send, or is silent past ApplySilentAfter; never a health reading. - end := "" - if p.Tier == len(p.Tiers)-1 { - reports, err := inv.LastReports(ctx) - if err != nil { - return false, err - } - waiting, failed, silent := walkReports(*p, reports, time.Now().UTC(), provLookup(ctx)) - if len(waiting) > 0 { - note := "sent; waiting for " + strings.Join(waiting, ", ") + " to report on their send" - changed := p.State != inventory.PlanRolling || p.Note != note - p.State = inventory.PlanRolling - p.Note = note - return changed, nil - } - var said []string - if len(failed) > 0 { - said = append(said, "failed or refused its send (apply-failing says it): "+strings.Join(failed, ", ")) - } - if len(silent) > 0 { - said = append(said, fmt.Sprintf("said nothing on its send within %s: %s", applySilentAfter, - strings.Join(silent, ", "))) - } - said = append(said, recordedSaid(*p)...) - end = strings.Join(said, "; ") - // **The new controller's user list follows its walk** (ADR 0236 §34, read without the gate since ADR 0360): - // once a walk that moved the controller ends, the bus's machine is sent the list it composes, if behind. - if s := p.Modules[catalogue.ControllerSeatName]; s != nil && s.SentAt != nil && s.Gate == nil && !noMove(s) { - carryUserList(ctx, open, p) - } - } - if end == "" { - end = leftAsWas - } - // A machine passed as away is said where the others passed are said (ADR 0435 §1, issue 1161). - if said := awaySaying(*p); said != "" { - end = strings.TrimPrefix(end+"; "+said, "; ") - } - p.Tier++ - p.State = inventory.PlanBuilding - p.Note = end - if p.Tier < len(p.Tiers) { - fmt.Printf("%s: tier %d done; asking tier %d: %s\n", p.ID, p.Tier-1, p.Tier, strings.Join(p.Tiers[p.Tier], ", ")) - } - return true, nil -} - -// walkReports is, for every machine a walk sent its rest, whether it reported on that send (novox/hq ADR 0360 §4): -// those still waiting, those that reported a failed or refused apply, and those silent past ApplySilentAfter, -// each sorted. Pure. -func walkReports(p inventory.Plan, reports []inventory.Reported, now time.Time, prov inventory.ProvLookup) (waiting, failed, silent []string) { - byNode := map[string]inventory.Reported{} - for _, r := range reports { - byNode[r.Node] = r - } - // The modules of this delivery sent to each machine: a machine is judged on those, not on every module its - // declaration hands to the provisioner (ADR 0362 §1-2, issue 840). - carried := map[string][]string{} - for m, s := range p.Modules { - if s == nil { - continue - } - for node := range s.Rest { - carried[node] = append(carried[node], m) - } - } - // **Each machine on the walk's last send to it** (novox/hq ADR 0531, issue 1886): a send carries the machine's whole declaration (ADR - // 0221), so a report on the last one answers every earlier one, and the bound runs from it. Judged on every send - // it had made, a walk whose tiers were sent hours apart — R4e, stopped at tier 1 and resumed — held tier 0's send of - // 00:23 against tier 1's templates and named its machines silent 35 seconds after tier 1 was sent. - last := map[string]inventory.SentDeclaration{} - lastAt := map[string]time.Time{} - for _, s := range p.Modules { - if s == nil { - continue - } - for node, sent := range s.Rest { - at := sent.At - if at.IsZero() && s.SentAt != nil { - at = *s.SentAt - } - was, ok := last[node] - if !ok || laterSend(sent, at, was, lastAt[node]) { - last[node], lastAt[node] = sent, at - } - } - } - for node, sent := range last { - at := lastAt[node] - sent = sent.Only(carried[node]...) - r := byNode[node] - // A module handed to the provisioner is applied only once its client said so too (ADR 0362 §1-5). - outcome, reported := provisionedOutcome(node, sent, r, r.At != nil && sent.ReportsOn(r), prov) - switch { - case reported: - switch { - case outcome == inventory.OutcomeFailed || outcome == inventory.OutcomeRefused: - failed = append(failed, node) - case strings.HasPrefix(outcome, inventory.OutcomeProvisioningFailed): - failed = append(failed, node+" ("+outcome+")") - } - case !at.IsZero() && now.Sub(at) > applySilentAfter: - silent = append(silent, node) - default: - waiting = append(waiting, node) - } - } - sort.Strings(waiting) - sort.Strings(failed) - sort.Strings(silent) - return waiting, failed, silent -} - -// laterSend is whether a, sent at aAt, is a later send to a machine than b, sent at bAt: by its sequence when both carry -// one, else by when it went. -func laterSend(a inventory.SentDeclaration, aAt time.Time, b inventory.SentDeclaration, bAt time.Time) bool { - if a.Sequence > 0 && b.Sequence > 0 && a.Sequence != b.Sequence { - return a.Sequence > b.Sequence - } - return aAt.After(bAt) -} - -// restOf is, for one module, every machine of its rest that the send reached and what it carried there. -func restOf(to, sent []string, what map[string]inventory.SentDeclaration) map[string]inventory.SentDeclaration { - out := map[string]inventory.SentDeclaration{} - for _, n := range to { - if slices.Contains(sent, n) { - out[n] = what[n] - } - } - if len(out) == 0 { - return nil - } - return out -} - -// firstSend sends one machine, in one send, every module of the plan's tier whose first machine it is -// (novox/hq issue 281), and records the send on each: what that machine ran of it before — read once, -// before the send, so a module the same send carries is never read as already moved — the machines it -// went to, and the gate. The gate is kept on the first of them and judges everything the send moved; -// the others are gated by it. -func firstSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string, - runningOf map[string][]string) error { - inv := open.inventory - before, beforeKnown, _ := inv.SentBuilds(ctx, node) - // **A machine already sent what this send is for is not sent again** (novox/hq ADR 0353 §6): sent by this plan - // before a controller was replaced and before it was kept, or by another walk. Judged from that send, and from - // the readings of a gate already judging it there. - // Only a send the machine reported it applied is one it has; one it never reported on is sent, as before - // (issue 280: what it ran before is then that build, and a failure leaves it as it was). - if beforeKnown && slices.IndexFunc(modules, func(m string) bool { return !sameCommit(before[m], p.Modules[m].Commit) }) < 0 && - appliedLastSend(ctx, inv, node) { - if others, err := gatedMoves(ctx, open, node, nil); err == nil && len(others) == 0 { - adoptSend(ctx, open, p, node, modules, before, runningOf) - return nil - } - } - // **What the machine ran before is kept before the send is made** (novox/hq ADR 0353 §6): a controller replaced - // between the send and keeping it reads it here, and a failure still puts the machine back. - kept := false - for _, m := range modules { - s := p.Modules[m] - if beforeKnown && (s.Previous == "" || !sameCommit(before[m], s.Commit)) && s.Previous != before[m] { - s.Previous, kept = before[m], true - } - } - if kept { - if err := inv.SavePlan(ctx, p); err != nil { - return fmt.Errorf("what %s ran before is not kept, so nothing is sent: %w", node, err) - } - } - owns := make([]inventory.CarriedMove, 0, len(modules)) - for _, m := range modules { - s := p.Modules[m] - owns = append(owns, inventory.CarriedMove{Module: m, Node: node, From: s.Previous, To: s.Commit, Build: s.Build}) - } - // What the assignments name before the send, for the rollout to say what it changed; and the send is made as this - // rollout, so what is recorded of it names it (novox/hq ADR 0408, issue 1095). - was := ranBefore(ctx, inv, node) - ctx = sendingAs(ctx, p) - // A gated send (ADR 0236): everything waiting on the machine goes with the tier, and the gate judges - // all of it there. - carried, sent, grants, err := gatedSend(ctx, open, node, owns) - if err != nil { - return err - } - now := time.Now().UTC() - lead := modules[0] - // What each machine was just sent, kept on the gate (novox/hq issue 352): its report is held against - // this send, whatever it is sent after. - sentWhat := sentNow(ctx, inv, sent) - for _, m := range modules { - s := p.Modules[m] - // What the first machine ran before (ADR 0236), kept above before the send. - s.First, s.FirstAt = sent, &now - s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning(sent, runningOf[m]), - From: s.Previous, To: s.Commit, Since: &now, Sent: sentWhat} - s.GatedBy = "" - if m == lead { - s.Gate.Carried = carried - s.Gate.Grants = grants - } else { - s.GatedBy = lead - } - } - noteSends(ctx, p.Tier, sent, modules, now) - for _, n := range sent { - noteChanges(p, n, modules, was, now) - } - p.State = inventory.PlanRolling - p.Note = fmt.Sprintf("tier %d built; sent %s to %s first, in one send", p.Tier, strings.Join(modules, ", "), - strings.Join(sent, ", ")) - fmt.Printf("%s: tier %d built; sent %d module(s) to %s first in one send (%s), the rest once its gate passes\n", - p.ID, p.Tier, len(modules), strings.Join(sent, ", "), strings.Join(modules, ", ")) - // The grants step taken before it (novox/hq issue 490), said with it. - if said := grantsSaid(grants); said != "" { - p.Note += "; " + said - } - // What the send recreates, said with it (novox/hq ADR 0245). - if said := recreationsSaid(carried); said != "" { - p.Note += "; " + said - fmt.Printf("%s: %s\n", p.ID, said) - } - return nil -} - -// adoptSend records a first send the machine already had (novox/hq ADR 0353 §6, §7): nothing is sent; the gate counts -// from the machine's last send, and takes over the readings of an open gate of another walk judging the same build -// there — whose carried move also says what the machine ran before, for a rollback. -func adoptSend(ctx context.Context, open *stores, p *inventory.Plan, node string, modules []string, - before map[string]string, runningOf map[string][]string) { - inv := open.inventory - since := time.Now().UTC() - last, found, err := inv.SentTo(ctx, node) - if err == nil && found && !last.At.IsZero() { - since = last.At - } - lead := modules[0] - other, id, from := judgingThere(ctx, inv, p.ID, lead, node, p.Modules[lead].Commit, p.Modules[lead].Build) - for _, m := range modules { - s := p.Modules[m] - // What it ran before: kept by this walk before its send was made (§6); else what the other walk's send - // moved it from; else the build itself, so a failure leaves it as it was (issue 280). - switch { - case s.Previous != "": - case m == lead && from != "": - s.Previous = from - default: - s.Previous = before[m] - } - at := since - s.First, s.FirstAt = []string{node}, &at - s.Gate = &inventory.PlanGate{Component: coreComponent(m), Machines: firstRunning([]string{node}, runningOf[m]), - From: s.Previous, To: s.Commit, Since: &at} - if found { - s.Gate.Sent = map[string]inventory.SentDeclaration{node: last} - } - s.GatedBy = "" - if m == lead { - s.Gate.TakeReadings(other, modules, id) - } else { - s.GatedBy = lead - } - } - p.State = inventory.PlanRolling - p.Note = fmt.Sprintf("tier %d built; %s already runs %s's build: nothing sent again, judged from that send", - p.Tier, node, strings.Join(modules, ", ")) - if id != "" { - p.Note += ", going on from the readings of " + id - } - fmt.Printf("%s: %s\n", p.ID, p.Note) -} - -// appliedLastSend says a machine reported it applied the declaration it was sent last. -func appliedLastSend(ctx context.Context, inv *inventory.Inventory, node string) bool { - reports, err := inv.LastReports(ctx) - if err != nil { - return false - } - for _, r := range reports { - if r.Node == node { - return r.Current && r.At != nil && r.Outcome == inventory.OutcomeApplied - } - } - return false -} - -// judgingThere is an open gate of another walk judging a module's build on a machine (novox/hq ADR 0353 §7), the -// walk's id, and what the machine ran before; nil when none does. -func judgingThere(ctx context.Context, inv *inventory.Inventory, self, module, node, commit, build string) (*inventory.PlanGate, string, string) { - plans, err := inv.OpenPlans(ctx) - if err != nil { - return nil, "", "" - } - // The same build, by its id where both name one, else by its commit. - same := func(b, c string) bool { - if b != "" && build != "" { - return b == build - } - return sameCommit(c, commit) - } - usable := func(g *inventory.PlanGate) bool { - return g != nil && !g.Pending && g.Since != nil && (g.Verdict == "" || g.Verdict == inventory.GatePassed) && - slices.Contains(g.Machines, node) - } - for _, q := range plans { - if q.ID == self { - continue - } - if r := q.Release; r != nil { - gates := []*inventory.PlanGate{r.Gate} - for _, g := range r.Gates { - gates = append(gates, g) - } - for _, g := range gates { - if !usable(g) { - continue - } - for _, c := range g.Carried { - if c.Module == module && c.Node == node && same(c.Build, c.To) { - return g, q.ID, c.From - } - } - } - continue - } - if s := q.Modules[module]; s != nil && usable(s.Gate) && same(s.Build, s.Commit) { - return s.Gate, q.ID, s.Previous - } - } - return nil, "", "" -} - -// passedWith keeps, on a module sent with others, the verdict of the gate that judged the send: the gate -// on the first of them passed, and its pass was kept for every build it carried (passCarried). -func passedWith(module string, s *inventory.PlanModule, lead string, g *inventory.PlanGate) { - if s.Gate == nil { - s.Gate = &inventory.PlanGate{Machines: g.Machines, From: s.Previous, To: s.Commit, Since: g.Since} - } - if s.Gate.Verdict != "" { - return - } - s.Gate.Verdict, s.Gate.Why, s.Gate.JudgedAt, s.Gate.Took = g.Verdict, "judged with "+lead+": "+whyFor(g, module), g.JudgedAt, g.Took - s.Gate.Passes, s.Gate.Kept = g.Passes, true -} - -// failFirstSend stops the plan at a first send whose gate failed, or whose machine refused or failed -// what it was sent: what the gate found wanting is put back there, each once — the module the gate is -// kept on only when it is among them (issue 281: the module whose gate it is was blamed for whatever -// the send carried) — and the machines after it are left as they were. -func failFirstSend(ctx context.Context, open *stores, p *inventory.Plan, module string, state *inventory.PlanModule, - machines []string, why string, rest []string) { - batched, back := batchingRollbacks(ctx) - defer func() { - sendRollbacks(ctx, open, p, back) - p.Note += fmt.Sprintf("; %s left as it was", orNone(strings.Join(rest, ", "))) - fmt.Printf("%s: %s\n", p.ID, p.Note) - }() - g := state.Gate - if g != nil && g.Verdict == inventory.GatePassed { - // **A guard: a build that passed its gate is never put back for what came after** (novox/hq issue 335). - // Not reached while nextRollout answers a passed gate with the rest to send, and advanceOnce reads it - // after a carried module takes over its lead's pass; it is here so that a path added later cannot - // overturn a verdict. If it is reached, the plan stops and says why, and the build is not marked failed. - // The module is left a stopped rollout (its Why said, sent first and not to the rest), which - // `plans retry` takes: it sends the first machine again, and the passed gate then sends the rest. - state.Why = "passed its gate; its walk then stopped: " + why - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("%s passed its gate on %s in tier %d (%s) and is kept; its walk stopped after: %s", - module, strings.Join(g.Machines, ", "), p.Tier, g.Why, why) - return - } - if g != nil && slices.Contains(g.Returned, module) { - // Put back at once when it broke: its rollback was made then, and is not made again. - state.Why = "put back when it broke; its send failed: " + g.Why - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s was put back when it broke", - strings.Join(g.Machines, ", "), p.Tier, g.Why, module) - } else if g == nil || g.Verdict == "" || len(g.Failing) == 0 || slices.Contains(g.Failing, module) { - gateFailed(batched, open, p, module, state, machines, why) - } else if slices.Contains(g.Passing, module) && state.Build != "" { - // **The module the gate is kept on keeps its own pass too** (issue 318 review): healthy for the passes - // asked while another of its send failed, its build has a verdict, and other walks do not wait on it. - if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: state.Build, Module: module, - Commit: state.Commit, Previous: state.Previous, Plan: p.ID, Machines: g.Machines, Verdict: inventory.GatePassed, - Why: passedAloneWhy(g, module), Component: g.Component, JudgingFrom: g.Since}); err != nil { - fmt.Printf("%s: %s passed its gate on its own, and the verdict could not be kept: %v\n", p.ID, module, err) - } - state.Why = "passed on its own; stopped with the send that carried it: " + g.Why - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s passed on its own and is kept", - strings.Join(g.Machines, ", "), p.Tier, g.Why, module) - } else { - state.Why = "not found wanting; stopped with the send that carried it: " + g.Why - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("the send to %s in tier %d failed its gate: %s; %s was not found wanting and is left as it is", - strings.Join(g.Machines, ", "), p.Tier, g.Why, module) - } - if state.Gate != nil && len(state.Gate.Carried) > 0 { - failCarried(batched, open, p, state.Gate, module) - } -} - -// rolloutStep is what a plan does next with one built module's machines (novox/hq issue 249). -type rolloutStep struct { - // send is the machines to send now; first, whether they are the first machine's send. - send []string - first bool - // waiting names the first machines whose report the rest wait for. - waiting string - // failed says how a first machine did not take it; rest is what is then left alone. - failed string - rest []string -} - -// nextRollout is the next step of one module's rollout in a plan (novox/hq issue 249, ADR 0218). -// -// Together, every machine running it at once, as the policy says. Otherwise one machine first — the -// first by name among those that have reported within the bound, so a laptop that is away is not -// the one the rest wait on; the first by name when none has; the same choice on every controller and -// every resume — and the rest once each machine the first send reached reports, about the -// declaration it was last sent, that it applied it. Compared by the store's own record of what was -// sent (Reported.Current), never by this controller's clock against the machine's. -// -// **A first machine that fails, refuses, or does not report within the bound stops the rollout -// there** (ADR 0218 §2), naming the machine; the rest are not sent. A wait with no end is not a -// rollout: it held the plan open for ever, read as work in progress (issue 254). -func nextRollout(s inventory.PlanModule, running []string, together bool, reports []inventory.Reported, - now time.Time, bound time.Duration) rolloutStep { - if len(running) == 0 { - return rolloutStep{} - } - if together { - return rolloutStep{send: running} - } - byNode := map[string]inventory.Reported{} - for _, r := range reports { - byNode[r.Node] = r - } - if s.FirstAt == nil { - sorted := append([]string{}, running...) - sort.Strings(sorted) - for _, n := range sorted { - if r, said := byNode[n]; said && r.At != nil && now.Sub(*r.At) <= bound { - return rolloutStep{send: []string{n}, first: true} - } - } - return rolloutStep{send: sorted[:1], first: true} - } - sentFirst := map[string]bool{} - for _, n := range s.First { - sentFirst[n] = true - } - var rest []string - for _, n := range running { - if !sentFirst[n] { - rest = append(rest, n) - } - } - // **A passed gate is the first machine's verdict, and its later reports are not** (novox/hq issue 335). - // Once the build passed there, what that machine reports next is about whatever it was sent after — - // another walk's send, a push — and says nothing of this build. On 2026-10-08 a build passed on the - // laptop, its send to the rest waited on another walk, that walk sent the laptop a new declaration it did - // not report for half an hour, and the plan read the silence as the first machine never applying the - // passed build: it marked it failed at its gate and put it back. The rest are sent, as the pass said. - if s.Gate != nil && s.Gate.Verdict == inventory.GatePassed { - return rolloutStep{send: rest} - } - var waiting, failed []string - for _, n := range s.First { - r, said := byNode[n] - // Only a report about what this plan sent it — or what it was sent after that — says anything about - // this build (novox/hq issue 352); a plan from before sends were kept on the gate reads the report - // against the send made last, as before. - reported := r.Current - if s.Gate != nil && s.Gate.Sent != nil { - sent, kept := s.Gate.Sent[n] - reported = kept && sent.ReportsOn(r) - } - if !said || r.At == nil || !reported { - waiting = append(waiting, n) - continue - } - switch r.Outcome { - case inventory.OutcomeApplied: - case inventory.OutcomeFailed, inventory.OutcomeRefused: - failed = append(failed, n+" "+r.Outcome+" what it was sent") - default: - waiting = append(waiting, n) - } - } - if len(failed) > 0 { - return rolloutStep{failed: strings.Join(failed, "; "), rest: rest} - } - if len(waiting) > 0 { - if now.Sub(*s.FirstAt) > bound { - return rolloutStep{failed: fmt.Sprintf("%s did not report it applied within %s", - strings.Join(waiting, ", "), bound), rest: rest} - } - return rolloutStep{waiting: strings.Join(waiting, ", ")} - } - return rolloutStep{send: rest} -} - -// sayUnsent adds to an ended plan's note the modules it built and never sent (novox/hq issue 249). -// An announced move of a module a plan held was left to that plan; a plan that ends without -// sending it — failed elsewhere, or closed by hand — would leave its machines behind with nothing -// saying so. A module whose rollout stopped at its first machine is not among them: that stop was -// the point. Said once. -func sayUnsent(p *inventory.Plan, rollsOut func(string) bool) { - var unsent []string - for name, s := range p.Modules { - if s != nil && s.State == "built" && s.SentAt == nil && s.FirstAt == nil && rollsOut(name) { - unsent = append(unsent, name) - } - } - if len(unsent) == 0 || strings.Contains(p.Note, "built and never sent") { - return - } - sort.Strings(unsent) - p.Note += "; built and never sent: " + strings.Join(unsent, ", ") + " — `upgrade release-backlog --why ` releases them" -} - -// planLine is one plan as `status` says it, given the least bound of a tier. -func planLine(p inventory.Plan, now time.Time) string { - return planLineWith(p, now, tierAtLeast) -} - -// inTierSince is when a plan began waiting on what it waits on now, which `plans`, `status` and the -// watchdog of a plan's progress (S3) all count from (novox/hq issue 296): a walk waiting for its -// delivery's word from when it was opened, as S16 counts; any other plan from when it entered its tier, -// or, when the delivery let it go after that, from the go. Never from the plan's last save: a plan is -// saved for many reasons while its tier stands still, and a clock reset by each save said a build queued -// for minutes had been building for a few seconds. A plan read without the stamp (one from before it -// was kept) counts from its last save, the only time it has. -func inTierSince(p inventory.Plan) time.Time { - if p.Waiting() { - return p.Created - } - since := p.TierEntered - if since.IsZero() { - since = p.Updated - } - if d := p.Delivery; d != nil && d.Go != nil && d.Go.After(since) { - since = *d.Go - } - return since -} - -// planLineWith is planLine with bound, the plan's tier bound: bound is -// the plan's tier bound, the one its stalled condition is raised at (tierBounds): LATE is that condition -// said on the line (novox/hq issue 296). The machines running what it moves are not named: planLineOn. -func planLineWith(p inventory.Plan, now time.Time, bound time.Duration) string { - return planLineOn(p, now, bound, nil) -} - -// planLineOn is planLineWith naming the plan by what it moves and where (planHeadline), given what runs each -// module; nil names no machine. -func planLineOn(p inventory.Plan, now time.Time, bound time.Duration, running func(string) []string) string { - return planLineSaid(p, now, bound, running, nil) -} - -// planLineSaid is planLineOn saying, of a walk waiting for its delivery's word, what the delivery's owner says -// holds it (novox/hq issue 529); nil waits says it as before. -func planLineSaid(p inventory.Plan, now time.Time, bound time.Duration, running func(string) []string, - waits walkWaits) string { - name := planHeadline(p, running) - where := fmt.Sprintf("tier %d of %d", min(p.Tier+1, len(p.Tiers)), len(p.Tiers)) - switch p.State { - case inventory.PlanAssembling, inventory.PlanQueued: - // A batch not yet a walk (novox/hq ADR 0276): what it holds and how long is left. - return batchWords(p, now) - case inventory.PlanDone: - return fmt.Sprintf("%s · done, %d tier(s)", name, len(p.Tiers)) - case inventory.PlanFailed: - return fmt.Sprintf("%s · FAILED at %s: %s", name, where, p.Note) - case inventory.PlanSuperseded: - return fmt.Sprintf("%s · %s", name, p.Note) - } - since := now.Sub(inTierSince(p)).Round(time.Second) - if p.Waiting() { - // Waiting for its delivery's word is no lateness of the walk's (novox/hq ADR 0239). - return fmt.Sprintf("%s · %s, %s, for %s", name, where, waitingSaid(p, waits[p.ID]), since) - } - if deferred(p) { - // Nor is waiting for the walk before it to end (a related walk, novox/hq ADR 0276, ADR 0407). - return fmt.Sprintf("%s · %s, %s, for %s", name, where, p.Note, since) - } - if waitingOnBackups(p) { - return fmt.Sprintf("%s · %s, %s, for %s", name, where, p.Note, since) - } - late := "" - if since > bound { - late = " — LATE" - } - what := "building" - if p.State == inventory.PlanRolling { - what = p.Note - } - if p.State == inventory.PlanBuilding && refused(p) && currentTierBuilt(p) { - // Every build of the tier is in, so the plan waits on something else, and its note says what: a send - // refused ("a build waits there for its gate: (...)") is kept there. "building" with LATE - // hid that reason from `plans`, `status` and the stalled condition (novox/hq issue 920). - what = p.Note - // And when the builds are held after a failed release, they wait for a person, not for a build, and the - // operator is asked to release them (novox/hq issue 1732): said so, and not late. - if strings.Contains(p.Note, errUngated.Error()) && backlogNow.held != "" { - what += " — held after a failed delivery until a person releases it (the operator is asked)" - late = "" - } - return fmt.Sprintf("%s · %s, held: %s, for %s%s", name, where, what, since, late) - } - return fmt.Sprintf("%s · %s, %s for %s%s", name, where, what, since, late) -} - -// planHeadline names a plan as a person knows it (asked by the operator, 2026-10-10: a plan called by its -// repository alone said nothing of what it delivers): each repository as its pull request and title, what the -// plan moves, and the machines running that — "mesh-catalog #175 a tap shows its outcome · messenger, -// telegram → novox". A record naming no pull request is named by its commit, as before; one naming no moves -// says none; running nil names no machine. -func planHeadline(p inventory.Plan, running func(string) []string) string { - var repos []string - moves := map[string]bool{} - for _, c := range p.Carried() { - seg := repoName(c.Repository) + " " + short(c.Commit) - if p.Delivery != nil { - for _, m := range p.Delivery.Merges { - if !strings.EqualFold(m.Repository, c.Repository) { - continue - } - for _, n := range m.Moves { - moves[n] = true - } - if m.Commit == c.Commit && m.Number > 0 { - seg = repoName(c.Repository) + " " + pullWords(m) - } - } - } - repos = append(repos, seg) - } - out := strings.Join(repos, " + ") - if len(moves) == 0 { - return out - } - names := sortedKeysOf(boolsToStrings(moves)) - out += " · " + strings.Join(names, ", ") - if running == nil { - return out - } - nodes := map[string]bool{} - for _, n := range names { - for _, node := range running(n) { - nodes[node] = true - } - } - if len(nodes) > 0 { - out += " → " + strings.Join(sortedKeysOf(boolsToStrings(nodes)), ", ") - } - return out -} - -func repositoryMatches(a, b string) bool { - trim := func(s string) string { return strings.ToLower(strings.TrimSuffix(s, ".git")) } - return trim(a) == trim(b) || strings.HasSuffix(trim(a), "/"+trim(b)) || strings.HasSuffix(trim(b), "/"+trim(a)) -} - -// planStatus is one plan as `status --json` says it. -type planStatus struct { - ID string `json:"id"` - Repository string `json:"repository"` - Commit string `json:"commit"` - State string `json:"state"` - Tier int `json:"tier"` - Tiers int `json:"tiers"` - Waiting string `json:"waiting,omitempty"` - Since time.Time `json:"since"` - Late bool `json:"late"` -} - -func planStatuses(plans []inventory.Plan, now time.Time, bounds tierBounds) []planStatus { - out := make([]planStatus, 0, len(plans)) - for _, p := range plans { - ps := planStatus{ID: p.ID, Repository: p.Repository, Commit: p.Commit, State: p.State, - Tier: p.Tier, Tiers: len(p.Tiers), Since: p.Updated} - if p.Open() { - ps.Since = inTierSince(p) - ps.Waiting = p.Note - if ps.Waiting == "" { - ps.Waiting = "builds of tier " + fmt.Sprint(p.Tier) - } - // A walk waiting for its delivery's word is no tier late (ADR 0239): S16 is its watchdog. - ps.Late = !p.Waiting() && now.Sub(ps.Since) > bounds.of(p.Repository) - } - out = append(out, ps) - } - return out -} - -// openPlans is the open plans among the recent ones, and how many have waited past the bound. -func openPlans(plans []inventory.Plan, now time.Time, bounds tierBounds) ([]inventory.Plan, int) { - var open []inventory.Plan - late := 0 - for _, p := range plans { - if p.Open() { - open = append(open, p) - if p.Waiting() { - continue - } - if now.Sub(inTierSince(p)) > bounds.of(p.Repository) { - late++ - } - } - } - return open, late -} - -// plansCommand says what the last merges produced and where each stands; given an id, one plan -// tier by tier with every module's state. -func plansCommand(ctx context.Context, args []string) error { - set := flag.NewFlagSet("plans", flag.ContinueOnError) - limit := set.Int("n", 10, "how many to show") - whatIf := set.String("what-if", "", "owner/repository: the plan a merge there would produce, saving nothing — with --paths or --modules") - paths := set.String("paths", "", "the files the merge would change, comma-separated, from the repository's root") - modules := set.String("modules", "", "or the modules it would change, comma-separated") - moduleDirs := set.String("module-dirs", "", "with --paths: the directories holding a module.json at the commit, "+ - "comma-separated, as the forge's announcer says them (novox/hq issue 278)") - // The what-if as data, the document `delivery plan` answers (novox/hq ADR 0450 §2, issue 1324). - data := set.Bool("data", false, "with --what-if: answer the change plan as data, the document `delivery plan` answers; "+ - "without it the answer is the text a person reads") - head := set.String("head", "", "with --data: the commit at hand") - base := set.String("base", "main", "with --data: the branch it merges into") - removed := set.String("removed", "", "with --data: the files among --paths it deletes, comma-separated") - // Following a plan (novox/hq issue 229): `plans wait `. - until := set.String("until", "", "with wait: what to wait for — moved (any change), tier (its next tier) or done (its end)") - bound := set.String("bound", "", "with wait: how long to wait at most, as 15s or seconds (default and at most the "+ - "longest a call is answered within)") - // Ending a plan by hand is a repair, and says why (novox/hq to-be 45 §7). - why := addHandActFlags(set) - positionals, err := parseAround(set, args) - if err != nil { - return err - } - if len(positionals) == 2 && (positionals[0] == "stop" || positionals[0] == "close" || positionals[0] == "go") { - // Refused before anything is opened: a repair by hand says why. - if err := why.require("plans " + positionals[0]); err != nil { - return err - } - } - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - inv := open.inventory - now := time.Now() - if len(positionals) == 2 && positionals[0] == "wait" { - return planWaitCommand(ctx, inv, positionals[1], *until, *bound) - } - if len(positionals) == 1 { - p, err := inv.PlanByID(ctx, positionals[0]) - if err != nil { - return err - } - bounds := readTierBounds(ctx, inv, now) - fmt.Printf("%s — %s\n", p.ID, planLineSaid(p, now, - bounds.of(p.Repository), func(module string) []string { - on, _ := inv.Running(ctx, module) - return on - }, waitsOf(ctx, inv, []inventory.Plan{p}))+heldSaid(p, heldNow(ctx, inv))) - if r := p.Release; r != nil { - // A release plan's walk (ADR 0236): machines done, the one judged, those to come. - fmt.Printf(" machines in order: %s; done: %s; skipped: %s\n", strings.Join(r.Order, ", "), - orNone(strings.Join(r.Done, ", ")), orNone(strings.Join(r.Skipped, ", "))) - if len(r.Failed)+len(r.Silent) > 0 { - fmt.Printf(" failed: %s; silent: %s\n", orNone(strings.Join(r.Failed, ", ")), orNone(strings.Join(r.Silent, ", "))) - } - gates := []*inventory.PlanGate{r.Gate} - for _, n := range r.Order { - gates = append(gates, r.Gates[n]) - } - for _, g := range gates { - if g == nil { - continue - } - fmt.Printf(" %s\n", gateLine(g)) - for _, c := range g.Carried { - fmt.Printf(" %-22s %s → %s\n", c.Module, short(c.From), short(c.To)) - if c.Recreates != "" { - fmt.Printf(" %-22s %s\n", "", c.Recreates) - } - } - } - return nil - } - for i, tier := range p.Tiers { - marker := " " - if i == p.Tier && p.Open() { - marker = ">" - } - fmt.Printf("%s tier %d\n", marker, i) - for _, m := range tier { - s := p.Modules[m] - state := "not yet asked" - if s != nil && s.State != "" { - state = s.State - if s.Commit != "" { - state += " from " + short(s.Commit) - } - if s.Build != "" && s.State != "built" { - state += " (" + s.Build + ")" - } - if s.Why != "" { - state += ": " + s.Why - } - } - fmt.Printf(" %-22s %s\n", m, state) - // The rollout's record at its gate (novox/hq to-be 45 §8): first machine, from and to, - // verdict, time to it, rolled back or not. - switch { - case s != nil && s.GatedBy != "" && (s.Gate == nil || s.Gate.Verdict == ""): - // Sent with others, judged by the gate of that send (issue 281). - fmt.Printf(" %-22s judged with %s, in the send to %s\n", "", s.GatedBy, strings.Join(s.First, ", ")) - case s != nil && s.Gate != nil: - fmt.Printf(" %-22s %s\n", "", gateLine(s.Gate)) - } - // Which way a recorded module went on each machine (novox/hq ADR 0377 §6). - if s != nil { - for _, n := range sortedKeysOf(s.Recorded) { - fmt.Printf(" %-22s on %s %s\n", "", n, s.Recorded[n]) - } - } - // What the send to its first machine did to its containers (ADR 0245). - if s != nil && s.Gate != nil { - for _, c := range s.Gate.Carried { - if c.Recreates != "" { - fmt.Printf(" %-22s on %s %s\n", "", c.Node, c.Recreates) - } - } - } - } - } - // The assignments it changed, each with the build and commit before and after (novox/hq ADR 0408, issue 1095). - for _, line := range changedLines(p) { - fmt.Println(line) - } - return nil - } - if *data && *whatIf == "" { - return errors.New("--data answers a what-if: give --what-if owner/repository") - } - if *whatIf != "" && *data { - if *paths == "" { - return errors.New("plans --what-if --data: the change plan is of the files a change touches; give --paths") - } - entries, read, edges, err := theGraph(ctx, inv) - if err != nil { - return err - } - m := changeMember(*whatIf, *base, *head, splitList(*paths), splitList(*removed), moduleDirsOf(*moduleDirs)) - enc := json.NewEncoder(os.Stdout) - enc.SetIndent("", " ") - return enc.Encode(deliveryPlanDocument(m, entries, read, edges, func(module string) (inventory.Upgrade, bool) { - u, err := inv.UpgradeOf(ctx, module) - return u, err == nil - })) - } - if *whatIf != "" { - return planWhatIf(ctx, inv, *whatIf, splitList(*paths), splitList(*modules), moduleDirsOf(*moduleDirs)) - } - // `go` (novox/hq ADR 0239): a person's word in place of the delivery's owner, for a walk that waits for it - // — mesh-delivery down, or a person who will not wait. Recorded in the hand-act log with why. - if len(positionals) == 2 && positionals[0] == "go" { - why.record(ctx, "plans go", positionals[1:]) - var p inventory.Plan - if err := sayingOnTheBus(ctx, inv, func() (err error) { - p, err = letGo(ctx, inv, positionals[1], "a person ("+whoAsked()+")", strings.TrimSpace(*why.why)) - return err - }); err != nil { - return err - } - fmt.Printf("%s (%s at %s) is let go by hand; its first tier is asked when the controller next looks at the plans, which it does as this ends\n", p.ID, p.Repository, - short(p.Commit)) - return nil - } - // `retry` (novox/hq ADR 0219): a failed plan's failed builds asked again, and the plan goes on. - if len(positionals) == 2 && positionals[0] == "retry" { - // Said on the bus (novox/hq ADR 0308 decision 1): a retried walk starts again, and its start is said - // by the verb that starts it. - var said string - if err := sayingOnTheBus(ctx, inv, func() (err error) { - said, err = retryPlan(ctx, open, positionals[1]) - return err - }); err != nil { - return err - } - fmt.Println(said) - return nil - } - // `stop`, or `close` (novox/hq issue 254): a person ending a plan that will not move again — one - // waiting on a report that cannot come — so it stops reading as work in progress. Marked failed - // with who ended it; what it asked still builds and registers. - if len(positionals) == 2 && (positionals[0] == "stop" || positionals[0] == "close") { - how := "stopped" - if positionals[0] == "close" { - how = "closed" - } - // Said on the bus (novox/hq ADR 0308 decision 1): the walk's end, outcome stopped, is said by the verb - // that ends it, so what it touched is not held until its bound. - var p inventory.Plan - if err := sayingOnTheBus(ctx, inv, func() error { - release, err := inv.HoldPlans(ctx, true) - if err != nil { - return err - } - defer release() - if p, err = inv.PlanByID(ctx, positionals[1]); err != nil { - return err - } - if !p.Open() { - return fmt.Errorf("%s is already %s", p.ID, p.State) - } - why.record(ctx, "plans "+positionals[0], positionals[1:]) - stopByHand(&p, how, "a person ("+whoAsked()+")", strings.TrimSpace(*why.why)) - sayUnsent(&p, func(m string) bool { - u, err := inv.UpgradeOf(ctx, m) - return err == nil && u.RollOut - }) - return inv.SavePlan(ctx, &p) - }); err != nil { - return err - } - fmt.Printf("%s %s at tier %d of %d; what was asked still builds and registers, nothing further is asked\n", - p.ID, how, p.Tier, len(p.Tiers)) - return nil - } - plans, err := inv.RecentPlans(ctx, *limit) - if err != nil { - return err - } - // **The batch being assembled first** (novox/hq ADR 0276 decision 7): what it holds, how long is left, - // and that its plan is not yet calculated. - batches, err := inv.Batches(ctx) - if err != nil { - return err - } - if len(plans) == 0 && len(batches) == 0 { - fmt.Println("no merge has produced a plan yet") - return nil - } - for _, b := range batches { - fmt.Printf("%-28s %s\n", b.ID, batchWords(b, now)) - // One line per repository: its pull request, what it answers, what it moves. - for _, line := range batchLines(b) { - fmt.Printf("%-28s %s\n", "", line) - } - } - bounds := readTierBounds(ctx, inv, now) - running := func(module string) []string { - on, _ := inv.Running(ctx, module) - return on - } - waits := waitsOf(ctx, inv, plans) - held := heldNow(ctx, inv) - for _, p := range plans { - if p.Batch() { - continue - } - fmt.Printf("%-28s %s%s\n", p.ID, planLineSaid(p, now, bounds.of(p.Repository), running, waits), heldSaid(p, held)) - } - return nil -} - -// waitsOf asks the delivery's owner what holds the walks among these that wait for its word (novox/hq issue 529), -// once, and only when one waits; nil when none waits, no holder is on record, or the bus or the owner does not -// answer within a few seconds — the lines then read as before. -func waitsOf(ctx context.Context, inv *inventory.Inventory, plans []inventory.Plan) walkWaits { - waiting := false - for _, p := range plans { - waiting = waiting || p.Open() && p.Waiting() - } - if !waiting { - return nil - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return nil - } - ctx, cancel := context.WithTimeout(ctx, 3*time.Second) - defer cancel() - var waits walkWaits - _ = onTheBus(func(conn *nats.Conn) error { - waits = walksWaiting(ctx, conn, deliverySeatHeld(entries)) - return nil - }) - return waits -} - -// planWhatIf is the plan a merge would produce, computed the way the merge handler computes one -// and saved nowhere: the modules the repository's changed files touch (or the modules named), what -// packages their source, everything reachable from them, in tiers. For reading before merging. -func planWhatIf(ctx context.Context, inv *inventory.Inventory, repository string, paths, modules []string, moduleDirs *[]string) error { - owner, repo, found := strings.Cut(repository, "/") - if !found { - return fmt.Errorf("--what-if takes owner/repository, not %q", repository) - } - m := link.SourceMoved{Owner: owner, Repo: repo, Base: "main", Commit: "what-if", Paths: paths} - if moduleDirs != nil { - m.ModuleDirs, m.ModuleDirsSaid = *moduleDirs, true - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return err - } - read, err := readForPlanning(ctx, inv) - if err != nil { - return err - } - var from, packaging []inventory.Entry - deleted := map[string]bool{} - named := map[string]bool{} - for _, name := range modules { - named[name] = true - } - if len(named) == 0 { - // The changed files: the planner's own answer, as the merge handler, the merge gate and a pull - // request's check ask it (novox/hq ADR 0238). - r := reachOfMerge(m, entries, read, nil) - from, packaging = append(append([]inventory.Entry{}, r.Touched...), r.Deleted...), r.Packaging - for _, e := range r.Deleted { - deleted[e.Manifest.Module] = true - } - } else { - for _, e := range entries { - switch { - case named[e.Manifest.Module]: - from = append(from, e) - case readsFrom(read[e.Manifest.Module], m): - packaging = append(packaging, e) - } - } - } - moved := append(append([]inventory.Entry{}, from...), packaging...) - if len(moved) == 0 { - fmt.Printf("a merge of %s changing %s would build nothing the mesh holds\n", repository, - orNone(strings.Join(append(paths, modules...), ", "))) - return nil - } - edges, err := inv.Dependencies(ctx) - if err != nil { - return err - } - var names []string - for _, e := range moved { - names = append(names, e.Manifest.Module) - } - p := planOfMerge(m, names, edges) - fmt.Printf("a merge of %s would build %d module(s) in %d tier(s):\n", repository, len(p.Modules), len(p.Tiers)) - why := map[string]string{} - for _, e := range packaging { - why[e.Manifest.Module] = whyMoved(e, read[e.Manifest.Module], m) - } - if len(named) == 0 { - for _, e := range from { - why[e.Manifest.Module] = whyMoved(e, read[e.Manifest.Module], m) - if deleted[e.Manifest.Module] { - why[e.Manifest.Module] = "its manifest is removed: deleted at its source, forgotten where nothing holds it, not built" - } - } - } - rolls := map[string]string{} - for i, tier := range p.Tiers { - fmt.Printf(" tier %d\n", i) - for _, name := range tier { - how := "built; its policy records, so nothing is sent" - if u, err := inv.UpgradeOf(ctx, name); err == nil && u.RollOut { - running, _ := inv.Running(ctx, name) - reports, _ := inv.LastReports(ctx) - how = "built, then sent to " + orNone(strings.Join(running, ", ")) - // One machine first unless the policy says together (novox/hq issue 249). - if first := nextRollout(inventory.PlanModule{}, running, u.Together, reports, time.Now(), planWaitBound); first.first && len(running) > 1 { - how = fmt.Sprintf("built, then sent to %s first and to the rest once it has applied it", - first.send[0]) - } - rolls[name] = how - } - fmt.Printf(" %-22s %s\n", name, how) - reason := why[name] - switch { - case reason == "" && len(named) > 0 && named[name]: - reason = "named" - case reason == "": - reason = "it stands on a module the merge moves" - } - fmt.Printf(" %-22s why: %s\n", "", reason) - } - } - if hasCycle(p.Tiers, edges) { - fmt.Println(" the last tier depends on itself and would be built together, in no order") - } - return nil -} - -// moduleDirsOf is --module-dirs as an announcer would say it: nothing when not given, so the what-if -// reads as an announcement from an announcer that does not say. -func moduleDirsOf(s string) *[]string { - if strings.TrimSpace(s) == "" { - return nil - } - dirs := splitList(s) - return &dirs -} - -func splitList(s string) []string { - var out []string - for _, part := range strings.Split(s, ",") { - if part = strings.TrimSpace(part); part != "" { - out = append(out, part) - } - } - return out -} - -// recordsOfAsked is what settleFromRecords reads: for every module of the tier still `asked`, its last -// builds and the record of its own ask, by id. -func recordsOfAsked(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan, tier []string) ( - map[string][]inventory.Build, map[string]inventory.Build, error) { - recorded := map[string][]inventory.Build{} - byID := map[string]inventory.Build{} - for _, m := range tier { - if s := p.Modules[m]; s != nil && s.State == "asked" { - builds, err := inv.Builds(ctx, m, 5) - if err != nil { - return nil, nil, err - } - recorded[m] = builds - // Its own ask's record, by id — found even when the outcome named no module (ADR 0219). - if s.Build != "" { - b, found, err := inv.BuildByID(ctx, s.Build) - if err != nil { - return nil, nil, err - } - if found { - byID[s.Build] = b - } - } - } - } - refused, err := refusalsOf(ctx, inv, recorded, byID) - if err != nil { - return nil, nil, err - } - overlayRefusals(recorded, byID, refused) - return recorded, byID, nil -} - -// overlayRefusals marks failed every build record whose registration the mesh refused (novox/hq issue 1708): the record -// keeps the builder's words, which worked, and settled from it as it is the refused build would be built and sent. -func overlayRefusals(recorded map[string][]inventory.Build, byID map[string]inventory.Build, refused map[string]string) { - if len(refused) == 0 { - return - } - fail := func(b *inventory.Build) { - if why, ok := refused[b.ID]; ok && b.Worked() { - b.Failed = why - } - } - for m := range recorded { - for i := range recorded[m] { - fail(&recorded[m][i]) - } - } - for id, b := range byID { - fail(&b) - byID[id] = b - } -} - -// refusalsOf is every refusal kept for the builds these records name, in the store and in this process. -func refusalsOf(ctx context.Context, inv *inventory.Inventory, recorded map[string][]inventory.Build, - byID map[string]inventory.Build) (map[string]string, error) { - var ids []string - for _, bs := range recorded { - for _, b := range bs { - ids = append(ids, b.ID) - } - } - for id := range byID { - ids = append(ids, id) - } - refused, err := inv.RefusedOf(ctx, ids) - if err != nil { - return nil, fmt.Errorf("the refused registrations cannot be read: %w", err) - } - for id, why := range refusalsKeptFor(ids) { - refused[id] = why - } - return refused, nil -} - -// settledEffects is what settling from the records did that needs more than the plan: the modules it found deleted -// at their source, and whether it failed the plan. -func settledEffects(before map[string]string, p *inventory.Plan, was string) (deleted []string, failed bool) { - for m, s := range p.Modules { - if s != nil && s.State == planDeleted && before[m] != planDeleted { - deleted = append(deleted, m) - } - } - sort.Strings(deleted) - return deleted, p.State == inventory.PlanFailed && was != inventory.PlanFailed -} - -// statesOf is each module's state in a plan, for settledEffects. -func statesOf(p *inventory.Plan) map[string]string { - out := map[string]string{} - for m, s := range p.Modules { - if s != nil { - out[m] = s.State - } - } - return out -} - -// settleFromRecords marks every module of the tier still `asked` built — or failed — from a build -// recorded after it was asked, and says whether it changed anything (novox/hq 04-ISSUES/214). -// Newest first, as Builds answers: the first record after the ask is the outcome of that ask. -// -// The record of the plan's own ask, by its id, is that outcome before anything else (novox/hq ADR -// 0219): the plan asked under it, and a failure recorded without a module — cancelled, killed — is -// found by nothing else. -func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]inventory.Build, - byID map[string]inventory.Build) bool { - changed := false - for _, m := range tier { - s := p.Modules[m] - if s == nil || s.State != "asked" || s.AskedAt == nil || s.Key { - continue - } - var outcome *inventory.Build - for i := range recorded[m] { - // Asked under an id, only that id's record answers (ADR 0219), and it is looked up below. - if s.Build != "" { - break - } - b := recorded[m][i] - if b.At.Before(*s.AskedAt) { - break - } - // Recorded after the ask and asked before it: an earlier ask's late outcome, not this - // one's (novox/hq 04-ISSUES/219). - if askedBefore(b.Asked, s.AskedAt) { - continue - } - outcome = &b - } - if own, found := byID[s.Build]; s.Build != "" && found { - outcome = &own - } - if outcome == nil { - continue - } - at := outcome.At - if !outcome.Worked() && deletedAtSource(outcome.Failed) { - s.State, s.Why = planDeleted, "deleted at its source: "+firstLine(outcome.Failed) - fmt.Printf("%s: %s was deleted at its source (%s): not built, and the plan goes on\n", p.ID, m, outcome.ID) - changed = true - continue - } - if outcome.Worked() { - s.State = "built" - s.BuiltAt = &at - s.Commit = outcome.Commit - // What the build did, when its outcome was left to the records with words (issue 1708). - if made := takeMadeOf(outcome.ID); made != "" { - s.Made = made - } - } else { - s.State = "failed" - s.Why = outcome.Failed - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("%s failed to build in tier %d", m, p.Tier) - } - fmt.Printf("%s: %s settled from the build records as %s (%s)\n", p.ID, m, s.State, outcome.ID) - changed = true - } - return changed -} - -// askedBefore is whether a build asked at asked was asked before a plan asked for its module — and -// so is not that plan's outcome (novox/hq 04-ISSUES/219). False when either time is not known. -func askedBefore(asked time.Time, planAsked *time.Time) bool { - return !asked.IsZero() && planAsked != nil && asked.Before(*planAsked) -} - -// firstRunning is the machines sent first that run the module — not the machine holding the bus, sent -// with them only for its user list. -func firstRunning(first, running []string) []string { - var out []string - for _, n := range first { - if slices.Contains(running, n) { - out = append(out, n) - } - } - if len(out) == 0 { - return first - } - return out -} - -// planDeleted is a plan's module that the merge deleted at its source: not built, not sent, and no -// failure of the plan (novox/hq ADR 0236). -const planDeleted = "deleted" - -// forgetDeleted forgets a module a plan found deleted at its source, where nothing holds it, and says -// it otherwise. -func forgetDeleted(ctx context.Context, inv *inventory.Inventory, module string, p *inventory.Plan) { - switch err := inv.ForgetModule(ctx, module); { - case err == nil: - fmt.Printf("%s: %s was deleted at its source: forgotten, and the plan goes on\n", p.ID, module) - default: - fmt.Printf("%s: %s was deleted at its source and is not built; it is kept until a person unassigns it and "+ - "`module forget %s`: %v\n", p.ID, module, module, firstLine(err.Error())) - } -} - -// stopByHand ends a walk a person stopped or closed (`plans stop|close`): failed, with who stopped it and why -// kept as the delivery's owner keeps its own stop, so the walk ends with outcome stopped, not failed (novox/hq -// ADR 0308 decision 1). -func stopByHand(p *inventory.Plan, how, by, why string) { - p.State = inventory.PlanFailed - p.Note = how + " by hand at tier " + fmt.Sprint(p.Tier) + ": " + why - if p.Delivery == nil { - p.Delivery = &inventory.PlanDelivery{} - } - p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why -} - -// keyed says a module of the tier still waits on the builder's key. -func keyed(p *inventory.Plan, tier []string) bool { - for _, m := range tier { - if s := p.Modules[m]; s != nil && s.Key && s.State == "asked" { - return true - } - } - return false -} diff --git a/cmd/mesh-controller/release_plan_test.go b/cmd/mesh-controller/release_plan_test.go deleted file mode 100644 index 5425de3c..00000000 --- a/cmd/mesh-controller/release_plan_test.go +++ /dev/null @@ -1,264 +0,0 @@ -package main - -import ( - "slices" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A merge produces a tiered plan (novox/hq ADR 0162): what moved and everything reachable from it, -// sorted so a tier depends only on earlier ones — with the three kinds of dependency told apart. -func TestAMergeIsPlannedInTiersAlongTheThreeKindsOfDependency(t *testing.T) { - edges := []inventory.Edge{ - // build dependencies: images on the runtime, a plugin on one of them - {From: "shop", To: "mesh-tools", Kind: inventory.EdgeStandsOn}, - {From: "postgres", To: "mesh-tools", Kind: inventory.EdgeStandsOn}, - {From: "shop-plugin", To: "shop", Kind: inventory.EdgeDeclared}, - // a code dependency: the proxy packages the controller's source — same tier - {From: "route-proxy", To: "mesh-controller", Kind: inventory.EdgePackages}, - {From: "builder", To: "mesh-controller", Kind: inventory.EdgePackages}, - // runtime dependencies: everything source-built is built by the builder - {From: "shop", To: "builder", Kind: inventory.EdgeBuiltBy}, - {From: "postgres", To: "builder", Kind: inventory.EdgeBuiltBy}, - {From: "shop-plugin", To: "builder", Kind: inventory.EdgeBuiltBy}, - {From: "route-proxy", To: "builder", Kind: inventory.EdgeBuiltBy}, - {From: "mesh-controller", To: "builder", Kind: inventory.EdgeBuiltBy}, - {From: "mesh-tools", To: "builder", Kind: inventory.EdgeBuiltBy}, - {From: "builder", To: "mesh-tools", Kind: inventory.EdgeStandsOn}, - // the build seat's holder follows the controller that defines its worker (hq issue 206) - {From: "builder", To: "mesh-controller", Kind: inventory.EdgeWorkerOf}, - {From: "unrelated", To: "alpine", Kind: inventory.EdgeStandsOn}, - } - // The runtime image moved: everything on it, and what is built by what is on it. - set := reachableFrom([]string{"mesh-tools"}, edges) - // What stands on the runtime, and the builder that stands on it; not the controller, which the - // builder merely builds, nor the proxy that packages the controller. - want := []string{"builder", "mesh-tools", "postgres", "shop", "shop-plugin"} - if len(set) != len(want) { - t.Fatalf("reachable from the runtime: %v, want %v", set, want) - } - tiers := tiersOf(set, edges) - pos := map[string]int{} - for i, tier := range tiers { - for _, m := range tier { - pos[m] = i - } - } - if pos["mesh-tools"] != 0 || pos["builder"] != 1 { - t.Fatalf("the runtime then the builder: %v", tiers) - } - // Built-by orders nothing (novox/hq ADR 0360 §3): what the builder builds is beside it, after the runtime it - // stands on. - if pos["shop"] != 1 || pos["postgres"] != 1 { - t.Fatalf("what the builder builds waits for the builder: %v", tiers) - } - if pos["shop-plugin"] <= pos["shop"] { - t.Fatalf("a plugin after what it is declared on: %v", tiers) - } - if hasCycle(tiers, edges) { - t.Fatalf("no cycle here: %v", tiers) - } - - // The controller alone moved: the controller alone — what packages its repository moves only when the - // change is in its own build source (novox/hq ADR 0267), so a packages edge widens nothing. - if alone := reachableFrom([]string{"mesh-controller"}, edges); len(alone) != 1 { - t.Fatalf("a controller merge rebuilds the controller alone: %v", alone) - } - // A change to a package all three build from moves all three. The builder holds - // the build seat, whose worker the controller defines, so it follows the controller (worker-of, - // novox/hq issue 206), and the controller's built-by edge to it yields: the controller is built by the - // build machine that is running. The proxy is built by whichever builder runs, so it goes beside the - // controller (novox/hq ADR 0360 §3; it read "the controller, the builder, the proxy" before). - small := reachableFrom([]string{"mesh-controller", "builder", "route-proxy"}, edges) - smallTiers := tiersOf(small, edges) - if got := tiered(smallTiers); got != "mesh-controller,route-proxy | builder" { - t.Fatalf("the controller and the proxy, then the builder: %v", smallTiers) - } - // The builder alone moved: the builder, and nothing it builds. - if only := reachableFrom([]string{"builder"}, edges); len(only) != 1 { - t.Fatalf("a build machine change rebuilds the build machine alone: %v", only) - } - - // Nothing waits on deployment for a built-by edge (novox/hq ADR 0360 §3): the builder gates no tier after it. - p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"}, []string{"mesh-tools"}, edges) - p.Tier = pos["builder"] - rollsOut := func(m string) bool { return m == "builder" } - if g := gates(p, edges, rollsOut); len(g) != 0 { - t.Fatalf("the builder gates the tier after it: %v", g) - } - p.Tier = 0 - if g := gates(p, edges, rollsOut); len(g) != 0 { - t.Fatalf("the runtime image is a build dependency and gates nothing: %v", g) - } - if g := gates(p, edges, func(string) bool { return false }); len(g) != 0 { - t.Fatalf("a module that only records its upgrade gates nothing: %v", g) - } -} - -// A gate is open once every machine running the module has reported after it was built. -func TestAGateOpensWhenTheMachinesHaveReportedSinceTheBuild(t *testing.T) { - built := time.Date(2026, 10, 1, 15, 0, 0, 0, time.UTC) - before, after := built.Add(-time.Minute), built.Add(time.Minute) - reports := []inventory.Reported{{Node: "anchor", At: &after}, {Node: "home-server", At: &before}} - ok, waiting := applied("builder", built, []string{"anchor", "home-server"}, reports) - if ok || len(waiting) != 1 || waiting[0] != "home-server" { - t.Fatalf("one machine has not reported since the build: ok=%v waiting=%v", ok, waiting) - } - if ok, _ := applied("builder", built, []string{"anchor"}, reports); !ok { - t.Fatal("the machine that reported after the build holds the gate open") - } - if ok, _ := applied("builder", built, nil, reports); !ok { - t.Fatal("a module running nowhere gates nothing") - } -} - -// A cycle is not lost: what remains is one last tier, and the caller says so. -func TestACycleIsOneLastTierAndSaidSo(t *testing.T) { - edges := []inventory.Edge{{From: "a", To: "b", Kind: inventory.EdgeStandsOn}, {From: "b", To: "a", Kind: inventory.EdgeStandsOn}} - tiers := tiersOf([]string{"a", "b"}, edges) - if len(tiers) != 1 || len(tiers[0]) != 2 || !hasCycle(tiers, edges) { - t.Fatalf("a cycle should be one tier of two, said: %v", tiers) - } -} - -// novox/hq 04-ISSUES/211: a merge moving the toolchain and a bundle compiled in it builds the -// bundle a tier after the toolchain, not beside it. -func TestABundleIsPlannedAfterTheToolchainItIsCompiledIn(t *testing.T) { - edges := []inventory.Edge{{From: "node-tools", To: "mesh-tools", Kind: inventory.EdgeStandsOn}} - p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-tools", Commit: "abc"}, - []string{"mesh-tools", "node-tools"}, edges) - if len(p.Tiers) != 2 || p.Tiers[0][0] != "mesh-tools" || p.Tiers[1][0] != "node-tools" { - t.Fatalf("the toolchain, then the bundle: %v", p.Tiers) - } -} - -// novox/hq 04-ISSUES/214: a plan whose build outcome was recorded while no controller followed it — -// the controller rebuilding itself — settles from the build records instead of waiting for ever. -func TestAPlanSettlesAnAskedBuildFromTheRecords(t *testing.T) { - asked := time.Date(2026, 10, 3, 19, 20, 0, 0, time.UTC) - p := inventory.Plan{ID: "plan-1", Tiers: [][]string{{"mesh-controller", "builder"}, {"route-proxy"}}, - Modules: map[string]*inventory.PlanModule{ - "mesh-controller": {State: "asked", AskedAt: &asked}, - "builder": {State: "asked", AskedAt: &asked}, - }} - records := map[string][]inventory.Build{ - // Newest first, as Builds answers: the build after the ask is the outcome. - "mesh-controller": { - {ID: "build-2", Commit: "2ebbb799", At: asked.Add(4 * time.Minute)}, - {ID: "build-1", Commit: "06ea2168", At: asked.Add(-10 * time.Minute)}, - }, - // Only a build from before the ask: not this ask's outcome. - "builder": {{ID: "build-0", Commit: "06ea2168", At: asked.Add(-time.Hour)}}, - } - if !settleFromRecords(&p, p.Tiers[0], records, nil) { - t.Fatal("nothing settled, though the controller's build is recorded after the ask") - } - if s := p.Modules["mesh-controller"]; s.State != "built" || s.Commit != "2ebbb799" || s.BuiltAt == nil { - t.Errorf("the controller's ask is %+v, want built from 2ebbb799", s) - } - if s := p.Modules["builder"]; s.State != "asked" { - t.Errorf("an ask with no record after it was settled: %+v", s) - } - - // novox/hq 04-ISSUES/219: a build recorded after the ask but asked before it — an earlier - // plan's late outcome — is not this ask's, built or failed. - r := inventory.Plan{ID: "plan-3", Tiers: [][]string{{"postgres"}}, - Modules: map[string]*inventory.PlanModule{"postgres": {State: "asked", AskedAt: &asked}}} - late := map[string][]inventory.Build{"postgres": { - {ID: "build-old", Commit: "efff5415", Asked: asked.Add(-18 * time.Minute), At: asked.Add(12 * time.Minute)}, - }} - if settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "asked" { - t.Errorf("an earlier ask's late outcome settled this ask: %+v", r.Modules["postgres"]) - } - // Newest heard first: the earlier ask's late outcome, then this ask's own, heard before it. - late["postgres"] = append(late["postgres"], inventory.Build{ID: "build-mine", Commit: "4bcd5f73", - Asked: asked.Add(time.Second), At: asked.Add(5 * time.Minute)}) - if !settleFromRecords(&r, r.Tiers[0], late, nil) || r.Modules["postgres"].State != "built" || - r.Modules["postgres"].Commit != "4bcd5f73" { - t.Errorf("this ask's own outcome, heard before the earlier ask's, did not settle it: %+v", r.Modules["postgres"]) - } - - // A failure recorded after the ask fails the plan, as hearing it would have. - q := inventory.Plan{ID: "plan-2", Tiers: [][]string{{"x"}}, - Modules: map[string]*inventory.PlanModule{"x": {State: "asked", AskedAt: &asked}}} - settleFromRecords(&q, q.Tiers[0], map[string][]inventory.Build{"x": {{ID: "b", Failed: "no", At: asked.Add(time.Minute)}}}, nil) - if q.State != inventory.PlanFailed || q.Modules["x"].State != "failed" { - t.Errorf("a recorded failure did not fail the plan: %+v %+v", q, q.Modules["x"]) - } -} - -// The first machine's report, made between the send to it and the send to the rest, opens the gate for it: -// each machine is judged from its own send, not from the last one (novox/hq issue 256). -func TestTheGateJudgesEachMachineFromItsOwnSend(t *testing.T) { - built := time.Date(2026, 10, 5, 18, 22, 0, 0, time.UTC) - firstSent := built.Add(31 * time.Second) - firstReported := built.Add(43 * time.Second) - restSent := built.Add(58 * time.Second) - restReported := built.Add(74 * time.Second) - reports := []inventory.Reported{ - {Node: "ace", At: &firstReported}, - {Node: "g14", At: &restReported}, - } - since := func(node string) time.Time { - if node == "ace" { - return firstSent - } - return restSent - } - if ok, waiting := appliedEach("build-agent", since, []string{"ace", "g14"}, reports); !ok { - t.Fatalf("the gate still waits on %v, though each reported after its own send", waiting) - } - // The old reading, every machine from the last send, is what held the plan. - if ok, _ := applied("build-agent", restSent, []string{"ace", "g14"}, reports); ok { - t.Fatal("the single-moment reading should hold the first machine back") - } - early := built.Add(10 * time.Second) - if ok, waiting := appliedEach("build-agent", since, []string{"ace"}, []inventory.Reported{{Node: "ace", At: &early}}); ok || waiting[0] != "ace" { - t.Fatal("a report from before the machine was sent opened the gate") - } -} - -// novox/hq issue 278 and ADR 0236's open question: a change to the build agent — its manifest alone, as -// the catalogue's data sections were, or its program — rebuilds the build agent and nothing it builds. -// What it builds is ordered after it in a plan that holds both, never added to one for its sake. The -// merge that rebuilt 103 modules with the agent in tier 0 rebuilt them for a file read as shared code; -// the agent stood first only because everything else is built by it. -func TestAChangeToTheBuildAgentRebuildsTheBuildAgentAlone(t *testing.T) { - const repo = "http://forge.internal:20000/novox/mesh-catalog.git" - agent := fromRepo("build-agent", repo, "modules/build-agent") - redis := fromRepo("redis", repo, "modules/redis") - postgres := fromRepo("postgres", repo, "modules/postgres") - entries := []inventory.Entry{agent, redis, postgres} - edges := []inventory.Edge{ - {From: "redis", To: "mesh-tools", Kind: inventory.EdgeStandsOn}, - {From: "redis", To: "build-agent", Kind: inventory.EdgeBuiltBy}, - {From: "postgres", To: "build-agent", Kind: inventory.EdgeBuiltBy}, - {From: "mesh-tools", To: "build-agent", Kind: inventory.EdgeBuiltBy}, - {From: "mesh-controller", To: "build-agent", Kind: inventory.EdgeBuiltBy}, - {From: "build-agent", To: "mesh-controller", Kind: inventory.EdgeWorkerOf}, - } - for _, paths := range [][]string{ - {"modules/build-agent/module.json"}, // its manifest alone - {"modules/build-agent/cmd/agent/main.go", "modules/build-agent/module.json"}, // its program too - } { - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "abc", Paths: paths, - ModuleDirs: []string{"modules/build-agent"}, ModuleDirsSaid: true} - touched := whatTheMergeTouched(entries, entries, m, nil) - if len(touched) != 1 || touched[0].Manifest.Module != "build-agent" { - t.Fatalf("%v touched %v", paths, touched) - } - p := planOfMerge(m, []string{"build-agent"}, edges) - if len(p.Tiers) != 1 || len(p.Tiers[0]) != 1 || p.Tiers[0][0] != "build-agent" || len(p.Modules) != 1 { - t.Fatalf("%v planned %v: the build agent alone", paths, p.Tiers) - } - } - // With what it builds moved beside it, both go in one tier (novox/hq ADR 0360 §3): built-by orders nothing. - p := planOfMerge(link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Commit: "abc"}, - []string{"build-agent", "redis"}, edges) - if len(p.Tiers) != 1 || !slices.Equal(p.Tiers[0], []string{"build-agent", "redis"}) || len(p.Modules) != 2 { - t.Fatalf("the agent and what moved beside it, in one tier: %v", p.Tiers) - } -} diff --git a/cmd/mesh-controller/release_provisioned_integration_test.go b/cmd/mesh-controller/release_provisioned_integration_test.go deleted file mode 100644 index 547e0b73..00000000 --- a/cmd/mesh-controller/release_provisioned_integration_test.go +++ /dev/null @@ -1,145 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "slices" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func stepRelease(t *testing.T, b *backlogMesh, n int) inventory.Plan { - t.Helper() - for i := 0; i < n; i++ { - advancePlans(t.Context(), b.open) - } - return b.release(t) -} - -func TestTheRestGateWaitsForTheProvisioner(t *testing.T) { - judgeLeavesAsTheCore(t) - applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: time.Now()} - - t.Run("a engine applied, the client has not said: no verdict, the release stays open", func(t *testing.T) { - b := aBacklog(t) - handingAnchor(t, b, map[string]string{"app": "t1"}) - p := stepRelease(t, b, 8) - g := p.Release.Gates["anchor"] - if !p.Open() || g == nil || g.Verdict != "" || g.Silent || slices.Contains(p.Release.Done, "anchor") { - t.Fatalf("anchor is judged before its provisioner reported: %s %s %+v", p.State, p.Note, g) - } - if !slices.Contains(p.Release.Done, "laptop") { - t.Fatalf("laptop hands nothing and passes on the engine's report: %+v", p.Release) - } - }) - t.Run("b the client failed: a failed apply named with the reason", func(t *testing.T) { - b := aBacklog(t) - says := handingAnchor(t, b, map[string]string{"app": "t1"}) - says["anchor/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: time.Now()} - p := stepRelease(t, b, 8) - g := p.Release.Gates["anchor"] - if g == nil || g.Verdict != inventory.GateFailed || !slices.Contains(p.Release.Failed, "anchor") || - !strings.Contains(p.Note, "provisioning failed (app: no value)") { - t.Fatalf("the failure is not named: %s %q %+v %+v", p.State, p.Note, g, p.Release) - } - }) - t.Run("c no client report past ApplySilentAfter: silent, never done", func(t *testing.T) { - b := aBacklog(t) - handingAnchor(t, b, map[string]string{"app": "t1"}) - p := stepRelease(t, b, 8) - g := p.Release.Gates["anchor"] - if g == nil { - t.Fatalf("no gate: %+v", p.Release) - } - since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) - g.Since = &since - if err := b.open.inventory.SavePlan(t.Context(), &p); err != nil { - t.Fatal(err) - } - p = stepRelease(t, b, 1) - if !slices.Equal(p.Release.Silent, []string{"anchor"}) || slices.Contains(p.Release.Done, "anchor") { - t.Fatalf("anchor is not silent: %s %s %+v", p.State, p.Note, p.Release) - } - }) - t.Run("d both applied: the gate passes", func(t *testing.T) { - b := aBacklog(t) - says := handingAnchor(t, b, map[string]string{"app": "t1"}) - says["anchor/app"] = applied - p := stepRelease(t, b, 8) - if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - }) - t.Run("e a send handing nothing passes on the engine's report", func(t *testing.T) { - b := aBacklog(t) - p := stepRelease(t, b, 8) - if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - }) - t.Run("f a module the delivery does not carry does not touch its end", func(t *testing.T) { - // anchor's declaration also hands `same`, rebuilt with the very bytes and so not carried: its failed report - // is none of this delivery's. - b := aBacklog(t) - says := handingAnchor(t, b, map[string]string{"app": "t1", "same": "t9"}) - says["anchor/app"] = applied - says["anchor/same"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t9", Reason: "stuck", At: time.Now()} - p := stepRelease(t, b, 8) - if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 { - t.Fatalf("another module's failure ended this delivery: %s %s %+v", p.State, p.Note, p.Release) - } - }) -} - -// handingAnchor makes every send to anchor hand the provisioner the given modules (module -> templates digest), and -// the clients' words the given reports (key `node/module`), as the stream would give them (issue 840, ADR 0362). -func handingAnchor(t *testing.T, b *backlogMesh, handed map[string]string) map[string]inventory.ProvReport { - t.Helper() - says := map[string]inventory.ProvReport{} - wasSays := provAppliedOn - provAppliedOn = func(_ context.Context, node, module string) (inventory.ProvReport, bool, error) { - r, ok := says[node+"/"+module] - return r, ok, nil - } - inner := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - got, err := inner(ctx, open, names) - if err != nil { - return got, err - } - for _, n := range names { - if n != "anchor" { - continue - } - inv := open.inventory - s, _, err := inv.SentTo(ctx, n) - if err != nil { - return nil, err - } - builds, _, err := inv.SentBuilds(ctx, n) - if err != nil { - return nil, err - } - plans := map[string]inventory.Provisioning{} - for m, d := range handed { - plans[m] = inventory.Provisioning{Templates: inventory.TemplatesArchive{Digest: d}} - } - raw, err := json.Marshal(plans) - if err != nil { - return nil, err - } - id := nodeID(t, open, n) - if err := inv.RecordSentWith(ctx, id, s.Digest, builds, 0, 0, inventory.Send{Node: id, Provisioning: raw}); err != nil { - return nil, err - } - } - return got, nil - } - t.Cleanup(func() { sendRollout, provAppliedOn = inner, wasSays }) - return says -} diff --git a/cmd/mesh-controller/release_provisioned_test.go b/cmd/mesh-controller/release_provisioned_test.go deleted file mode 100644 index da197dfa..00000000 --- a/cmd/mesh-controller/release_provisioned_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// Issue 840 (novox/hq ADR 0362 §1-5): the release's rest gate, which every delivery passes since ADR 0360, counts a -// machine's send of a provisioned module applied only once the engine and the machine's provisioner client both said -// so; a failed client is a failed apply named with its reason; none past ApplySilentAfter is silent, never done. - -// Issue 840: a walk's end waits on the modules it sent to a machine, not on every module that machine's declaration -// hands to the provisioner. -func TestAWalkEndWaitsOnlyOnTheModulesItSent(t *testing.T) { - now := time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC) - at := now.Add(-10 * time.Second) - handed := map[string]string{"app": "t1", "other": "t2"} - p := inventory.Plan{Modules: map[string]*inventory.PlanModule{"app": {Rest: map[string]inventory.SentDeclaration{ - "ace": {Digest: "d-ace", At: now.Add(-time.Minute), Provisioned: handed}}}}} - reports := []inventory.Reported{{Node: "ace", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-ace"}} - says := map[string]inventory.ProvReport{"ace/app": {Status: inventory.ProvApplied, Templates: "t1", At: at}} - prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok } - // other has said nothing: not this walk's to wait for. - if waiting, failed, silent := walkReports(p, reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 { - t.Fatalf("missing: waiting %v, failed %v, silent %v", waiting, failed, silent) - } - says["ace/other"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t2", Reason: "stuck", At: at} - if waiting, failed, silent := walkReports(p, reports, now, prov); len(waiting)+len(failed)+len(silent) != 0 { - t.Fatalf("failed: waiting %v, failed %v, silent %v", waiting, failed, silent) - } - // once the walk carries it too, it does. - p.Modules["other"] = &inventory.PlanModule{Rest: p.Modules["app"].Rest} - if _, failed, _ := walkReports(p, reports, now, prov); len(failed) != 1 || !strings.Contains(failed[0], "other: stuck") { - t.Fatalf("a carried module's failure is not named: %v", failed) - } -} diff --git a/cmd/mesh-controller/release_test.go b/cmd/mesh-controller/release_test.go deleted file mode 100644 index 6e052994..00000000 --- a/cmd/mesh-controller/release_test.go +++ /dev/null @@ -1,271 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "reflect" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" -) - -// The backlog the old default left — builds registered and sent nowhere — is released by a plan, one -// machine at a time, each judged, never by the next send of something else (novox/hq ADR 0236). - -type backlogMesh struct { - open *stores - sent [][]string - broken map[string]bool -} - -// aBacklog is a mesh where `app` moved c1 → c2 on anchor and laptop, `late` moved on laptop only, and -// `same` was rebuilt with the very artifacts it had: two machines heard, every send applied at once. -func aBacklog(t *testing.T) *backlogMesh { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - b := &backlogMesh{open: open, broken: map[string]bool{}} - withConditionsInMemory(t) - was := doctorFrom - doctorFrom = nil - t.Cleanup(func() { doctorFrom = was }) - - build := func(module, commit, made string, asked time.Time) { - manifest, _ := json.Marshal(catalogue.Manifest{Module: module, Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit, - Repository: "novox/mesh-catalog", Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: made}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit, - Asked: asked}); err != nil { - t.Fatal(err) - } - } - old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) - on := map[string][]string{"app": {"anchor", "laptop"}, "late": {"laptop"}, "same": {"anchor", "laptop"}} - for _, m := range []string{"app", "late", "same"} { - build(m, "c1", "sha256:"+m+"-1", old) - for _, n := range on[m] { - if _, err := inv.Assign(ctx, n, m); err != nil { - t.Fatal(err) - } - } - } - for _, n := range []string{"anchor", "laptop"} { - sent := map[string]string{} - for m, nodes := range on { - for _, x := range nodes { - if x == n { - sent[m] = "c1" - } - } - } - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, sent); err != nil { - t.Fatal(err) - } - } - build("app", "c2", "sha256:app-2", now) - build("late", "c2", "sha256:late-2", now) - build("same", "c2", "sha256:same-1", now) // rebuilt, the same bytes - - assigned := func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { - modules, err := open.inventory.Assigned(ctx, node) - if err != nil { - return nil, err - } - sent, known, err := open.inventory.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - return f.moves(node, modules, sent, known, all), nil - } - wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts - machineMoves = assigned - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { - return map[string]bool{"anchor": true, "laptop": true}, nil - } - n := 0 - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - b.sent = append(b.sent, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - modules, _ := open.inventory.Assigned(ctx, node) - carried := map[string]string{} - for _, m := range modules { - carried[m] = current[m].Commit - } - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { - return nil, err - } - // A broken machine reports its apply failed (novox/hq ADR 0360 §7: the report is the verdict). - outcome := inventory.OutcomeApplied - if b.broken[node] { - outcome = inventory.OutcomeFailed - } - if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: outcome, Declared: digest, Applied: 1, At: time.Now()}); err != nil { - return nil, err - } - } - return names, nil - } - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return f, err - } - for _, r := range reports { - if b.broken[r.Node] { - r.Outcome = inventory.OutcomeFailed - } - f.reports[r.Node] = r - } - return f, nil - } - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - gateWithin, gateEvery = time.Hour, time.Hour - t.Cleanup(func() { - machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather - gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound - }) - return b -} - -func (b *backlogMesh) release(t *testing.T) inventory.Plan { - t.Helper() - plans, err := b.open.inventory.RecentPlans(t.Context(), 10) - if err != nil { - t.Fatal(err) - } - for _, p := range plans { - if p.Release != nil { - return p - } - } - t.Fatal("no release plan") - return inventory.Plan{} -} - -// The backlog goes out one machine at a time: the first judged before the second is sent, each build's -// pass kept; a rebuild with the same bytes is no move at all; and a send outside a gate is refused. -func TestTheBacklogIsReleasedOneMachineAtATimeEachJudged(t *testing.T) { - withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - - f, err := readMoveFacts(ctx, inv) - if err != nil { - t.Fatal(err) - } - moves, _ := machineMoves(ctx, b.open, f, "laptop", false) - var names []string - for _, mv := range moves { - names = append(names, mv.Module) - } - if !reflect.DeepEqual(names, []string{"app", "late"}) { - t.Fatalf("laptop waits for %v; a rebuild with the same bytes is no move", names) - } - // Nothing else may carry them: a send that judges nothing is refused. - if _, err := ungatedIn(ctx, b.open, []string{"laptop"}, ""); !errors.Is(err, errUngated) { - t.Fatalf("a send that judges nothing would carry them: %v", err) - } - - advancePlans(ctx, b.open) - if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: the first machine alone, before it is judged", b.sent) - } - p := b.release(t) - if !reflect.DeepEqual(p.Release.Order, []string{"anchor", "laptop"}) || p.Release.Gate == nil { - t.Fatalf("the release plan is %+v", p.Release) - } - gateEvery = 0 - for i := 0; i < 4; i++ { - advancePlans(ctx, b.open) - } - if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("sent %v", b.sent) - } - p = b.release(t) - if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) { - t.Fatalf("the release plan is %s: %s %+v", p.State, p.Note, p.Release) - } - for _, build := range []string{"build-app-c2", "build-late-c2"} { - if v, found, err := inv.GateOf(ctx, build); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("%s's pass was not kept: %+v %v %v", build, v, found, err) - } - } - // Nothing waits now, and nothing opens again. - if p, err := releaseBacklog(ctx, b.open, ""); err != nil || p != nil { - t.Fatalf("a release opened with nothing waiting: %+v %v", p, err) - } -} - -// A release that fails on its first machine puts back what it carried there, goes no further, and the -// next one waits for a person, said as a condition; a person releases it with why. -func TestAFailedReleaseIsPutBackAndTheNextWaitsForAPerson(t *testing.T) { - withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - gateEvery = 0 - b.broken["anchor"] = true - advancePlans(ctx, b.open) - - p := b.release(t) - if p.State != inventory.PlanFailed { - t.Fatalf("the release is %s: %s", p.State, p.Note) - } - if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}, {"anchor"}}) { - t.Fatalf("sent %v: the first machine, then the put-back to it, and nothing to laptop", b.sent) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { - t.Fatalf("app is at %s, not put back", current["app"].Commit) - } - if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed { - t.Fatal("app's build is not marked failed at its gate") - } - // late still waits on laptop, and is not released on its own after a failure. - if p, err := releaseBacklog(ctx, b.open, ""); err != nil || p != nil { - t.Fatalf("a release opened after a failed one: %+v %v", p, err) - } - if obs := backlogObservation(); len(obs) != 1 || !strings.Contains(obs[0].Summary, "release-backlog") { - t.Fatalf("the held release is not said: %+v", obs) - } - b.broken["anchor"] = false - if err := backlogCommand(ctx, "release-backlog", []string{"--why", "anchor is fixed"}); err != nil { - t.Fatal(err) - } - for i := 0; i < 4; i++ { - advancePlans(ctx, b.open) - } - if p := b.release(t); p.State != inventory.PlanDone || p.Release.By == "" { - t.Fatalf("the person's release is %s (%+v)", p.State, p.Release) - } - if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { - t.Fatalf("late was not released to laptop: %v", sent) - } -} diff --git a/cmd/mesh-controller/removed_module_test.go b/cmd/mesh-controller/removed_module_test.go deleted file mode 100644 index 298949d8..00000000 --- a/cmd/mesh-controller/removed_module_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package main - -import ( - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// 1922/AC1: a tier naming a module the catalogue no longer holds asks it without failing the plan: the module ends -// as deleted (not built, not sent) and the tier's other modules go on. -func TestAModuleTheCatalogueNoLongerHoldsEndsDeletedAndTheTierGoesOn(t *testing.T) { - p, byName := aWalkOfOne("sudo") - p.Tiers = [][]string{{"gone", "sudo"}} - for _, m := range p.Tiers[0] { - askModule(t.Context(), p, m, byName) - } - if g := p.Modules["gone"]; g == nil || g.State != planDeleted || g.Build != "" || g.SentAt != nil { - t.Fatalf("the removed module is not deleted: %+v", g) - } - if p.State == inventory.PlanFailed { - t.Fatalf("the plan failed: %q", p.Note) - } - if s := p.Modules["sudo"]; s == nil || s.State != "asked" { - t.Fatalf("the other module of the tier did not go on: %+v", s) - } -} - -// 1922/AC2: a plan that failed on such a module before the fix is retried: the removed module ends deleted and the -// plan is building again, to go on to its next tier. -func TestARetryOfAPlanFailedOnARemovedModuleResumes(t *testing.T) { - p := inventory.Plan{ID: "plan-1", State: inventory.PlanFailed, Note: "gone is no longer in the catalogue", - Tiers: [][]string{{"gone"}, {"next"}}, - Modules: map[string]*inventory.PlanModule{ - "gone": {State: "failed", Why: "no longer in the catalogue"}, - }} - failed := toRetry(&p, nil, nil) - if len(failed) != 1 || failed[0] != "gone" { - t.Fatalf("the failed set is %v", failed) - } - for _, m := range failed { - askModule(t.Context(), &p, m, map[string]inventory.Entry{}) - } - resumed(&p, "retried") - if p.State != inventory.PlanBuilding || p.Modules["gone"].State != planDeleted { - t.Fatalf("the retry did not resume: %s %+v", p.State, p.Modules["gone"]) - } -} - -// 1922/AC3: an ended plan's note names the verb that releases the modules waiting for a gate, not `push --behind`. -func TestAnEndedPlansHintNamesTheReleaseOfTheBacklog(t *testing.T) { - p := inventory.Plan{State: inventory.PlanFailed, Note: "failed", - Modules: map[string]*inventory.PlanModule{"agent": {State: "built"}}} - sayUnsent(&p, func(string) bool { return true }) - if strings.Contains(p.Note, "push --behind") || !strings.Contains(p.Note, "upgrade release-backlog") { - t.Fatalf("the note reads %q", p.Note) - } -} diff --git a/cmd/mesh-controller/replay201_test.go b/cmd/mesh-controller/replay201_test.go deleted file mode 100644 index 6e37021b..00000000 --- a/cmd/mesh-controller/replay201_test.go +++ /dev/null @@ -1,92 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// TestReplay201ThePlansSend replays novox/hq issue 201 (2026-10-02): two merges a minute apart made two plans; the -// controller's plan rolled the newer controller onto the control node, and the node-engine's plan then sent that -// machine the controller build before it, whose binary refused the seat row the newer one had widened. Here the -// control node, anchor, was last sent the newer controller c2 while the mesh's record of the controller is the -// build before it, c1, which passed its gate. The outcome asserted: a send that is no person's push — composed as a -// plan's send composes it (keepingRecorded, what sendKeeps gives every send through sendToEach) — carries the -// controller at c2, the build the machine was last sent, and not the older record; a person's push still carries -// the record. Written with only what the controller had before its fix (mesh-controller #273) — the inventory's -// build, gate, assignment and send records, keepingRecorded, planFor and declarationFor — so it is laid over the -// commit before; the fix's own TestReplay201 reads what the fix added. Needs the store (MESH_TEST_POSTGRES), and -// is named in testdata/store-tests. -func TestReplay201ThePlansSend(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - const older, newer = "c1aaaaaaaaaa", "c2bbbbbbbbbb" - old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) - record := func(commit string, asked time.Time) inventory.Build { - t.Helper() - manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"}) - b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit, - Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}} - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - return b - } - - // The older controller rolled out and passed its gate; the newer one was asked after it and registered. - before := record(older, old) - if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: older, - Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil { - t.Fatal(err) - } - record(newer, now) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-controller", BuiltFrom: newer, Head: newer, Asked: now}); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil { - t.Fatal(err) - } - // The controller's plan sent anchor the newer controller. - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": newer}); err != nil { - t.Fatal(err) - } - // And the mesh's record of the controller is the build before it — whatever made it so. - if err := inv.RestoreModule(ctx, before); err != nil { - t.Fatal(err) - } - - // The controller each send to anchor carries: a plan's send keeps recorded builds (keepingRecorded, what - // sendKeeps gives every send but a person's push); a person's push composes the mesh's record. - controllerSent := func(sends string, keeping bool) string { - t.Helper() - c := ctx - if keeping { - c = keepingRecorded(ctx) - } - plan, settings, err := resolveNode(c, open, "anchor") - if err != nil { - t.Fatalf("%s to anchor did not compose: %v", sends, err) - } - declared, err := declarationFor(c, open, "anchor", plan, settings) - if err != nil { - t.Fatalf("%s to anchor did not compose: %v", sends, err) - } - return declared.Builds["mesh-controller"] - } - - if got := controllerSent("a plan's send", true); got != newer { - t.Fatalf("a plan's send carried anchor the controller %s, older than the %s anchor was last sent: the "+ - "older binary would read the row its successor widened", short(got), short(newer)) - } - if got := controllerSent("a person's push", false); got != older { - t.Fatalf("a person's push carried the controller %s, not the mesh's record %s", short(got), short(older)) - } -} diff --git a/cmd/mesh-controller/replay309_test.go b/cmd/mesh-controller/replay309_test.go deleted file mode 100644 index c7e46c5d..00000000 --- a/cmd/mesh-controller/replay309_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package main - -import ( - "reflect" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq issue 309, replayed. The group `feat/a-machine-joins-through-the-tunnel` (2026-10-08): the controller's pull request -// moved the controller and the build agent, the node-engine's moved the node-engine, the lab's moved -// nothing. The build agent builds the node-engine (built by: the controller's member first), and the -// node-engine goes before the controller (engine before controller: the node-engine's member first). Both -// rules applied to one pair in opposite directions, and the group was rejected as a cycle; the person merged -// the controller first by hand. A rule read from the graph outranks the rollout default, so the group is -// ordered, the controller's member first. -func TestReplay309(t *testing.T) { - members := []orderMember{ - {ID: "novox/mesh-controller@8bbfdb53db2b", Repository: "novox/mesh-controller", Number: 132}, - {ID: "novox/mesh-host@86cbebd10f8a", Repository: "novox/mesh-host", Number: 52}, - {ID: "novox/mesh-lab@e80a4b1642bf", Repository: "novox/mesh-lab", Number: 61}, - } - reach := map[string]orderReach{ - "novox/mesh-controller@8bbfdb53db2b": {Moved: []string{"build-agent", "mesh-controller", "route-proxy"}, - Manifests: []string{"module.json"}}, - "novox/mesh-host@86cbebd10f8a": {Moved: []string{"mesh-host"}}, - "novox/mesh-lab@e80a4b1642bf": {}, - } - // Every source-built module is built by the build seat's holder; the holder follows the controller. - edges := []inventory.Edge{ - {From: "build-agent", To: "mesh-controller", Kind: inventory.EdgeWorkerOf}, - {From: "mesh-controller", To: "build-agent", Kind: inventory.EdgeBuiltBy}, - {From: "mesh-host", To: "build-agent", Kind: inventory.EdgeBuiltBy}, - {From: "route-proxy", To: "build-agent", Kind: inventory.EdgeBuiltBy}, - } - got := orderOf(members, reach, edges, map[string]moduleChange{"mesh-controller": {Manifest: true}}) - if len(got.Cycle) > 0 { - t.Fatalf("the group is refused as a cycle %v; pairs %+v", got.Cycle, got.Pairs) - } - // Being built by the build seat's holder orders nothing (ADR 0360 section 3, ADR 0419): the node-engine goes - // first by the rollout default, and no pair puts the controller's member before it. - want := []string{"novox/mesh-host@86cbebd10f8a", "novox/mesh-controller@8bbfdb53db2b", "novox/mesh-lab@e80a4b1642bf"} - if !reflect.DeepEqual(got.Order, want) { - t.Fatalf("ordered %v, wanted %v; pairs %+v", got.Order, want, got.Pairs) - } -} diff --git a/cmd/mesh-controller/replay318_test.go b/cmd/mesh-controller/replay318_test.go deleted file mode 100644 index 39ce20ec..00000000 --- a/cmd/mesh-controller/replay318_test.go +++ /dev/null @@ -1,257 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "fmt" - "reflect" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" - "github.com/novox/mesh-controller/internal/link" -) - -// novox/hq issue 318, replayed with only what the controller had before its fix, so it can be laid over the -// older commit: the statements are read from the node-engine's JSON, as the controller hears them. -// -// 2026-10-08, 11:08 UTC: the walk of the builds waiting for a gate sent the laptop three moves in one send — -// docker ba6058ab → c78b5fc9, node-tools 9a4140b4 → d4845f7c, openrazer 88135ad0 → c78b5fc9 — and judged -// them together. openrazer's build put the operator's account in the group `openrazer` (ADR 0252); the -// laptop said the account "relogin needed", and the daemon's unit failed in the account's own service -// manager, which began before the group. The gate read "not yet healthy" for ten minutes, failed at its -// bound, put openrazer back (which by ADR 0252 also took the group back), and the walk failed: docker and -// node-tools, healthy at every judging, never reached the workstation, and every other walk was refused there. -// -// The rule's outcome: a wait for a person is not a failure. The send passes with the wait carried along, -// nothing is put back, the walk goes on to the workstation, and openrazer's condition says, in one sentence -// for the operator, that a new login is needed on the laptop. -func TestReplay318(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - keeper, _ := withConditionsInMemory(t) - was := doctorFrom - doctorFrom = nil - t.Cleanup(func() { doctorFrom = was }) - - build := func(module, repository, commit string, asked time.Time) { - m := catalogue.Manifest{Module: module, Version: "1"} - if module == "openrazer" && commit == "c78b5fc9" { - // The build that put the operator's account in the group `openrazer` (ADR 0252). - m.Resources = []map[string]any{{"id": "operator", "type": "user", "name": "operator", - "groups": []any{"openrazer"}}} - } - manifest, _ := json.Marshal(m) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + module + "-" + commit, Module: module, Commit: commit, - Repository: repository, Path: "modules/" + module, Manifest: manifest, Asked: asked, At: asked, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + module + "-" + commit}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, m, inventory.Source{ - Repository: repository, Seat: "git", Path: "modules/" + module, BuiltFrom: commit, Head: commit, - Asked: asked}); err != nil { - t.Fatal(err) - } - } - type move struct{ module, repository, from, to string } - moves := []move{ - {"docker", "novox/mesh-catalog", "ba6058ab", "c78b5fc9"}, - {"node-tools", "novox/mesh-tools", "9a4140b4", "d4845f7c"}, - {"openrazer", "novox/mesh-catalog", "88135ad0", "c78b5fc9"}, - } - machines := []string{"laptop", "workstation"} - if _, err := inv.AddNode(ctx, "workstation"); err != nil { - t.Fatal(err) - } - old, recent := time.Now().Add(-3*time.Hour), time.Now().Add(-time.Minute) - for _, mv := range moves { - build(mv.module, mv.repository, mv.from, old) - for _, n := range machines { - if _, err := inv.Assign(ctx, n, mv.module); err != nil { - t.Fatal(err) - } - } - } - for _, n := range machines { - sent := map[string]string{} - for _, mv := range moves { - sent[mv.module] = mv.from - } - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, sent); err != nil { - t.Fatal(err) - } - } - for _, mv := range moves { - build(mv.module, mv.repository, mv.to, recent) - } - - // What each machine says, as its node-engine words it: openrazer's account waits for a new login, and the - // daemon's unit failed in that account's own manager; docker's container is healthy. - statement := func(at time.Time) link.Health { - raw := fmt.Sprintf(`{"contract": %d, "at": %q, "resources": [ - {"module": "docker", "resource": "docker.engine", "kind": "service", "target": "docker.service", - "state": "healthy", "since": %q}, - {"module": "openrazer", "resource": "openrazer.operator", "kind": "account", "target": "operator", - "account": "operator", "state": "unhealthy", "since": %q, "streak": 3, - "reason": "relogin needed: operator is in the group openrazer, and its running session began before it was; log out of every session and in again, or reboot"}, - {"module": "openrazer", "resource": "openrazer.daemon", "kind": "unit", "target": "openrazer-daemon.service", - "account": "operator", "state": "unhealthy", "since": %q, "streak": 3, - "reason": "failed in the account's own service manager (exit-code)"}]}`, - link.ReadinessContract, at.Format(time.RFC3339Nano), at.Format(time.RFC3339Nano), at.Format(time.RFC3339Nano), - at.Format(time.RFC3339Nano)) - var h link.Health - if err := json.Unmarshal([]byte(raw), &h); err != nil { - t.Fatal(err) - } - return h - } - - wasMoves, wasHeard, wasSend, wasGather := machineMoves, releaseHeard, sendRollout, gatherGateFacts - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - t.Cleanup(func() { - machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather - gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound - }) - machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { - modules, err := open.inventory.Assigned(ctx, node) - if err != nil { - return nil, err - } - sent, known, err := open.inventory.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - return f.moves(node, modules, sent, known, all), nil - } - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { - return map[string]bool{"laptop": true, "workstation": true}, nil - } - var sends [][]string - n := 0 - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - sends = append(sends, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - modules, _ := open.inventory.Assigned(ctx, node) - carried := map[string]string{} - for _, m := range modules { - carried[m] = current[m].Commit - } - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { - return nil, err - } - if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { - return nil, err - } - } - return names, nil - } - // Every judging hears each machine's newest statement first, as the controller does between judgings: - // the account's wait and the failed unit are raised as openrazer's condition, after the send. - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - now := time.Now() - f := gateFacts{now: now, reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}, health: map[string]inventory.NodeHealth{}, - judged: true} - for _, m := range machines { - if err := stateHealth(ctx, open.inventory, keeper, m, statement(now), now); err != nil { - return f, err - } - f.served[m] = served{runtime: true, tools: map[string]bool{}} - } - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return f, err - } - for _, r := range reports { - f.reports[r.Node] = r - } - if f.health, err = open.inventory.Healths(ctx); err != nil { - return f, err - } - f.open, f.openErr = keeper.Open(ctx) - return f, nil - } - // The bound is reached at the first judging that is not a pass: what happened at 11:21 happens at once. - gateWithin, gateEvery, gateBound = time.Hour, 0, 0 - - for i := 0; i < 12; i++ { - advancePlans(ctx, open) - time.Sleep(5 * time.Millisecond) - } - - var p inventory.Plan - plans, err := inv.RecentPlans(ctx, 10) - if err != nil { - t.Fatal(err) - } - for _, q := range plans { - if q.Release != nil { - p = q - break - } - } - if p.Release == nil { - t.Fatal("no walk of the builds waiting for a gate was opened") - } - if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, machines) { - t.Fatalf("the walk is %s on %v: %s; want it done on the laptop and then the workstation", p.State, - p.Release.Done, p.Note) - } - if !reflect.DeepEqual(sends, [][]string{{"laptop"}, {"workstation"}}) { - t.Fatalf("sent %v; want the laptop, then the workstation, and nothing put back", sends) - } - current, err := inv.CurrentBuilds(ctx) - if err != nil { - t.Fatal(err) - } - if current["openrazer"].Commit != "c78b5fc9" { - t.Fatalf("openrazer is registered at %s: its build was put back for a wait for a person", current["openrazer"].Commit) - } - for _, mv := range moves { - v, found, err := inv.GateOf(ctx, "build-"+mv.module+"-"+mv.to) - if err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("%s's build %s: verdict %+v (found %v, %v); want a pass", mv.module, mv.to, v, found, err) - } - if mv.module == "openrazer" && !strings.Contains(v.Why, "relogin needed on ") { - t.Errorf("openrazer's pass does not carry its wait for a person: %q", v.Why) - } - } - // What the operator reads: one condition per machine for openrazer, saying a new login is needed there, - // a warning for a person, and no fault of the build. - list, err := keeper.Open(ctx) - if err != nil { - t.Fatal(err) - } - var said []string - for _, c := range list { - said = append(said, c.Key+": "+c.Summary) - if c.Subject.ID == "openrazer.laptop" { - if !strings.HasPrefix(c.Summary, "relogin needed on laptop") || c.Severity == conditions.Urgent || - c.Resolver != conditions.ResolverOperator { - t.Errorf("openrazer's condition on the laptop: %s %s %s %q", c.Key, c.Severity, c.Resolver, c.Summary) - } - } - } - if !strings.Contains(strings.Join(said, "\n"), "relogin needed on laptop") { - t.Errorf("nothing says a new login is needed on the laptop:\n%s", strings.Join(said, "\n")) - } -} diff --git a/cmd/mesh-controller/replay325_test.go b/cmd/mesh-controller/replay325_test.go deleted file mode 100644 index b4dacb4f..00000000 --- a/cmd/mesh-controller/replay325_test.go +++ /dev/null @@ -1,57 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "slices" - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// novox/hq issue 325, replayed with only what the controller had before its fix, so it can be laid over the -// older commit. On 2026-10-08 the catalogue's pull request adding `sensors` merged, and the merge asked for -// its build (issue 300). About a minute later `assign g14 sensors` answered "no module of that name: -// sensors"; a few minutes later the same call worked, because the build had registered the module. The -// answer read as "nobody registered it". An assignment made while the build runs says the build — where it -// is from, since when, its id — is kept, and is made when the build registers the module. -func TestReplay325(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "networkmanager", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - // The build the merge asked before R4e (nothing asks one since, ADR 0501 §6), kept as it was. - mergeAdding(t, open, "modules/sensors", "5e450125aa") - - said, err := assign(ctx, open, "laptop", "sensors") - if err != nil { - t.Fatalf("assign while the merge's build runs was refused: %v", err) - } - for _, want := range []string{"being built", "novox/mesh-catalog", "modules/sensors", "b-modules/sensors"} { - if !strings.Contains(said, want) { - t.Fatalf("the answer does not say %q:\n%s", want, said) - } - } - - manifest, _ := json.Marshal(catalogue.Manifest{Module: "sensors", Version: "1"}) - if err := (builds{open.inventory, open}).Built(ctx, link.BuildResult{ID: "b-modules/sensors", - Repository: "novox/mesh-catalog", Path: "modules/sensors", Ref: "main", Commit: "5e450125aa", On: "anchor", - Module: "sensors", Manifest: manifest, - Source: &link.SourceOnSeat{Repository: "novox/mesh-catalog", Seat: "git"}}); err != nil { - t.Fatal(err) - } - assigned, err := open.inventory.Assigned(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if !slices.Contains(assigned, "sensors") { - t.Fatalf("the build registered sensors and laptop runs %v", assigned) - } -} diff --git a/cmd/mesh-controller/replay335_integration_test.go b/cmd/mesh-controller/replay335_integration_test.go deleted file mode 100644 index 98d142a2..00000000 --- a/cmd/mesh-controller/replay335_integration_test.go +++ /dev/null @@ -1,68 +0,0 @@ -//go:build integration - -package main - -import ( - "reflect" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - tm := aTierMesh(t, "m01", "m02") - ctx := t.Context() - inv := tm.open.inventory - - advancePlans(ctx, tm.open) - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: the first machine once, for the tier", tm.sent) - } - p := tm.plan(t) - lead, carried := p.Modules["m01"], p.Modules["m02"] - if lead.Gate == nil || carried.GatedBy != "m01" { - t.Fatalf("m02 is not carried by m01's send: lead %+v, carried %+v", lead.Gate, carried) - } - // The lead passed; the carried module's turn did not come. The first send is past the wait's bound. - sent := time.Now().UTC().Add(-planWaitBound - time.Minute) - judged := sent.Add(2 * time.Minute) - lead.FirstAt, carried.FirstAt, lead.Gate.Since = &sent, &sent, &sent - lead.Gate.Verdict, lead.Gate.Why, lead.Gate.JudgedAt, lead.Gate.Kept = inventory.GatePassed, - "healthy 3 times over 2m5s", &judged, true - carried.Gate = nil - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - // Another walk's send reached anchor, which has not reported on it. - if err := inv.RecordSent(ctx, nodeID(t, tm.open, "anchor"), "d-anchor-elsewhere", - map[string]string{"m01": "c2", "m02": "c2"}); err != nil { - t.Fatal(err) - } - - advancePlans(ctx, tm.open) - p = tm.plan(t) - if p.State == inventory.PlanFailed { - t.Fatalf("the plan failed after its gate passed: %s", p.Note) - } - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("sent %v: the rest once, as the pass said", tm.sent) - } - current, err := inv.CurrentBuilds(ctx) - if err != nil { - t.Fatal(err) - } - for _, m := range []string{"m01", "m02"} { - if current[m].Commit != "c2" { - t.Errorf("%s was put back to %s after its gate passed", m, current[m].Commit) - } - if failed, _ := inv.GateFailed(ctx, "build-"+m+"-2"); failed { - t.Errorf("%s's build was marked failed at its gate after the gate passed", m) - } - } - if g := p.Modules["m02"].Gate; g == nil || g.Verdict != inventory.GatePassed { - t.Errorf("m02 did not take over m01's pass: %+v", g) - } -} diff --git a/cmd/mesh-controller/replay335_test.go b/cmd/mesh-controller/replay335_test.go deleted file mode 100644 index 5cd51649..00000000 --- a/cmd/mesh-controller/replay335_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package main - -import ( - "reflect" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// TestReplay335 replays novox/hq issue 335 (2026-10-08): dunst's new build passed its gate on the laptop -// (healthy 3 times over 2m5s); its send to the rest was refused while another walk's build waited on the -// workstation; that walk then sent the laptop a declaration the laptop did not report on; and thirty minutes -// after the first send the plan read the laptop's silence as the passed build never applied, marked it failed -// at its gate with the pass's own words, and put it back. Written with only what the controller had before -// its fix, so it is laid over the commit before. -func TestReplay335(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - t.Run("the first machine's later reports", testAPassedGateIsNotJudgedAgainFromTheFirstMachinesLaterReports) - t.Run("a walk stopped after the pass", testAWalkStoppedAfterItsGatePassedKeepsThePass) -} - -// novox/hq issue 335: a build that passed its gate on its first machine is not judged again from that -// machine's later reports. On 2026-10-08 the send to the rest waited on another walk, that walk sent the -// first machine a declaration it did not report for half an hour, and the plan failed the passed build at -// the wait's bound and put it back. -func testAPassedGateIsNotJudgedAgainFromTheFirstMachinesLaterReports(t *testing.T) { - sentAt := time.Date(2026, 10, 8, 16, 44, 45, 0, time.UTC) - judged := sentAt.Add(2 * time.Minute) - later := sentAt.Add(5 * time.Minute) - state := inventory.PlanModule{First: []string{"laptop"}, FirstAt: &sentAt, - Gate: &inventory.PlanGate{Machines: []string{"laptop"}, Verdict: inventory.GatePassed, - Why: "healthy 3 times over 2m5s", JudgedAt: &judged, Kept: true}} - running := []string{"laptop", "workstation"} - now := sentAt.Add(30*time.Minute + 9*time.Second) - - for what, reports := range map[string][]inventory.Reported{ - "no report about what it was sent since": {{Node: "laptop", At: &later, Outcome: inventory.OutcomeApplied, Current: false}}, - "another send failed there since": {{Node: "laptop", At: &later, Outcome: inventory.OutcomeFailed, Current: true}}, - "no report at all": nil, - } { - step := nextRollout(state, running, false, reports, now, 30*time.Minute) - if step.failed != "" || step.waiting != "" || !reflect.DeepEqual(step.send, []string{"workstation"}) { - t.Errorf("%s: %+v, want the rest sent as the pass said", what, step) - } - } -} - -// novox/hq issue 335: whatever stops a walk after its gate passed, the passed build is not marked failed at -// its gate, nor put back. -func testAWalkStoppedAfterItsGatePassedKeepsThePass(t *testing.T) { - sentAt := time.Date(2026, 10, 8, 16, 44, 45, 0, time.UTC) - g := &inventory.PlanGate{Machines: []string{"laptop"}, Verdict: inventory.GatePassed, Why: "healthy 3 times over 2m5s"} - state := &inventory.PlanModule{Build: "build-1", First: []string{"laptop"}, FirstAt: &sentAt, Gate: g} - p := &inventory.Plan{ID: "plan-1", Modules: map[string]*inventory.PlanModule{"dunst": state}} - // No stores: a walk that keeps the pass touches none, and one that reaches for them is putting it back. - defer func() { - if r := recover(); r != nil { - t.Fatalf("the passed build was taken to be failed and put back: %v", r) - } - }() - failFirstSend(t.Context(), nil, p, "dunst", state, []string{"laptop"}, "laptop did not report it applied within 30m0s", - []string{"workstation"}) - if g.Verdict != inventory.GatePassed || g.Rollback != "" { - t.Fatalf("the passed gate became %q, rollback %q", g.Verdict, g.Rollback) - } - if strings.Contains(p.Note, "failed its gate") || strings.Contains(p.Note, "put back") || - !strings.Contains(p.Note, "did not report it applied") { - t.Fatalf("the note reads %q", p.Note) - } -} - -// novox/hq issue 335 review: **a module carried by its lead's send takes over the lead's pass before its own -// step is read.** The state is the one a step leaves when the lead's gate passed and the step ended before the -// carried module's turn (an error read before it, kept with the plan): the lead passed, the carried module has -// no gate of its own yet. Since then another send reached the first machine and it has not reported on it, and -// the first send is older than the wait for a first machine's report. Read before the pass is taken over, the -// carried module's step said the first machine never applied it, and the passed build was put back. diff --git a/cmd/mesh-controller/replay336_test.go b/cmd/mesh-controller/replay336_test.go deleted file mode 100644 index 29471508..00000000 --- a/cmd/mesh-controller/replay336_test.go +++ /dev/null @@ -1,95 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// TestReplay336 replays novox/hq issue 336 (2026-10-08): a catalogue merge built a new bus build for the -// control-node; from then on every send to that machine was refused for the bus's planned step, which only a -// person starts, and for 28 minutes nothing but the walks' notes said so. The outcome asserted: the first -// watchdog tick after the first refusal opens a condition for the operator that names what waits and the verb -// that ends it, and it clears once the bus's machine runs the new build. Written with only what the controller -// had before its fix — the stores, register, assign, a plan saved and advanced, the watchdogs' gathering and -// seeing — so it is laid over the commit before. -func TestReplay336(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - bus := catalogue.Manifest{Module: "nats", Version: "2", Provides: []catalogue.Offer{{Name: "mesh-bus"}}} - if err := inv.RegisterModule(ctx, bus, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", - Path: "modules/nats", BuiltFrom: "32307bd1", Head: "32307bd1"}); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "engine", Version: "1"}) - for _, m := range []string{"nats", "engine"} { - if _, err := inv.Assign(ctx, "anchor", m); err != nil { - t.Fatal(err) - } - } - // The control-node runs the bus build it was last sent; the mesh holds a newer one, which waits for its step. - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "88135ad0"}); err != nil { - t.Fatal(err) - } - // A walk of the engine, built, whose tier sends to the control-node. - now := time.Now().UTC() - plan := inventory.Plan{ID: "plan-engine", Repository: "novox/mesh-host", Commit: "e1e1e1e1", Created: now, - State: inventory.PlanBuilding, Tiers: [][]string{{"engine"}}, - Modules: map[string]*inventory.PlanModule{"engine": {State: "built", BuiltAt: &now, Commit: "e1e1e1e1", Build: "b"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - advancePlans(ctx, open) - p, err := inv.PlanByID(ctx, "plan-engine") - if err != nil { - t.Fatal(err) - } - if !strings.Contains(p.Note, errBusWaits.Error()) { - t.Fatalf("the walk's send to the bus's machine was not refused for the bus: %s %q", p.State, p.Note) - } - - store := conditions.NewInMemory() - k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, Teller: &conditions.Told{}}) - defer k.Close(context.Background()) - w := &watchdogs{open: open, keeper: k, started: now.Add(-time.Hour)} - waiting := func() []conditions.Condition { - t.Helper() - w.see(ctx, w.gather(ctx)) - all, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - var out []conditions.Condition - for _, c := range all { - if strings.HasPrefix(c.Key, "bus.") && c.Resolver == conditions.ResolverOperator { - out = append(out, c) - } - } - return out - } - got := waiting() - if len(got) != 1 { - t.Fatalf("the first tick after the refusal told the operator nothing about the bus's step: %d condition(s)", len(got)) - } - for _, want := range []string{"anchor", "engine", "mesh-controller.bus", "upgrade", "32307bd1"} { - if !strings.Contains(got[0].Summary, want) { - t.Errorf("the condition does not say %q: %s", want, got[0].Summary) - } - } - // The step taken: the control-node is sent the new bus build, and the wait is over. - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2", map[string]string{"nats": "32307bd1"}); err != nil { - t.Fatal(err) - } - if got := waiting(); len(got) != 0 { - t.Fatalf("the bus's machine runs the new build, and the operator is still asked: %+v", got) - } -} diff --git a/cmd/mesh-controller/replay348_test.go b/cmd/mesh-controller/replay348_test.go deleted file mode 100644 index 30f23918..00000000 --- a/cmd/mesh-controller/replay348_test.go +++ /dev/null @@ -1,96 +0,0 @@ -//go:build integration - -package main - -import ( - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" -) - -// Replay of novox/hq issue 348 (2026-10-09): the control node's resolver stopped answering on its private -// address at 10:57:57 UTC, and the machine's network condition was raised at 10:58:45. The controller and the -// node-engine were sent to their first machines at 10:59:34. The restarted node-engine's first statement judged -// the names part once — unknown, "one look failed; a second decides" — and that statement cleared the condition, -// though the resolver still refused. Its next look raised it again at 11:00:23, after the send, and at 11:10:29 -// both builds failed their gate with "raised since it was sent: machine..network" and were put -// back, for a fault that began before they were sent. -// -// Written only with what the controller had before its fix (mesh-controller #179): a mesh, the machine's -// statements as stateHealth keeps them, the keeper's open conditions and the gate's aboutTheMachine — so the -// prover (mesh-lab replays/cmd/prove) lays it over the commit before. Its outcome is the operator's acceptance: -// a build is never put back for a problem that started before it was sent; and the fault is not forgotten -// while the resolver still refuses. Needs a store: MESH_TEST_POSTGRES. -func TestReplay348AFaultFromBeforeTheSendFailsNoGate(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv, k := open.inventory, conditionsFrom - const key = "machine.anchor.network" - names := func(state string, streak int) link.NetworkPart { - p := link.NetworkPart{Part: link.PartNames, State: state, Since: h0, Streak: streak} - if state == link.StateUnhealthy { - p.Reason = "1 of its 2 resolvers do not answer as the mesh's do" - p.Said = "10.77.0.1 — anchor.internal (IPv4): read udp 10.77.0.1:35244->10.77.0.1:53: read: connection refused" - p.Toward = []string{"10.77.0.1"} - } - return p - } - say := func(at time.Time, state string, streak int) { - t.Helper() - n := &link.NetworkHealth{State: state, Since: h0, Parts: []link.NetworkPart{names(state, streak)}} - if err := stateHealth(ctx, inv, k, "anchor", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil { - t.Fatal(err) - } - } - network := func() (conditions.Condition, bool) { - t.Helper() - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - for _, c := range list { - if c.Key == key { - return c, true - } - } - return conditions.Condition{}, false - } - - // 10:58:45 — the node-engine's second failing look: raised. - say(h0, link.StateUnhealthy, 2) - if _, ok := network(); !ok { - t.Fatal("the resolver refusing every question on the control node raised nothing") - } - // 10:59:34 — the controller and the node-engine are sent. - time.Sleep(10 * time.Millisecond) - sent := time.Now().UTC() - time.Sleep(10 * time.Millisecond) - - // 10:59:42 — the restarted engine's first statement: one look failed, a second decides. - say(h0.Add(time.Minute), link.StateUnknown, 1) - if _, ok := network(); !ok { - t.Error("the restarted node-engine's first, undecided statement cleared the network condition while " + - "the resolver still refused") - } - // 11:00:23 — its second look: unhealthy again. - time.Sleep(10 * time.Millisecond) - say(h0.Add(2*time.Minute), link.StateUnhealthy, 2) - if _, ok := network(); !ok { - t.Fatal("the resolver still refusing raised nothing") - } - - // 11:10:29 — the gate on the control node, for the builds sent after the fault began. - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - for _, moved := range [][]string{{"mesh-host"}, {"mesh-controller"}} { - w := aboutTheMachine("anchor", moved, sent, gateFacts{judged: true, open: list}) - if w.whole != "" || len(w.on) != 0 { - t.Errorf("%s, sent after the resolver stopped answering, was held for that fault: whole %q, on %v", - moved[0], w.whole, w.on) - } - } -} diff --git a/cmd/mesh-controller/replay349_test.go b/cmd/mesh-controller/replay349_test.go deleted file mode 100644 index 930961fb..00000000 --- a/cmd/mesh-controller/replay349_test.go +++ /dev/null @@ -1,108 +0,0 @@ -//go:build integration - -package main - -import ( - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// Replay of novox/hq issue 349 (2026-10-09): two merges into the catalogue's main were acted on in reverse order. -// The later one (a082615b, a security fix to the forge's module) was planned first; the earlier one (8ff8197a, -// another module), handed over late, was planned after it, and its plan superseded the later merge's at tier 0 -// with "gitea planned there again": the forge's module was to be built for the commit without the fix. Written -// only with what the controller had before its fix (mesh-controller #179): a mesh, two registered modules of the -// catalogue, the merges as the forge announces them, the plans the inventory keeps — so it is laid over the -// commit before. Since novox/hq ADR 0276 the earlier merge waits in a batch behind the later merge's walk and is -// planned only when that walk ends and the cutter looks again; the replay ends the walk and has a merge of another -// repository heard, which looks, and a plan of the two merges left standing with nothing planned fails it. -func TestReplay349(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asksWithPaths(t) - for _, module := range []string{"app", "notes"} { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: module, Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + module, Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "other", Version: "1"}, - inventory.Source{Repository: "novox/other", Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - // Both made after the modules were last looked at, so neither is history. - made := time.Now().UTC().Add(time.Minute) - merge := func(commit, module string, at time.Time) link.SourceMoved { - return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, - MergedAt: at.Format(time.RFC3339Nano), Paths: []string{"modules/" + module + "/module.json"}, - ModuleDirs: []string{"modules/" + module}, ModuleDirsSaid: true} - } - const fix, earlier = "a082615bfix", "8ff8197aearlier" - // The later merge, the fix to notes, is acted on first; the earlier one, moving app, after it. - if err := (following{open: open}).SourceMoved(ctx, merge(fix, "notes", made.Add(time.Minute))); err != nil { - t.Fatal(err) - } - if err := (following{open: open}).SourceMoved(ctx, merge(earlier, "app", made)); err != nil { - t.Fatal(err) - } - - // The later merge's plan builds and ends; then a merge of another repository comes. Whatever holds the earlier - // merge back while that plan runs (a batch queued behind it, novox/hq ADR 0276) is decided by now: every plan - // of the two merges left standing says what it builds. - plans, err := inv.RecentPlans(ctx, 20) - if err != nil { - t.Fatal(err) - } - for _, p := range plans { - if p.Commit == fix && p.Open() { - p.State = inventory.PlanDone - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - } - } - if err := (following{open: open}).SourceMoved(ctx, link.SourceMoved{Owner: "novox", Repo: "other", Base: "main", - Commit: "c1other", MergedAt: made.Add(2 * time.Minute).Format(time.RFC3339Nano), - Paths: []string{"main.go"}}); err != nil { - t.Fatal(err) - } - if plans, err = inv.RecentPlans(ctx, 20); err != nil { - t.Fatal(err) - } - // A plan superseded hands its modules to the plan that took it over; the plans left standing are what the mesh - // builds. Each plan of the two merges left standing plans modules; one builds app, the earlier merge's module; - // one builds notes, the module the later merge fixed, and every one that does builds it at the commit holding - // the fix. - app, notes := 0, 0 - for _, p := range plans { - t.Logf("%s %s at %.8s: %v %s", p.ID, p.State, p.Commit, p.Tiers, p.Note) - if p.State == inventory.PlanSuperseded || (p.Commit != fix && p.Commit != earlier) { - continue - } - if len(p.Modules) == 0 { - t.Errorf("%s, of the merge at %s, is left standing and plans nothing: %s", p.ID, p.Commit, p.Note) - } - if _, holds := p.Modules["app"]; holds { - app++ - } - if _, holds := p.Modules["notes"]; holds { - notes++ - if p.Commit != fix { - t.Errorf("%s plans the fixed module notes at %s, a commit without the fix %s: %s", p.ID, p.Commit, fix, - p.Note) - } - } - } - if app == 0 { - t.Error("no plan left standing builds app, the module the earlier merge moved") - } - if notes == 0 { - t.Error("no plan left standing builds notes, the module the later merge fixed") - } -} diff --git a/cmd/mesh-controller/replay614_test.go b/cmd/mesh-controller/replay614_test.go deleted file mode 100644 index 4c2f9d9a..00000000 --- a/cmd/mesh-controller/replay614_test.go +++ /dev/null @@ -1,76 +0,0 @@ -//go:build integration - -package main - -import ( - "reflect" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// **A new SDK plans the toolchain, and then every bundle compiled in it** (novox/hq issue 614), reasoned -// as `mesh-controller.delivery-plan` reasons it: the modules registered as a build registers them — -// resolved, so carrying no `build` — their builds recorded with what they stood on and what they made, -// the graph read by theGraph and the change scoped by pullScope. On 2026-10-13 mesh-sdk 0.1.14 planned -// "builds mesh-sdk, sent nowhere": mesh-tools' `build.on` naming the SDK's package and the providers' -// TypeScript bundles were edges only while their manifests were unbuilt. -// -// **The replay of issue 614** (mesh-lab R614): written only with what the controller had before its fix — a -// store, modules registered and builds recorded, theGraph and pullScope — so it is laid over the commit before -// and fails there with the plan of that day, [[mesh-sdk]]. Needs a store: MESH_TEST_POSTGRES. -func TestReplay614ANewSDKPlansTheToolchainAndThenItsBundles(t *testing.T) { - inv := inventory.ForTest(t) - ctx := t.Context() - asked := time.Now().Add(-time.Hour) - digest := func(c string) string { return "@sha256:" + strings.Repeat(c, 64) } - register := func(m catalogue.Manifest, repository, path string, against []string, made []inventory.Artifact) { - t.Helper() - if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: repository, Seat: "git", Path: path, - Ref: "main", BuiltFrom: "old", Asked: asked}); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-" + m.Module, Repository: repository, Ref: "main", - Module: m.Module, Commit: "old", On: "builder", Path: path, Against: against, Made: made, Asked: asked}); err != nil { - t.Fatal(err) - } - } - bundle := func(module, language string) catalogue.Manifest { - return catalogue.Manifest{Module: module, Version: "1", Bundles: []catalogue.Bundle{{ - Name: "code", Language: language, Source: catalogue.ArtifactStoreScheme + module + "/code" + digest("2"), - Entrypoints: []string{"tools/index.js"}}}} - } - register(catalogue.Manifest{Module: "build-agent", Version: "1", - Claims: []catalogue.Claim{{Name: "node-build-agent", Scope: catalogue.ScopeNode}}}, - "novox/mesh-catalog", "modules/build-agent", nil, nil) - register(catalogue.Manifest{Module: "mesh-sdk", Version: "1"}, "novox/mesh-sdk", "", nil, - []inventory.Artifact{{Name: "lib", Kind: "package", Reference: "@novox/mesh-sdk@0.1.13"}}) - register(catalogue.Manifest{Module: "mesh-tools", Version: "1"}, "novox/mesh-tools", "", - []string{"@novox/mesh-sdk@0.1.13", catalogue.ArtifactStoreScheme + "upstream/docker.io/library/node" + digest("0")}, - []inventory.Artifact{{Name: "build", Kind: "image", Reference: catalogue.ArtifactStoreScheme + "mesh-tools/build" + digest("1")}}) - register(catalogue.Manifest{Module: "mesh-tools-go", Version: "1"}, "novox/mesh-tools", "go", nil, nil) - register(bundle("minio", "typescript"), "novox/mesh-catalog", "modules/minio", nil, nil) - register(bundle("umami", "typescript"), "novox/mesh-catalog", "modules/umami", nil, nil) - register(bundle("dnsmasq", "go"), "novox/mesh-catalog", "modules/dnsmasq", nil, nil) - - entries, read, edges, err := theGraph(ctx, inv) - if err != nil { - t.Fatal(err) - } - m := orderMember{ID: "novox/mesh-sdk@new", Repository: "novox/mesh-sdk", Head: "new", - Paths: []string{"src/provisioner/index.ts", "package.json"}} - s := pullScope(m.pull(), entries, read, edges) - // Since R4g no bundle is compiled in a toolchain image (526eed99, issue 1851 C1b: the toolchain edge is gone), so a - // new SDK plans itself and what stands on its package; the TypeScript bundles move by their own build keys, whose - // fingerprints name what they are built from (issue 1938: this test was kept from older runs until then). - want := [][]string{{"mesh-sdk"}, {"mesh-tools"}} - if !reflect.DeepEqual(s.Reach.Plan.Tiers, want) { - t.Fatalf("a new SDK plans %v, want %v (edges %v)", s.Reach.Plan.Tiers, want, edges) - } - if !reflect.DeepEqual(s.Dependents, []string{"mesh-tools"}) { - t.Errorf("dependents %v", s.Dependents) - } -} diff --git a/cmd/mesh-controller/replays_integration_test.go b/cmd/mesh-controller/replays_integration_test.go deleted file mode 100644 index 76f28db8..00000000 --- a/cmd/mesh-controller/replays_integration_test.go +++ /dev/null @@ -1,473 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "os" - "slices" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -func TestReplay263AnIdentityTooLongNeverRefusesItsProvidersMachine(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "resolver", Version: "1", - Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}}) - register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1", - Requires: []string{"wildcard-resolution"}}) - for _, a := range [][2]string{{"anchor", "resolver"}, {"laptop", "networkmanager"}} { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - // The consumer's machine composes first, as a push does: what it is sent is what its provider grants. - for _, node := range []string{"laptop", "anchor"} { - plan, settings, err := resolveNode(ctx, open, node) - if err != nil { - t.Fatalf("%s does not resolve: %v", node, err) - } - if _, err := declarationFor(ctx, open, node, plan, settings); err != nil { - t.Fatalf("%s cannot be sent anything — the consumer networkmanager on laptop, identified "+ - "mesh_laptop_networkmanager (26 characters), refused it: %v", node, err) - } - } -} - -func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { - t.Fatal(err) - } - for _, m := range []catalogue.Manifest{ - {Module: "store", Version: "1", - Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}, - Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}}, - Serves: map[string]map[string]any{"postgres-database": {"port": 5432}}, - Grants: map[string]string{"postgres-database": "/var/lib/mesh/store/grants"}}, - {Module: "resolver", Version: "1", - Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}, - Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}}, - {Module: "network", Version: "1", Requires: []string{"wildcard-resolution"}}, - {Module: "board", Version: "1", Requires: []string{"postgres-database"}}, - } { - register(t, open, m) - } - assignAll := func(pairs ...[2]string) { - for _, a := range pairs { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - } - // The control node holds the mesh's store and resolver seats; the home server runs its own of each. - assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"}) - for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} { - if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil { - t.Fatal(err) - } - } - assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"}, - [2]string{"laptop", "board"}) - - plan, _, err := resolveNode(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - var board, network string - for _, n := range plan.Needs { - switch { - case n.For == "board" && n.Name == "postgres-database": - board = n.From - case n.For == "network" && n.Name == "wildcard-resolution": - network = n.From - } - } - if board != "laptop" { - t.Fatalf("the consumer beside its store was bound to the store on %q, which would make it a new, empty "+ - "database there (issue 273)", board) - } - if network != "anchor" { - t.Fatalf("the resolver was bound to %q; its seat is held on anchor (issue 258)", network) - } -} - -func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) { - withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - - crashLoop := []byte(`{"contract":1,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` + - `"kind":"container","target":"app-server","state":"unhealthy","reason":"restarting","since":"2026-10-07T00:00:00Z",` + - `"streak":5,"restarts":2}]}`) - if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" { - raw, err := os.ReadFile(path) - if err != nil { - t.Fatalf("the engine's statement of the crash loop: %v", err) - } - crashLoop = raw - } - // `null` is an engine that states nothing — older than the judging — and its reports carry no health. - var stated map[string]any - if err := json.Unmarshal(crashLoop, &stated); err != nil { - t.Fatal(err) - } - - for _, b := range []inventory.Build{ - {ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app", - Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)}, - {ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app", - Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)}, - } { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit}) - b.Manifest = manifest - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - registerAt := func(commit string, asked time.Time) { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit, - Head: commit, Asked: asked}); err != nil { - t.Fatal(err) - } - } - registerAt("c1", time.Now().Add(-2*time.Hour)) - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, "app"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - } - registerAt("c2", time.Now().Add(-time.Minute)) - - // The machine: what it is sent is applied, and its report says what runs is starting. - listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}} - sequence := int64(0) - say := func(node, digest, state string) { - t.Helper() - sequence++ - health := map[string]any{} - for k, v := range stated { - health[k] = v - } - health["at"] = time.Now().UTC().Format(time.RFC3339Nano) - if state != "" && stated != nil { - resources := []any{} - for _, r := range stated["resources"].([]any) { - kept := map[string]any{} - for k, v := range r.(map[string]any) { - kept[k] = v - } - kept["state"], kept["reason"] = state, "" - resources = append(resources, kept) - } - health["resources"] = resources - } - said := map[string]any{"node": node, "applied": []string{"app.server"}, "declared": digest, - "report_sequence": sequence} - if stated != nil { - said["health"] = health - } - body, _ := json.Marshal(said) - var report link.Report - if err := json.Unmarshal(body, &report); err != nil { - t.Fatal(err) - } - if _, err := listener.Heard(ctx, report); err != nil { - t.Fatal(err) - } - } - var sent [][]string - last := map[string]string{} - n := 0 - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - sent = append(sent, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil { - return nil, err - } - last[node] = digest - say(node, digest, "starting") - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - gateWithin, gateEvery = time.Hour, 0 - t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) - - built := time.Now().UTC() - plan := inventory.Plan{ID: "plan-crashloop", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", - Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-2"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - - advancePlans(ctx, open) // the first machine is sent the new build, and starts it - if len(sent) == 0 || !slices.Equal(sent[0], []string{"anchor"}) { - t.Fatalf("sent %v, not the first machine first", sent) - } - advancePlans(ctx, open) // judged while it starts - // Its container exits at start, and the runtime restarts it: the machine says so, again and again. - for i := 0; i < 6 && len(sent) == 1; i++ { - say("anchor", last["anchor"], "") - advancePlans(ctx, open) - } - gateBound = -time.Second // and the bound passes - advancePlans(ctx, open) - - p, err := inv.PlanByID(ctx, "plan-crashloop") - if err != nil { - t.Fatal(err) - } - gate := p.Modules["app"].Gate - for _, names := range sent { - if slices.Contains(names, "laptop") { - t.Fatalf("the crash loop passed its gate on anchor and was sent to laptop: sent %v, the gate %+v", sent, gate) - } - } - if gate == nil || gate.Verdict != inventory.GateFailed || p.State != inventory.PlanFailed { - t.Fatalf("a crash-looping build did not fail its gate on the first machine: the plan is %s (%s), its gate %+v", - p.State, p.Note, gate) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { - t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit) - } -} - -func TestReplaySilentWebAppIsRaisedWithinTwoLooks(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - silent := []byte(`{"contract":2,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` + - `"kind":"container","target":"app-server","state":"unhealthy","reason":"http / on web: no answer within 5s",` + - `"since":"2026-10-07T00:00:00Z","streak":3,"check":"http"}]}`) - if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" { - raw, err := os.ReadFile(path) - if err != nil { - t.Fatalf("the engine's statement of the silent web application: %v", err) - } - silent = raw - } - var h link.Health - if err := json.Unmarshal(silent, &h); err != nil { - t.Fatal(err) - } - if h.Contract == 0 { - t.Fatal("the engine states nothing of the silent web application: it is older than the judging") - } - open1 := func() []conditions.Condition { - t.Helper() - list, err := conditionsFrom.Open(ctx) - if err != nil { - t.Fatal(err) - } - return list - } - for look := 1; look <= 2; look++ { - at := time.Now().Add(time.Duration(look) * time.Second) - h.At = at - if err := stateHealth(ctx, open.inventory, conditionsFrom, "anchor", h, at); err != nil { - t.Fatal(err) - } - raised := open1() - switch { - case look == 1 && len(raised) != 0: - t.Fatalf("one look raised %v", raised[0].Key) - case look == 2 && (len(raised) != 1 || raised[0].Key != "module.app.anchor.unhealthy"): - t.Fatalf("two looks in a row did not raise the module's condition: %v", raised) - case look == 2: - t.Logf("raised on the second look: %s", raised[0].Summary) - } - } -} - -func TestReplay296APlanStandingStillInItsTierIsNotSavedAgain(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := asksRecorded(t) - // app's key is not built, so the plan stands on its tier as it did while the build ran (see plan_clock_integration_test.go). - holdKeys(t, "app") - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3ad22356aa", - Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - advanceHeld(ctx, open) - recent, err := inv.RecentPlans(ctx, 1) - if err != nil || len(recent) != 1 || len(*asked) != 0 { - t.Fatalf("no plan waiting on its build: %v %v, asked %v", recent, err, *asked) - } - before := recent[0] - if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" || - !before.Modules["app"].Key || before.Modules["app"].KeyAt != "3ad22356aa" { - t.Fatalf("the plan is not waiting on its build: %+v", before) - } - for range 3 { - advanceHeld(ctx, open) - } - after, err := inv.PlanByID(ctx, before.ID) - if err != nil { - t.Fatal(err) - } - if after.Revision != before.Revision || !after.Updated.Equal(before.Updated) { - t.Fatalf("a plan that did not move was saved again: revision %d → %d, saved %s → %s", - before.Revision, after.Revision, before.Updated, after.Updated) - } -} - -func TestReplay299AHolderOfASeatWhoseVerbsAreAllOptionalIsNotSilent(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - catchingUp := catalogue.Seat{Name: "node-catching-up", Scope: catalogue.ScopeNode, Decision: "novox/hq issue 299", - Serves: []catalogue.Verb{{Name: "first", Optional: true}, {Name: "second", Optional: true}}} - if _, err := inv.SeedSeats(ctx, append(catalogue.DefaultSeats(), catchingUp)); err != nil { - t.Fatal(err) - } - rows, err := inv.Seats(ctx) - if err != nil { - t.Fatal(err) - } - // The working set as the serving controller holds it: the store's rows, the compiled seats' marks. - // A seat the compiled set does not know keeps its mark only as defined here. - for i := range rows { - if rows[i].Name == catchingUp.Name { - rows[i].Serves = catchingUp.Serves - } - } - catalogue.UseSeats(rows) - t.Cleanup(func() { catalogue.UseSeats(catalogue.DefaultSeats()) }) - - seats := []string{catchingUp.Name} - if uplink, ok := catalogue.SeatNamed("node-uplink"); ok && len(uplink.Serves) > 0 { - allOptional := true - for _, v := range uplink.Serves { - allOptional = allOptional && v.Optional - } - if allOptional { - seats = append(seats, "node-uplink") - } else { - t.Logf("node-uplink's verbs are required at this commit; the live seat is not replayed, its kind is") - } - } - - register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1"}) - now := time.Now() - var heard []machineFacts - for _, node := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, node, "networkmanager"); err != nil { - t.Fatal(err) - } - for _, seat := range seats { - if err := inv.HoldSeat(ctx, seat, catalogue.ScopeNode, node, "networkmanager"); err != nil { - t.Fatal(err) - } - } - heard = append(heard, machineFacts{name: node, lastHeard: now}) - } - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - d := &doctor{open: open, js: js, watchdogs: &watchdogs{last: &signalFacts{now: now, machines: heard}}} - - found, err := probeHolders(ctx, d) - if err != nil { - t.Fatal(err) - } - for _, o := range found { - for _, seat := range seats { - if strings.HasPrefix(o.ID, seat+".") { - t.Errorf("a holder serving none of a seat's optional verbs was said silent: %s (%s)", o.Summary, o.Said) - } - } - } -} - -func TestReplay301APersonsPushOfAHeldRecordedBuildIsNoRepair(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - start := time.Now().Add(-time.Hour) - image := "registry.invalid:5000/resolver/server@sha256:" + strings.Repeat("e", 64) - if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, aContainerBuild(t, "resolver", "c1111111", catalogue.PolicyRecord, start, - map[string][2]string{"server": {image, ""}}))); err != nil { - t.Fatal(err) - } - for _, node := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, node, "resolver"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, node), "d-"+node, map[string]string{"resolver": "c1111111"}); err != nil { - t.Fatal(err) - } - } - // The merge: the resolver's new build, registered and held for a person's word. - if _, _, err := takeIn(ctx, inv, aContainerBuild(t, "resolver", "c2222222", catalogue.PolicyRecord, - start.Add(time.Minute), map[string][2]string{"server": {image, "runtime"}})); err != nil { - t.Fatal(err) - } - - conn := onATestBus(t) - // The operator's yes, machine by machine, as the seat's push records it before it sends. - for _, node := range []string{"anchor", "laptop"} { - set := flag.NewFlagSet("push", flag.ContinueOnError) - why := addHandActFlags(set) - if err := set.Parse([]string{"--why", "operator's yes: the resolver's new build on " + node, - "--cause", "split-dns"}); err != nil { - t.Fatal(err) - } - why.record(ctx, "push", []string{node}) - } - acts, err := link.HandActs(ctx, conn, start) - if err != nil { - t.Fatal(err) - } - if len(acts) != 2 { - t.Fatalf("the two pushes were recorded as %d act(s): %+v", len(acts), acts) - } - f := calm(time.Now()) - f.handActs = acts - for _, o := range watchHandActs(f) { - if strings.Contains(o.ID, "split-dns") { - t.Fatalf("a person's push of a held recorded build was counted as a repair: %s — %s", o.ID, o.Summary) - } - } -} diff --git a/cmd/mesh-controller/replays_test.go b/cmd/mesh-controller/replays_test.go deleted file mode 100644 index b9c93fe2..00000000 --- a/cmd/mesh-controller/replays_test.go +++ /dev/null @@ -1,108 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// The replays of the controller's incidents (novox/hq to-be 45 §9, M9): each a scripted replay of what -// happened, asserting the rule's outcome rather than the fix's mechanism, so it can be run against the -// commit before the fix and fail there, and against the fix and pass. **Written only with what the -// controller had before each fix** — the stores, register, assign, planFor, declarationFor — so the -// prover (mesh-lab replays/cmd/prove) can lay this file over the older commit and run it there. -// -// Registered in mesh-lab's replays/register.go with the fix each one proves; run by this repository's -// merge check on every pull request with the rest of the suite. - -// **R263 — a consumer's identity never refuses its provider's machine.** On 2026-10-06 the network -// manager came to require the mesh's resolver; on a machine whose name made its identity 23 to 26 -// characters against the one global bound of 20, the anchor — the resolver's holder, carrying every -// consumer's grant — could not compose, and no push to it could go through. The outcome asserted: the -// provider's machine composes whatever its consumers are called, and a provision that mints no -// credential holds no consumer to a key's length. - -// **R273 — a binding to a consumer's data does not move by itself.** On 2026-10-05 a rule written for -// the resolver re-bound every database consumer on the home server — which runs its own store, beside -// its applications' data — to the store seat's holder on the control node, which made each a new empty -// database; five applications ran on empty data for twenty hours. The outcome asserted: a consumer on a -// machine running its own store stays bound to it while another machine holds the store's seat; the -// resolver, which every holder answers alike, follows its seat. - -// **R-crashloop — a module that crash-loops after it applied fails its gate on its first machine** (novox/hq -// ADR 0240; research 032 §6). On the home server the agent server restarted about a hundred times while the -// mesh read it applied, its tools served and no condition raised — the gate judged a module by what the -// mesh saw from outside, and nothing looked at what it ran. The outcome asserted: a build whose container -// exits at start never passes its gate on the first machine, is put back there at the bound, and never -// reaches the second. -// -// The machine is heard as a node-engine says it: its report of the apply, then the same account said again -// later, each carrying what the engine states of what it runs — `starting` as the apply ends, then the -// crash loop. What it states of the crash loop is MESH_REPLAY_STATEMENT when the lab's replay ran the -// engine against a real container (mesh-lab replays, R-crashloop), and otherwise what the engine said of -// one, kept below. Heard as bytes, so an older controller reads them as it reads any report — this file is -// written only with what the controller had before the judging, for the prover to lay over that commit. - -// **R145 — a web application that accepts TCP and answers nothing is raised within two looks** (novox/hq -// ADR 0240 rule 4 and Phase B, issue 145). For eleven hours a web application's port was open and its -// program ran while every request hung, and the mesh said its machine was healthy; a person found it. -// Liveness cannot see it and a TCP check cannot either: the port is open. The module's declared HTTP check -// can. The engine's half (mesh-host internal/liveness TestReplaySilentWebAppIsSaidUnhealthy) states what it -// found looking at such a program; here the controller hears that statement on two looks in a row and -// raises the module's condition — the second, never the first. `null` is an engine older than the -// readiness check: it states the program alive, and nothing is raised. - -// **R296 — a plan's clock counts from its tier, not from its last save.** On 2026-10-07 a plan for a -// merge to the catalogue, its one module's build asked at 19:48:41 and queued at the build seat for -// minutes, read "tier 1 of 1, building for 1s", then 4s, then 9s: the line counted from the plan's last -// save, and every advance — on the 30-second tick and after every build outcome on the mesh — saved it -// whether or not anything moved. A clock reset by each save is never LATE, so `plans` and `status` could -// not say a plan was stuck. The outcome asserted: a plan in its tier for forty minutes, saved seconds ago, -// says forty minutes and LATE. -func TestReplay296APlansLineCountsFromWhenItEnteredItsTier(t *testing.T) { - now := time.Date(2026, 10, 7, 20, 30, 0, 0, time.UTC) - entered := time.Date(2026, 10, 7, 19, 48, 41, 0, time.UTC) - asked := entered - p := inventory.Plan{ID: "plan-296", Repository: "novox/mesh-catalog", Branch: "main", Commit: "3ad22356", - State: inventory.PlanBuilding, Created: entered, TierEntered: entered, Updated: now.Add(-4 * time.Second), - Tiers: [][]string{{"app"}}, Modules: map[string]*inventory.PlanModule{"app": {State: "asked", AskedAt: &asked}}} - line := planLine(p, now) - if !strings.Contains(line, "for 41m19s") || !strings.Contains(line, "LATE") { - t.Fatalf("a plan in its tier since 19:48:41, read at 20:30:00 and saved 4s before, reads %q", line) - } -} - -// **R296, the save.** The same live case from the store's side: a plan whose only build was asked and -// still waits is advanced as the tick and the mesh's build outcomes advance it, and is not written again — -// its revision and its last save stay where its last change left them. Before the fix each advance saved -// it, and its revision went from 3 to 6. - -// **R299 — a holder of a seat whose verbs are all optional is not silent.** On 2026-10-07 the -// node-uplink seat gained its first verbs, `resolvers` and `links`, both optional while its holders catch -// up (ADR 0246, step 1); its holders served neither, so they registered nothing on the bus's discovery for -// it, and the self-check's D3 raised `seat.node-uplink..silent` on every machine of the mesh, -// which the healer then acted on. The outcome asserted: through D3 itself — the seats read back from the -// store as the controller reads them, a holder on record on every machine heard from, and nothing -// answering the discovery — no holder of a seat whose verbs are all optional is said silent. -// -// The live seat is replayed while its verbs are all optional; beside it, a seat whose verbs are all -// optional by definition, so the outcome is still held once node-uplink's verbs are required (step 3). - -// **R300 — a merge that adds a module builds it.** On 2026-10-07 a pull request to the catalogue added -// one module, systemd-resolved; its delivery plan said "builds new: modules/systemd-resolved, sent -// nowhere", and at the merge the controller said it "changed nothing any module the mesh holds is built -// from" and built nothing, so the module was never registered and `assign` refused it until a person -// built it by hand. The outcome asserted: the merge asks the build seat for the new module's directory, at -// the branch merged into, from the repository as the mesh spells it, and opens no plan for it. - -// **R301 — a person's push of a held recorded build is no repair.** On 2026-10-07 the operator approved, -// machine by machine, new builds of modules whose upgrade policy records rather than rolls out — the -// resolver, the FortiClient adapter, the network managers, the packet filter, mail. A recorded build -// moves only by a person's push (ADR 0242), so each was `push ` through the seat with the -// operator's yes as its why and a word for the change as its cause. The hand-act log counted every one -// as a repair, and S15 raised `mesh.hand-acts.split-dns.healer-wanted` ("repaired by hand 4 times in 14 -// days … a healer is wanted"), and the same for `words` and `uplink-verbs`. The outcome asserted: a push -// that carries only held recorded builds, recorded as the push records it, wants no healer however -// often the person gives that word — through S15 itself, reading the log as the watchdog reads it. diff --git a/cmd/mesh-controller/rest_at_once_integration_test.go b/cmd/mesh-controller/rest_at_once_integration_test.go deleted file mode 100644 index 0c10c878..00000000 --- a/cmd/mesh-controller/rest_at_once_integration_test.go +++ /dev/null @@ -1,293 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "slices" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -func aBacklogOnThree(t *testing.T) *backlogMesh { - t.Helper() - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - if _, err := inv.AddNode(ctx, "desk"); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "desk", "app"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, b.open, "desk"), "d-desk", map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - wasHeard := releaseHeard - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { - return map[string]bool{"anchor": true, "laptop": true, "desk": true}, nil - } - t.Cleanup(func() { releaseHeard = wasHeard }) - return b -} - -func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklogOnThree(t) - ctx := t.Context() - said := events(t) - gateEvery = 0 - for i := 0; i < 12; i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop") - if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) { - t.Fatalf("sends and judgings: %v", *said) - } - if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") { - t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said) - } - if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) - } -} - -func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklogOnThree(t) - ctx := t.Context() - inv := b.open.inventory - gateEvery = 0 - b.broken["desk"] = true // desk fails what it is sent in the rest's step - advancePlans(ctx, b.open) - for i := 0; i < 12 && b.release(t).Open(); i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) { - t.Fatalf("laptop was never sent: %v", b.sent) - } - if !slices.Contains(p.Release.Done, "laptop") { - t.Fatalf("laptop has no verdict of its own: %+v", p.Release) - } - // late waited on laptop alone and was healthy there: its pass is kept, whatever desk did. - if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) - } - // app failed on desk: marked, and put back wherever it was sent (ADR 0236). - if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed { - t.Fatal("app's build is not marked failed") - } - if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" { - t.Fatalf("desk was not put back: %v", sent) - } -} - -func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once - release := b.release(t) - judging := release.Release.Gate - if judging == nil || judging.Pending || judging.Passes == 0 { - t.Fatalf("the release is not judging anchor: %+v", judging) - } - built := time.Now().UTC() - plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", - Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", - Build: "build-app-c2"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - if len(b.sent) != 1 { - t.Fatalf("anchor was sent again what it runs: %v", b.sent) - } - p, err := inv.PlanByID(ctx, "plan-app") - if err != nil { - t.Fatal(err) - } - g := p.Modules["app"].Gate - if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID { - t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging) - } - if p.Modules["app"].Previous != "c1" { - t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous) - } -} - -func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - // The old controller sends anchor app c2 and is replaced before it keeps the send. - made := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - if _, err := made(ctx, open, names); err != nil { - return nil, err - } - return nil, context.Canceled - } - advancePlans(ctx, g.open) - sendRollout = made - if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil { - t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent) - } - last, _, err := g.open.inventory.SentTo(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - advancePlans(ctx, g.open) - if len(g.sent) != 1 { - t.Fatalf("the send was made again: %v", g.sent) - } - s := g.plan(t).Modules["app"] - if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) { - t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate) - } - // What anchor ran before was kept before the send, so a failure still puts it back (§5). - if s.Previous != "c1" || s.Gate.From != "c1" { - t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From) - } -} - -func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - p, err := releaseBacklog(ctx, b.open, "") - if err != nil || p == nil { - t.Fatalf("no release opened: %v", err) - } - if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending { - t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) - } - if err := inv.SavePlan(ctx, p); err != nil { - t.Fatal(err) - } - // The send is made, and the controller is replaced before it keeps it. - if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - if len(b.sent) != 1 { - t.Fatalf("anchor was sent again: %v", b.sent) - } - g := b.release(t).Release.Gate - if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { - t.Fatalf("anchor is not judged on the send made: %+v", g) - } -} - -func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) { - withTheGate(t) - b := aBacklogOnThree(t) - ctx := t.Context() - gateEvery = 0 - b.broken["anchor"] = true - advancePlans(ctx, b.open) - for i := 0; i < 6 && b.release(t).Open(); i++ { - advancePlans(ctx, b.open) - } - if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - for _, s := range b.sent { - if !slices.Equal(s, []string{"anchor"}) { - t.Fatalf("sent %v: the rest after a failed first machine", b.sent) - } - } -} - -func TestTheControlNodeIsSentLastInAStep(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklogOnThree(t) - ctx := t.Context() - register(t, b.open, catalogue.Manifest{Module: catalogue.ControllerSeatName, Version: "1"}) - if _, err := b.open.inventory.Assign(ctx, "anchor", catalogue.ControllerSeatName); err != nil { - t.Fatal(err) - } - said := events(t) - gateEvery = 0 - for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ { - advancePlans(ctx, b.open) - } - if p := b.release(t); !slices.Equal(p.Release.Order, []string{"desk", "laptop", "anchor"}) { - t.Fatalf("the order is %v", p.Release.Order) - } - laptop, anchor := slices.Index(*said, "send laptop"), slices.Index(*said, "send anchor") - if laptop < 0 || anchor < laptop || slices.Contains((*said)[laptop:anchor], "judge") { - t.Fatalf("the control node is not sent last in the rest's step: %v", *said) - } -} - -func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - reporting := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing - b.sent = append(b.sent, names) - current, _ := open.inventory.CurrentBuilds(ctx) - return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep", - map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit}) - } - return reporting(ctx, open, names) - } - t.Cleanup(func() { sendRollout = reporting }) - gateEvery = 0 - for i := 0; i < 8; i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - g := p.Release.Gates["laptop"] - if !p.Open() || g == nil || g.Verdict != "" || g.Silent { - t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g) - } - since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) - g.Since = &since - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - p = b.release(t) - if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed { - t.Fatal("a silent machine's build was marked failed") - } - if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { - t.Fatalf("laptop was put back: %v", sent) - } - obs, err := silentObservation(ctx, inv) - if err != nil { - t.Fatal(err) - } - if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) { - t.Fatalf("the silence is not said: %+v", obs) - } -} diff --git a/cmd/mesh-controller/rest_at_once_test.go b/cmd/mesh-controller/rest_at_once_test.go deleted file mode 100644 index 053ab931..00000000 --- a/cmd/mesh-controller/rest_at_once_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package main - -import ( - "context" - "slices" - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq ADR 0353, kept behind gateFirstMachine for walks a controller before ADR 0360 started: after its first machine passes, a build goes to every other machine at once, each judged by -// its own gate and a failing one put back by itself; and a gate's readings outlive the controller that took them — -// a machine already sent what a send is for is not sent again, and its judging goes on from what was read. -// -// Seen 2026-10-14: release-1791930194184991192 sent ace, g14, shanks and the control node one after another in -// eleven minutes; the new controller its last send started sent the control node again the controller build it -// had just applied, and judged it from its first reading. - -// aBacklogOnThree is the backlog fixture with a third machine, desk, running app at c1 like the others. - -// events records every send and every judging, in order. -func events(t *testing.T) *[]string { - t.Helper() - var said []string - wasSend, wasGather := sendRollout, gatherGateFacts - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - said = append(said, "send "+strings.Join(names, ",")) - return wasSend(ctx, open, names) - } - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - said = append(said, "judge") - return wasGather(ctx, open, component) - } - t.Cleanup(func() { sendRollout, gatherGateFacts = wasSend, wasGather }) - return &said -} - -// Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second -// was judged to its verdict before the third was sent. - -// A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own -// verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent. - -// A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that -// gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading. - -// A walk's first send that was made and not kept — the controller replaced between the two — is not made again -// by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6). - -// A release's send kept before it was made, and made, is judged by the controller that takes over without being -// made again (ADR 0353 §8). - -// A build no gate has passed goes to one machine of a step, its first; a machine that would be its second waits -// for the next step, and one whose every build passed goes now (ADR 0353 §2). -func TestABuildThatPassedNoGateGoesToOneMachineOfAStep(t *testing.T) { - withTheGate(t) - waiting := map[string][]inventory.CarriedMove{ - "desk": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}}, - "laptop": {{Module: "app", To: "c2"}, {Module: "new", To: "n1"}}, - "shanks": {{Module: "app", To: "c2"}}, - "novox": {{Module: "app", To: "c2"}, {Module: "ctl", To: "k1"}}, - } - passed := func(module, commit string) bool { return module == "app" && commit == "c2" } - never := func(string) bool { return false } - step := restStep([]string{"desk", "laptop", "shanks", "novox"}, waiting, passed, never) - if !slices.Equal(step, []string{"desk", "shanks", "novox"}) { - t.Fatalf("the step is %v: laptop would be new's second machine before it passed", step) - } - if step := restStep([]string{"laptop"}, waiting, func(string, string) bool { return true }, never); !slices.Equal(step, []string{"laptop"}) { - t.Fatalf("a machine whose every build passed waits: %v", step) - } -} - -// The node-engine and the mesh's resolver go to the rest one machine at a time, even passed (ADR 0353 §9): a step -// carries either to one machine only, while what else passed goes everywhere at once. -func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) { - withTheGate(t) - waiting := map[string][]inventory.CarriedMove{ - "desk": {{Module: hostModule, To: "h2"}}, - "laptop": {{Module: hostModule, To: "h2"}, {Module: "app", To: "c2"}}, - "shanks": {{Module: "app", To: "c2"}}, - "novox": {{Module: "dns", To: "d2"}}, - "ace": {{Module: "dns", To: "d2"}}, - } - resolver := catalogue.Manifest{Module: "dns", Claims: []catalogue.Claim{{Name: resolverSeat}}} - alone := func(m string) bool { - if m == "dns" { - return oneAtATime(m, resolver) - } - return oneAtATime(m, catalogue.Manifest{Module: m}) - } - step := restStep([]string{"desk", "laptop", "shanks", "novox", "ace"}, waiting, - func(string, string) bool { return true }, alone) - if !slices.Equal(step, []string{"desk", "shanks", "novox"}) { - t.Fatalf("the step is %v: the node-engine and the resolver each reach one machine of it", step) - } - if oneAtATime("app", catalogue.Manifest{Module: "app"}) { - t.Fatal("a leaf module is held to one machine at a time") - } -} - -// A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent. - -// The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is -// ordered after every other. - -// A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put -// back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4). diff --git a/cmd/mesh-controller/retry_words_test.go b/cmd/mesh-controller/retry_words_test.go deleted file mode 100644 index 4796ff52..00000000 --- a/cmd/mesh-controller/retry_words_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package main - -import ( - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// Issue 1403: a retry whose every module still waits for the bus asked nothing, and does not say "asked" with an empty list. -func TestARetryWhoseModulesAllWaitForTheBusSaysNothingWasAsked(t *testing.T) { - p := inventory.Plan{ID: "plan-9", Tier: 1, Tiers: [][]string{{"a"}, {"b", "c"}}} - note, said := retryWords(p, []string{"b", "c"}, nil, []string{"b", "c"}) - for _, text := range []string{note, said} { - if strings.Contains(text, "asked ;") || strings.Contains(text, "asked ") || strings.HasSuffix(strings.Split(text, ";")[0], "asked") { - t.Errorf("a retry that asked nothing says %q", text) - } - if !strings.Contains(text, "b, c not asked yet: the bus did not answer") { - t.Errorf("it does not say what waits for the bus: %q", text) - } - } - if !strings.Contains(said, "nothing asked yet") || !strings.Contains(note, "nothing could be asked yet") { - t.Errorf("it does not say that nothing was asked: %q / %q", said, note) - } -} - -func TestARetryThatAskedSomeAndLeftSomeForTheBusSaysBoth(t *testing.T) { - p := inventory.Plan{ID: "plan-9", Tier: 0, Tiers: [][]string{{"a", "b"}}} - note, said := retryWords(p, []string{"a", "b"}, []string{"a as build-1"}, []string{"b"}) - if !strings.Contains(note, "a asked again") || strings.Contains(note, "a, b asked again") { - t.Errorf("the note names b as asked again: %q", note) - } - if !strings.Contains(said, "asked a as build-1;") || !strings.Contains(said, "b not asked yet: the bus did not answer") { - t.Errorf("the answer says %q", said) - } -} - -func TestARetryThatAskedEverythingSaysSoAsBefore(t *testing.T) { - p := inventory.Plan{ID: "plan-9", Tier: 0, Tiers: [][]string{{"a"}}} - note, said := retryWords(p, []string{"a"}, []string{"a as build-1"}, nil) - if note != "tier 0 retried by hand: a asked again" || - said != "plan-9 retried at tier 0 of 1: asked a as build-1; the plan goes on from there as any plan does" { - t.Errorf("a retry that asked everything says %q / %q", note, said) - } -} diff --git a/cmd/mesh-controller/rollout_events_test.go b/cmd/mesh-controller/rollout_events_test.go deleted file mode 100644 index 5818cff8..00000000 --- a/cmd/mesh-controller/rollout_events_test.go +++ /dev/null @@ -1,385 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "fmt" - "strings" - "sync" - "testing" - "time" - - "github.com/nats-io/nats.go" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -// Acceptance of the step (novox/hq ADR 0406 §5, §6, acceptance 7; issue 974), live, with the store and a real bus and no -// ticker: a rollout of two tiers advances on the events alone — the plan kept, the machines' reports, a silent key's -// expiry — a machine that reports in time is never named silent and one that never reports is named silent when its -// key expires, and a controller replaced in the middle of it resumes with no tier asked twice and none skipped. -func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - - // A quarter of an hour of silence is two seconds here. - wasSilent := applySilentAfter - // Ten seconds here: long enough that laptop's key outlives the controller's replacement under a loaded run. anchor's - // report never races it: anchor answers tier 1 inside its send (reportOnSend below). - applySilentAfter = 10 * time.Second - t.Cleanup(func() { applySilentAfter = wasSilent }) - - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - legacy, err := js.Conn().JetStream() - if err != nil { - t.Fatal(err) - } - checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy} - t.Cleanup(func() { checkEvents = nil }) - sentHeard := make(chan *nats.Msg, 32) - sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeySent), sentHeard) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - - // Every save of a plan is kept, said and looked at by the serving controller's own hook. - saved := &movesSaid{revs: map[string][]int64{}} - wasSaved := inventory.PlanSaved - serving := servingPlanSaved(ctx, checkEvents, open) - inventory.PlanSaved = func(i *inventory.Inventory, p inventory.Plan) { saved.hear(i, p); serving(i, p) } - t.Cleanup(func() { inventory.PlanSaved = wasSaved }) - - // The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept. - var mu sync.Mutex - var sends [][]string - asked := asksRecorded(t) // each module taken from a done key as the walk reaches it, said as repository#build id - digests := 0 - var reportOnSend func(node string) // set below, once a report can be made - anchorReported := make(chan error, 1) - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - mu.Lock() - defer mu.Unlock() - sends = append(sends, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - builds := map[string]string{} - for _, m := range []string{"a", "b"} { - builds[m] = current[m].Commit - } - for _, node := range names { - digests++ - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), fmt.Sprintf("d-%s-%d", node, digests), builds); err != nil { - return nil, err - } - } - // anchor answers tier 1 as soon as it is sent (issue 1938): its report is in before the silence bound can run, by - // order and not by a race with the bound, which twice named it silent under a loaded run. laptop never reports. - if len(sends) == 2 && reportOnSend != nil { - for _, node := range names { - if node == "anchor" { - go reportOnSend(node) - } - } - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - - register := func(m, commit string, asked time.Time) { - t.Helper() - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit}, - inventory.Source{Repository: "novox/" + m, Seat: "git", Path: "modules/" + m, BuiltFrom: commit, Head: commit, - Asked: asked}); err != nil { - t.Fatal(err) - } - } - record := func(id, m, commit string, at time.Time) { - t.Helper() - b := inventory.Build{ID: id, Module: m, Commit: commit, Repository: "novox/" + m, Path: "modules/" + m, Asked: at, At: at} - b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit}) - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - old := time.Now().Add(-2 * time.Hour) - for _, m := range []string{"a", "b"} { - record("build-"+m+"-1", m, "c1", old) - register(m, "c1", old) - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, m); err != nil { - t.Fatal(err) - } - } - } - for _, n := range []string{"anchor", "laptop"} { - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"a": "c1", "b": "c1"}); err != nil { - t.Fatal(err) - } - } - record("build-a-2", "a", "c2", time.Now().Add(-time.Minute)) - register("a", "c2", time.Now().Add(-time.Minute)) - - // A machine's report on what it was sent, heard as the controller hears it. - listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}} - seq := int64(0) - reports := func(node string) { - t.Helper() - last, found, err := inv.SentTo(ctx, node) - if err != nil || !found { - t.Fatalf("%s was sent nothing: %v", node, err) - } - seq++ - body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest, - "report_sequence": seq}) - var report link.Report - if err := json.Unmarshal(body, &report); err != nil { - t.Fatal(err) - } - if _, err := listener.Heard(ctx, report); err != nil { - t.Fatal(err) - } - } - // tell is a report heard as the controller hears it, for a goroutine: its error is handed back, not fatal. - tell := func(node string) error { - last, found, err := inv.SentTo(ctx, node) - if err != nil || !found { - return fmt.Errorf("%s was sent nothing: %v", node, err) - } - mu.Lock() - seq++ - n := seq - mu.Unlock() - body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest, - "report_sequence": n}) - var report link.Report - if err := json.Unmarshal(body, &report); err != nil { - return err - } - _, err = listener.Heard(ctx, report) - return err - } - reportOnSend = func(node string) { anchorReported <- tell(node) } - heldKeys := func() map[silentWait]time.Time { - c := silentClockOf() - if c == nil { - return nil - } - held, err := c.Held(ctx) - if err != nil { - t.Fatal(err) - } - return held - } - plan := func() inventory.Plan { - t.Helper() - p, err := inv.PlanByID(ctx, "plan-974") - if err != nil { - t.Fatal(err) - } - return p - } - - // Tier 1's module b has its key unbuilt while the first controller runs, so the plan stands at tier 1 as it did while - // its build ran; with the key done at once, the same look would take it and send tier 1 as well. - heldB := holdKeys(t, "b") - // The first controller. Its plan has tier 0 built and tier 1 not asked: nothing but the save asks anything. - first, stopFirst := context.WithCancel(ctx) - defer stopFirst() - waitFirst := startRolloutEvents(first, open, js) - built := time.Now().UTC() - p := inventory.Plan{ID: "plan-974", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: built, - State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, - Modules: map[string]*inventory.PlanModule{ - "a": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-a-2"}, - "b": {}, - }} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - eventually(t, "tier 0 sent to both machines and tier 1 taken from its key, on the save alone", func() bool { - mu.Lock() - defer mu.Unlock() - // b is taken from its key; the walk carries a commit of novox/mesh-catalog only, and b is registered from - // novox/b, so the key is read at b's own trunk value and KeyAt stays empty (Plan.CommitOn names no other - // repository's commit). - b := plan().Modules["b"] - return len(sends) == 1 && len(*asked) == 0 && plan().Tier == 1 && b != nil && b.Key && b.KeyAt == "" - }) - eventually(t, "a key on the bus for each machine tier 0 was sent to", func() bool { return len(heldKeys()) == 2 }) - - // The controller is replaced: its looks and its watcher end. anchor reports meanwhile, to nobody. - stopFirst() - waitFirst() - time.Sleep(300 * time.Millisecond) - reports("anchor") - - heldB.Store(false) // b's key is built by the time the new controller looks - second, stopSecond := context.WithCancel(ctx) - defer stopSecond() - waitSecond := startRolloutEvents(second, open, js) - // Ends with the test before the cleanups restore what the look reads, which run after it (issue 1338). - t.Cleanup(func() { stopSecond(); waitSecond() }) - // Its first look finds anchor reported, so its key goes; laptop's stays, and is the new controller's to hear expire. - eventually(t, "the new controller's first look deletes the key of the machine that reported", func() bool { - h := heldKeys() - _, anchor := h[silentWait{Plan: "plan-974", Tier: 0, Node: "anchor"}] - return !anchor - }) - - // tier 1's module is taken from its key by the new controller's look. - eventually(t, "tier 1 sent to both machines", func() bool { - mu.Lock() - defer mu.Unlock() - return len(sends) == 2 - }) - // anchor reported on tier 1 inside its send. - select { - case err := <-anchorReported: - if err != nil { - t.Fatalf("anchor's report on tier 1: %v", err) - } - case <-time.After(15 * time.Second): - t.Fatal("anchor never reported on tier 1") - } - - // laptop never reports: its key expires and the new controller names it silent, which ends the rollout. - eventually(t, "the rollout ended, naming laptop silent and not anchor", func() bool { - p := plan() - return p.State == inventory.PlanDone - }) - p = plan() - if !strings.Contains(p.Note, "laptop") || strings.Contains(p.Note, "anchor") || !strings.Contains(p.Note, "said nothing") { - t.Errorf("the plan ended with the note %q; want laptop named silent and not anchor", p.Note) - } - eventually(t, "no key left on the bus", func() bool { return len(heldKeys()) == 0 }) - - // No tier asked twice and none skipped: one ask for tier 1 (tier 0 was built when the plan was kept), each tier sent - // once to both machines, and said as sent once for each. - mu.Lock() - if len(*asked) != 1 || !strings.HasPrefix((*asked)[0], "novox/b#key-") { // b is registered from novox/b - t.Errorf("taken from keys = %v; want tier 1's module taken once", *asked) - } - if len(sends) != 2 || strings.Join(sends[0], ",") != "anchor,laptop" || strings.Join(sends[1], ",") != "anchor,laptop" { - t.Errorf("sends = %v; want both machines sent each tier once", sends) - } - mu.Unlock() - said := map[string]int{} - for deadline := time.After(5 * time.Second); len(said) < 4; { - select { - case m := <-sentHeard: - var body planSentBody - if err := json.Unmarshal(m.Data, &body); err != nil { - t.Fatal(err) - } - said[fmt.Sprintf("%d/%s", body.Tier, body.Node)]++ - case <-deadline: - t.Fatalf("sent was said for %v; want tiers 0 and 1 to both machines", said) - } - } - time.Sleep(300 * time.Millisecond) - for len(sentHeard) > 0 { - var body planSentBody - _ = json.Unmarshal((<-sentHeard).Data, &body) - said[fmt.Sprintf("%d/%s", body.Tier, body.Node)]++ - } - for k, n := range said { - if n != 1 { - t.Errorf("sent for %s was said %d times", k, n) - } - } - // And every move of the plan was said as plan-moved. - saved.noUnsaid(t, open) -} - -// Acceptance of the verb path (issue 974, review 1), live, with the store and a bus: a walk the delivery's owner lets -// go by the verb `deliver` after the operator's approval is asked its first tier with no other event. The verb runs as a -// child process, whose saves the serving controller does not hear; the handler asks the look as the verb ends. -func TestAWalkLetGoByTheDeliverVerbIsAskedWithNoOtherEvent(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - legacy, err := js.Conn().JetStream() - if err != nil { - t.Fatal(err) - } - checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy} - t.Cleanup(func() { checkEvents = nil }) - asked := asksRecorded(t) - register(t, open, catalogue.Manifest{Module: "app", Version: "1"}) - if _, err := assign(ctx, open, "anchor", "app"); err != nil { - t.Fatal(err) - } - cut := time.Now().UTC() - waiting := inventory.Plan{ID: "plan-974-deliver", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c8c8c8c8", - Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {}}, - Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}, - Times: &inventory.PlanTimes{Cut: &cut, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(ctx, &waiting); err != nil { - t.Fatal(err) - } - - // The serving controller, as serve starts it. - wasSaved := inventory.PlanSaved - inventory.PlanSaved = servingPlanSaved(ctx, checkEvents, open) - t.Cleanup(func() { inventory.PlanSaved = wasSaved }) - started, stop := context.WithCancel(ctx) - defer stop() - wait := startRolloutEvents(started, open, js) - time.Sleep(500 * time.Millisecond) // the first look: the walk waits, and is asked nothing - - // The verb as its child process runs it: the delivery command, whose saves are not the serving controller's. - wasRun := runTheVerb - runTheVerb = func(ctx context.Context, argv []string) (verbAnswer, error) { - was := inventory.PlanSaved - inventory.PlanSaved = func(*inventory.Inventory, inventory.Plan) {} - defer func() { inventory.PlanSaved = was }() - return verbAnswer{}, deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}) - } - t.Cleanup(func() { runTheVerb = wasRun }) - // Registered after every variable cleanup above, so it runs before them (last in, first out): the look at the plans - // has returned when checkEvents, inventory.PlanSaved and runTheVerb are restored (issue 1338). - t.Cleanup(func() { stop(); wait() }) - handlers, _, err := seatToolHandlers() - if err != nil { - t.Fatal(err) - } - if _, err := handlers["deliver"](ctx, []byte(`{"plan":"`+waiting.ID+`","why":"its turn"}`)); err != nil { - t.Fatal(err) - } - eventually(t, "the walk's first tier asked, with no event but the verb's end", func() bool { - p, err := open.inventory.PlanByID(ctx, waiting.ID) - return err == nil && p.Modules["app"] != nil && p.Modules["app"].Build != "" && len(*asked) == 1 - }) -} diff --git a/cmd/mesh-controller/rollout_first_test.go b/cmd/mesh-controller/rollout_first_test.go deleted file mode 100644 index 02996756..00000000 --- a/cmd/mesh-controller/rollout_first_test.go +++ /dev/null @@ -1,152 +0,0 @@ -package main - -import ( - "reflect" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq issue 249, ADR 0218: a plan rolls a module out to one machine first and the rest only -// once that machine has reported it applied; a module whose policy says together goes everywhere at -// once, as before. -func TestAPlanSendsOneMachineFirstAndTheRestAfterItsReport(t *testing.T) { - running := []string{"novox", "ace", "g14"} - sentAt := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) - now := sentAt.Add(5 * time.Minute) - bound := 30 * time.Minute - after := sentAt.Add(time.Minute) - next := func(s inventory.PlanModule, running []string, together bool, reports []inventory.Reported) rolloutStep { - return nextRollout(s, running, together, reports, now, bound) - } - - // Together: every machine at once. - if step := next(inventory.PlanModule{}, running, true, nil); !reflect.DeepEqual(step.send, running) || step.first { - t.Fatalf("a together policy did not send every machine at once: %+v", step) - } - - // Otherwise the first by name, alone, when none has reported lately. - step := next(inventory.PlanModule{}, running, false, nil) - if !step.first || !reflect.DeepEqual(step.send, []string{"ace"}) { - t.Fatalf("the first send was %+v, wanted ace alone", step) - } - - state := inventory.PlanModule{First: []string{"ace"}, FirstAt: &sentAt} - report := func(outcome string, current bool) []inventory.Reported { - return []inventory.Reported{{Node: "ace", At: &after, Outcome: outcome, Current: current}, - {Node: "g14", At: &after, Outcome: inventory.OutcomeApplied, Current: true}} - } - - // No report yet, or one about an older declaration than it was last sent: wait. - for what, reports := range map[string][]inventory.Reported{ - "no report": nil, - "a report about older": report(inventory.OutcomeApplied, false), - } { - step := next(state, running, false, reports) - if len(step.send) != 0 || step.waiting != "ace" || step.failed != "" { - t.Errorf("%s: %+v, wanted to wait for ace", what, step) - } - } - - // Applied what it was last sent: the rest, and only the rest. - step = next(state, running, false, report(inventory.OutcomeApplied, true)) - if step.first || !reflect.DeepEqual(step.send, []string{"novox", "g14"}) { - t.Fatalf("after ace applied it the plan sent %+v, wanted novox and g14", step) - } - - // Failed or refused: stop, the rest untouched. - for _, outcome := range []string{inventory.OutcomeFailed, inventory.OutcomeRefused} { - step := next(state, running, false, report(outcome, true)) - if len(step.send) != 0 || !strings.Contains(step.failed, "ace "+outcome) || - !reflect.DeepEqual(step.rest, []string{"novox", "g14"}) { - t.Errorf("a first machine that %s it: %+v", outcome, step) - } - } - - // The machine holding the bus went with the first send: the rest wait for it too, and it is - // not sent again. - both := inventory.PlanModule{First: []string{"novox", "ace"}, FirstAt: &sentAt} - half := report(inventory.OutcomeApplied, true) - if step := next(both, running, false, half); step.waiting != "novox" { - t.Fatalf("the plan did not wait for the bus's machine sent first: %+v", step) - } - all := append(half, inventory.Reported{Node: "novox", At: &after, Outcome: inventory.OutcomeApplied, Current: true}) - if step := next(both, running, false, all); !reflect.DeepEqual(step.send, []string{"g14"}) { - t.Fatalf("after both applied it the plan sent %+v, wanted g14 alone", step) - } - - // One machine, or none: nothing is waited for that cannot come. - if step := next(inventory.PlanModule{}, nil, false, nil); len(step.send) != 0 || step.first { - t.Fatalf("a module nothing runs was sent: %+v", step) - } - if step := next(state, []string{"ace"}, false, report(inventory.OutcomeApplied, true)); len(step.send) != 0 || step.waiting != "" { - t.Fatalf("a module on one machine waited for more: %+v", step) - } -} - -// ADR 0218 §2: a first machine that does not report within the bound stops the rollout there, -// naming the machine and the bound; the rest are left alone. -func TestAFirstMachineThatDoesNotReportStopsTheRollout(t *testing.T) { - sentAt := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) - state := inventory.PlanModule{First: []string{"ace"}, FirstAt: &sentAt} - step := nextRollout(state, []string{"ace", "g14"}, false, nil, sentAt.Add(31*time.Minute), 30*time.Minute) - if !strings.Contains(step.failed, "ace did not report it applied within 30m") || - !reflect.DeepEqual(step.rest, []string{"g14"}) || len(step.send) != 0 { - t.Fatalf("a silent first machine: %+v", step) - } -} - -// The first machine is the first by name among those heard from lately: a laptop that is away is -// not the one the rest wait on. When none has been heard from, the first by name. -func TestTheFirstMachineIsOneThatHasReportedLately(t *testing.T) { - now := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) - lately, long := now.Add(-time.Minute), now.Add(-3*time.Hour) - reports := []inventory.Reported{ - {Node: "ace", At: &long}, {Node: "g14", At: &lately}, {Node: "novox", At: &lately}, - } - step := nextRollout(inventory.PlanModule{}, []string{"novox", "ace", "g14"}, false, reports, now, 30*time.Minute) - if !step.first || !reflect.DeepEqual(step.send, []string{"g14"}) { - t.Fatalf("the first send was %+v, wanted g14, the first heard from lately", step) - } -} - -// An announced move of a module an open plan is still rolling out is left to the plan: sending it -// here as well put the bundle on every machine at once (novox/hq issue 249). -func TestAnAnnouncedMoveIsLeftToThePlanRollingItOut(t *testing.T) { - sent := time.Now() - plans := []inventory.Plan{ - {ID: "plan-done", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{"agent": {}}}, - {ID: "plan-1", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{ - "agent": {State: "built", First: []string{"ace"}, FirstAt: &sent}, "gitea": {State: "built", SentAt: &sent}}}, - } - if got := rolledOutByAPlan(plans, "agent"); got != "plan-1" { - t.Fatalf("a module the plan is rolling out was not left to it: %q", got) - } - for _, m := range []string{"gitea", "keycloak"} { - if got := rolledOutByAPlan(plans, m); got != "" { - t.Errorf("%s, which no plan will send, was left to %s", m, got) - } - } -} - -// A plan that ends without sending what it built says so, with the remedy; a module whose rollout -// stopped at its first machine is not among them. -func TestAnEndedPlanSaysWhatItBuiltAndNeverSent(t *testing.T) { - at := time.Now() - p := inventory.Plan{State: inventory.PlanFailed, Note: "closed by hand at tier 1", - Modules: map[string]*inventory.PlanModule{ - "agent": {State: "built"}, - "stopped": {State: "built", First: []string{"ace"}, FirstAt: &at}, - "sent": {State: "built", SentAt: &at}, - "notes": {State: "built"}, - "later": {}, - }} - rollsOut := func(m string) bool { return m != "notes" } - sayUnsent(&p, rollsOut) - sayUnsent(&p, rollsOut) - if p.Note != "closed by hand at tier 1; built and never sent: agent — `upgrade release-backlog --why ` releases them" { - t.Fatalf("the note reads %q", p.Note) - } -} diff --git a/cmd/mesh-controller/rollout_silent_bus.go b/cmd/mesh-controller/rollout_silent_bus.go deleted file mode 100644 index 28a46b9b..00000000 --- a/cmd/mesh-controller/rollout_silent_bus.go +++ /dev/null @@ -1,150 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "strconv" - "strings" - "time" - - "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/broker" -) - -// A rollout's wait for a machine is a key on the bus (novox/hq ADR 0406 §5, issue 974): each send to a machine puts -// a key `..` that lives as long as the machine may stay silent (ApplySilentAfter); the machine's -// report on that send deletes it, so only a missing report lets it expire; and the expiry, heard by a watcher, -// ends the wait for that machine, which the plan then names silent. No timer looks every so often whether a -// machine has reported. - -// silentWait is one machine's wait on one send of a plan's tier. -type silentWait struct { - Plan string - Tier int - Node string -} - -// silentKey is a wait's key. A plan id may hold a dot, so it is read from the right (parseSilentKey). -func silentKey(w silentWait) string { return w.Plan + "." + strconv.Itoa(w.Tier) + "." + w.Node } - -// parseSilentKey reads a key back: the node is the last token, the tier the one before, the plan what is left. -func parseSilentKey(key string) (silentWait, bool) { - last := strings.LastIndexByte(key, '.') - if last < 1 || last == len(key)-1 { - return silentWait{}, false - } - rest, node := key[:last], key[last+1:] - mid := strings.LastIndexByte(rest, '.') - if mid < 1 || mid == len(rest)-1 { - return silentWait{}, false - } - tier, err := strconv.Atoi(rest[mid+1:]) - if err != nil || tier < 0 { - return silentWait{}, false - } - return silentWait{Plan: rest[:mid], Tier: tier, Node: node}, true -} - -// silentClock is the bus's side of the wait for a machine's report. -type silentClock interface { - // Arm puts a wait's key, to live ttl: made again if the machine is sent again. The key holds when the send was. - Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error - // Held is every wait that has a key now, with when its send was. - Held(ctx context.Context) (map[silentWait]time.Time, error) - // Reported takes a wait's key away: the machine reported on the send, and nothing is to wake the controller. - Reported(ctx context.Context, w silentWait) error -} - -// busSilent is the clock on the controller's bucket. -type busSilent struct { - kv jetstream.KeyValue -} - -// newBusSilent opens the bucket and starts the watcher, which names each wait whose key expired until the context -// ends. Only expiry wakes it: the marker of an expiry is a purge, and a key a report deleted, or one Arm made again, -// leaves a delete (the vendored client reads the server's MaxAge marker as a purge, kv.go:971). -func newBusSilent(ctx context.Context, api jetstream.JetStream, silent func(silentWait)) (*busSilent, error) { - kv, err := api.KeyValue(ctx, broker.RolloutSilentBucket) - if err != nil { - return nil, fmt.Errorf("the rollout's silent keys' bucket: %w", err) - } - w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) - if err != nil { - return nil, fmt.Errorf("watching the rollout's silent keys: %w", err) - } - go func() { - defer w.Stop() - for { - select { - case <-ctx.Done(): - return - case e, ok := <-w.Updates(): - if !ok { - return - } - if e == nil || e.Operation() != jetstream.KeyValuePurge { - continue // a put, or a delete: a report's, or a send made again - } - if wait, ok := parseSilentKey(e.Key()); ok && silent != nil { - silent(wait) - } - } - } - }() - return &busSilent{kv: kv}, nil -} - -func (s *busSilent) Arm(ctx context.Context, w silentWait, ttl time.Duration, sent time.Time) error { - key := silentKey(w) - // Made again, not changed: a key's time to live is set when it is made. - if err := s.kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { - return err - } - _, err := s.kv.Create(ctx, key, []byte(sent.UTC().Format(time.RFC3339Nano)), jetstream.KeyTTL(leastSecond(ttl))) - return err -} - -func (s *busSilent) Reported(ctx context.Context, w silentWait) error { - if err := s.kv.Delete(ctx, silentKey(w)); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { - return err - } - return nil -} - -// leastSecond rounds a time to live up to a whole second, at least one: the server rounds a key's time down to -// whole seconds, so a fraction would close the key before the time it stands for. -func leastSecond(d time.Duration) time.Duration { - if d < time.Second { - return time.Second - } - return (d + time.Second - 1) / time.Second * time.Second -} - -func (s *busSilent) Held(ctx context.Context) (map[silentWait]time.Time, error) { - keys, err := s.kv.Keys(ctx) - if errors.Is(err, jetstream.ErrNoKeysFound) { - return nil, nil - } - if err != nil { - return nil, err - } - held := map[silentWait]time.Time{} - for _, key := range keys { - w, ok := parseSilentKey(key) - if !ok { - continue - } - e, err := s.kv.Get(ctx, key) - if err != nil { - if errors.Is(err, jetstream.ErrKeyNotFound) { - continue - } - return nil, err - } - at, _ := time.Parse(time.RFC3339Nano, string(e.Value())) - held[w] = at - } - return held, nil -} diff --git a/cmd/mesh-controller/rollout_silent_bus_integration_test.go b/cmd/mesh-controller/rollout_silent_bus_integration_test.go deleted file mode 100644 index 571c73fe..00000000 --- a/cmd/mesh-controller/rollout_silent_bus_integration_test.go +++ /dev/null @@ -1,92 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "testing" - "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/testbus" -) - -func silentOnABus(t *testing.T) (*busSilent, <-chan silentWait) { - t.Helper() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - api, err := jetstream.New(conn) - if err != nil { - t.Fatal(err) - } - heard := make(chan silentWait, 16) - ctx, stop := context.WithCancel(context.Background()) - t.Cleanup(stop) - s, err := newBusSilent(ctx, api, func(w silentWait) { heard <- w }) - if err != nil { - t.Fatal(err) - } - return s, heard -} - -func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) { // 1157/AC3: the send keys stay until release 6 (ADR 0433 §5) - s, heard := silentOnABus(t) - want := silentWait{Plan: "plan.7", Tier: 2, Node: "ace"} - if err := s.Arm(t.Context(), want, time.Second, time.Now()); err != nil { - t.Fatal(err) - } - if got, ok := hearsSilent(t, heard, 10*time.Second); !ok || got != want { - t.Fatalf("heard %+v (%v); want %+v", got, ok, want) - } -} - -func TestAReportDeletesTheKeyAndNoOneIsNamedSilent(t *testing.T) { - s, heard := silentOnABus(t) - ctx := t.Context() - w := silentWait{Plan: "plan-1", Tier: 0, Node: "g14"} - if err := s.Arm(ctx, w, time.Second, time.Now()); err != nil { - t.Fatal(err) - } - if err := s.Reported(ctx, w); err != nil { - t.Fatal(err) - } - if got, ok := hearsSilent(t, heard, 2500*time.Millisecond); ok { - t.Fatalf("%+v was named silent after it reported", got) - } - // Reporting twice, or on a send that put no key, is no error. - if err := s.Reported(ctx, w); err != nil { - t.Fatalf("a second report: %v", err) - } -} - -func TestASendMadeAgainPutsTheKeyAgain(t *testing.T) { - s, heard := silentOnABus(t) - ctx := t.Context() - w := silentWait{Plan: "plan-3", Tier: 1, Node: "novox"} - if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil { - t.Fatal(err) - } - time.Sleep(1200 * time.Millisecond) - if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil { - t.Fatal(err) - } - if got, ok := hearsSilent(t, heard, 1500*time.Millisecond); ok { - t.Fatalf("%+v expired at the first send's time, not the second's", got) - } - if got, ok := hearsSilent(t, heard, 5*time.Second); !ok || got != w { - t.Fatalf("heard %+v (%v); want %+v after the second send's time", got, ok, w) - } -} diff --git a/cmd/mesh-controller/rollout_silent_bus_test.go b/cmd/mesh-controller/rollout_silent_bus_test.go deleted file mode 100644 index aaf3aa85..00000000 --- a/cmd/mesh-controller/rollout_silent_bus_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import ( - "testing" - "time" -) - -// silentOnABus is the rollout's silent keys on a real bus, and the waits its watcher heard expire. - -func hearsSilent(t *testing.T, heard <-chan silentWait, within time.Duration) (silentWait, bool) { - t.Helper() - select { - case w := <-heard: - return w, true - case <-time.After(within): - return silentWait{}, false - } -} - -// A machine that never reports on its send is named silent when its key expires (issue 974, novox/hq ADR 0406 §5): -// the watcher names the plan, the tier and the machine, which a plan id or a machine name with a dot does not blur. - -// A machine that reports in time is never named silent: the report deletes the key, and a delete is not an expiry. - -// A send made again to a machine puts its key again with the full time, and only the last send's key can expire. - -func TestASilentKeyIsParsedFromTheRightOfTheNode(t *testing.T) { - for _, w := range []silentWait{{"plan-1", 0, "ace"}, {"a.b", 12, "g14"}} { - got, ok := parseSilentKey(silentKey(w)) - if !ok || got != w { - t.Errorf("%+v became %+v (%v)", w, got, ok) - } - } - for _, bad := range []string{"", "plan", "plan.x.ace", "plan..ace", ".1.ace"} { - if _, ok := parseSilentKey(bad); ok { - t.Errorf("%q was read as a wait", bad) - } - } -} diff --git a/cmd/mesh-controller/router_ids_integration_test.go b/cmd/mesh-controller/router_ids_integration_test.go deleted file mode 100644 index 2f238fe9..00000000 --- a/cmd/mesh-controller/router_ids_integration_test.go +++ /dev/null @@ -1,52 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "slices" - "testing" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/testbus" -) - -// Issue 1336: the catch-up lists the ids of an asker's asks in the router's bucket. The key lister stops its own watch when the last -// key is read, and a second stop answers "nats: invalid subscription"; read as a failure, it made every warrant unreadable on novox -// (2026-10-20, 110 gaps named, none planned). -func TestTheAskIDsOfAnAskerAreReadFromTheRoutersBucketWithoutAFalseFailure(t *testing.T) { - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - js, err := jetstream.New(conn) - if err != nil { - t.Fatal(err) - } - ctx := t.Context() - kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "router_asks_test"}) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = js.DeleteKeyValue(context.Background(), "router_asks_test") }) - for _, key := range []string{"mesh-delivery.merge-378-b9104454f9a5-1", "mesh-delivery.merge-377-aaaaaaaaaaaa-1", - "mesh-delivery-teller.merge-feat_r4b_sdk_users-ba59b03220f8-989afe0a3bcf", "keycloak.something"} { - if _, err := kv.Put(ctx, key, []byte(`{}`)); err != nil { - t.Fatal(err) - } - } - got, err := askIDsOf(ctx, kv, "mesh-delivery") - if err != nil { - t.Fatalf("the ids of an asker's asks could not be read: %v", err) - } - slices.Sort(got) - if want := []string{"merge-377-aaaaaaaaaaaa-1", "merge-378-b9104454f9a5-1"}; !slices.Equal(got, want) { - t.Fatalf("read %v, want %v: the teller's and another module's asks are not mesh-delivery's", got, want) - } - if none, err := askIDsOf(ctx, kv, "nobody"); err != nil || len(none) != 0 { - t.Fatalf("an asker with no asks read as %v, %v", none, err) - } -} diff --git a/cmd/mesh-controller/same_program.go b/cmd/mesh-controller/same_program.go deleted file mode 100644 index b044536e..00000000 --- a/cmd/mesh-controller/same_program.go +++ /dev/null @@ -1,155 +0,0 @@ -package main - -import ( - "context" - "fmt" - "slices" - "sort" - "strings" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// **A module moves only when its source changes** (novox/hq issue 721; from R4e by its fingerprint, ADR 0453 §6, ADR -// 0499; issue 1851, A9). A walk is cut without a module whose fingerprint at the commit the walk carries is the -// fingerprint of the build the mesh holds of it, and says so among the modules it left unchanged: a merge of a file no -// module reads moves nothing. A build the mesh holds of the old form (build-agent's `src1:`) has no fingerprint to -// compare: the module stays in the walk, and the rule of the standing build (equal digests, no move) judges it once its -// key is taken. - -// sameByFingerprint are the modules whose fingerprint now (`src2:`, the indexer's trunk value at the walk's commit) is -// the fingerprint of the build the mesh holds, each with why; one whose either is unknown, or of the old form, is not. -func sameByFingerprint(now, held map[string]string) map[string]string { - out := map[string]string{} - for m, fp := range now { - if strings.HasPrefix(fp, "src2:") && held[m] == fp { - out[m] = fmt.Sprintf("unchanged: its fingerprint %s is the build the mesh holds", short(fp)) - } - } - if len(out) == 0 { - return nil - } - return out -} - -// heldFingerprint is the source fingerprint of the build the mesh holds of a module: the newest build of the commit -// it is registered at. -func heldFingerprint(ctx context.Context, inv *inventory.Inventory, module string) (string, error) { - src, err := inv.SourceOf(ctx, module) - if err != nil { - return "", err - } - fps, err := inv.BuildSourceFingerprints(ctx, module) - if err != nil { - return "", err - } - for commit, fp := range fps { - if sameCommitID(commit, src.BuiltFrom) { - return fp, nil - } - } - return "", nil -} - -// leaveSameAsRunning takes from a walk being cut every module whose fingerprint is unchanged (sameByFingerprint), and -// every dependent in the walk only for one of those (withDependentsOf): what was taken, with why. **Anything not read -// leaves the walk as it was**, said. -func leaveSameAsRunning(ctx context.Context, inv *inventory.Inventory, plan *inventory.Plan, moved []string, - edges []inventory.Edge) map[string]string { - if len(plan.Modules) == 0 { - return nil - } - entries, err := inv.Catalogued(ctx) - if err != nil { - fmt.Printf(" the catalogue could not be read, so every module moves: %v\n", err) - // empty-on-error: said above; nothing left out of the walk is the safe answer, every module moves as before - return nil - } - keys, closeKeys, err := openPlanKeys() - if err != nil { - fmt.Printf(" the indexer's fingerprints could not be read, so every module moves: %v\n", err) - // empty-on-error: said above; nothing left out of the walk is the safe answer, every module moves as before - return nil - } - defer closeKeys() - byName := map[string]inventory.Entry{} - for _, e := range entries { - byName[e.Manifest.Module] = e - } - now, held := map[string]string{}, map[string]string{} - for m := range plan.Modules { - e, known := byName[m] - if !known { - continue - } - v, err := keys.ModuleAt(ctx, m, plan.CommitOn(e.Source.Repository, followedBranch(e.Source.Ref))) - if err != nil || v.FP2 == "" { - continue // not written yet, or none: the module moves, and its key is waited for in the walk - } - fp, err := heldFingerprint(ctx, inv, m) - if err != nil { - continue - } - now[m], held[m] = v.FP2, fp - } - same := withDependentsOf(sameByFingerprint(now, held), plan, moved, edges) - if len(same) == 0 { - return nil - } - names := make([]string, 0, len(same)) - for n := range same { - names = append(names, n) - } - sort.Strings(names) - withhold(plan, names, nil) - for _, n := range names { - fmt.Printf(" %s left out: %s\n", n, same[n]) - } - return same -} - -// withDependentsOf adds to the modules found the same every module of the walk that is in it only as a dependent of -// one of them (novox/hq ADR 0267 rule 3): **a dependent is left out when its base comes out the same**, since the -// base it stands on stays the build the mesh holds. A module a merge moves itself, or a dependent of a base that -// still moves, stays. moved are the modules the walk's merges move; edges the graph the walk was planned along. -func withDependentsOf(same map[string]string, plan *inventory.Plan, moved []string, edges []inventory.Edge) map[string]string { - if len(same) == 0 { - return same - } - var still []string - for _, n := range moved { - if same[n] == "" { - still = append(still, n) - } - } - keep := map[string]bool{} - for _, n := range reachableFrom(still, edges) { - keep[n] = true - } - // Until nothing more is found: a dependent of a dependent left out is left out too. - for grew := true; grew; { - grew = false - for n := range plan.Modules { - if keep[n] || same[n] != "" { - continue - } - var bases []string - for _, e := range edges { - ordering := e.Kind == inventory.EdgeBuiltBy || e.Kind == inventory.EdgeWorkerOf || - e.Kind == inventory.EdgePackages || e.Kind == edgeChangeOrder - if !ordering && e.From == n && same[e.To] != "" && !slices.Contains(bases, e.To) { - bases = append(bases, e.To) - } - } - sort.Strings(bases) - if len(bases) == 0 { - // In the walk for no base found the same: whatever brought it in is not said here, so it moves. - continue - } - same[n] = "unchanged: it is in the walk only for " + strings.Join(bases, ", ") + - ", which came out the same as the build the mesh holds (ADR 0267 rule 3)" - grew = true - } - } - return same -} diff --git a/cmd/mesh-controller/same_program_integration_test.go b/cmd/mesh-controller/same_program_integration_test.go deleted file mode 100644 index 0a4f5131..00000000 --- a/cmd/mesh-controller/same_program_integration_test.go +++ /dev/null @@ -1,51 +0,0 @@ -//go:build integration - -package main - -import ( - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" -) - -// R4e A9 (ADR 0453 §6, ADR 0499; issue 1851), against the store and a real bus: a merge of a file no module reads moves -// nothing. Its module's fingerprint at the merge, as the indexer wrote it, is the fingerprint of the build the mesh holds, -// so the walk is cut done with nothing to walk, and its record names the module unchanged and why — the walk -// mesh-delivery reads as delivered, never stranded; no build is asked. -func TestAWalkWhoseEveryFingerprintIsUnchangedIsDone(t *testing.T) { - open := windowed(t) - endsItsLine(t) - noBuildAsked(t) - b := aBuilderOnTheBus(t) - ctx := t.Context() - inv := open.inventory - fp := fpOf('5') - if err := inv.RecordBuild(ctx, inventory.Build{ID: "key-held", Repository: "novox/one", Ref: "main", - Module: "one", Commit: "c0", On: "ace", SourceFingerprint: fp, - Made: []inventory.Artifact{{Name: "one", Kind: "image", Reference: "store/one@sha256:aaaa"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{ - {"id": "app", "type": "container", "name": "one", "image": "store/one@sha256:aaaa"}}}, - inventory.Source{Repository: "novox/one", Ref: "main", BuiltFrom: "c0", Asked: time.Now()}); err != nil { - t.Fatal(err) - } - code := repoMerge("one", "one1aaaaaaaa", t0) - b.indexed(t, "one", "1", "novox/one", "", code.Commit, fp) - hear(t, open, code, t0.Add(time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - if len(ws) != 1 { - t.Fatalf("walks: %+v", ws) - } - w := ws[0] - if w.State != inventory.PlanDone || len(w.Tiers) != 0 || len(w.Modules) != 0 { - t.Fatalf("the walk reads %s, tiers %v, modules %v; want done with nothing to walk", w.State, w.Tiers, w.Modules) - } - if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "is the build the mesh holds") { - t.Fatalf("the walk does not say one's fingerprint is unchanged: %+v", w.Delivery) - } -} diff --git a/cmd/mesh-controller/same_program_test.go b/cmd/mesh-controller/same_program_test.go deleted file mode 100644 index 39b4a467..00000000 --- a/cmd/mesh-controller/same_program_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package main - -import ( - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// TestAWalkLeavesOutAModuleWhoseFingerprintDidNotChange is novox/hq issue 721 on fingerprints (R4e A9, ADR 0453 §6): -// mesh-controller #335 changed one sentence that mesh-builder never reaches; the indexer's fingerprint of build-agent at -// the merge is the fingerprint of the build the mesh holds, so the walk moves mesh-controller alone and says build-agent -// is unchanged. A held build of the old form, or a fingerprint not written, leaves the module in the walk. -func TestAWalkLeavesOutAModuleWhoseFingerprintDidNotChange(t *testing.T) { - a, b := "src2:"+strings.Repeat("a", 64), "src2:"+strings.Repeat("b", 64) - now := map[string]string{"build-agent": a, "mesh-controller": b, "old": a, "unwritten": ""} - held := map[string]string{"build-agent": a, "mesh-controller": a, "old": "src1:" + strings.Repeat("a", 64)} - same := sameByFingerprint(now, held) - if len(same) != 1 || !strings.Contains(same["build-agent"], "is the build the mesh holds") { - t.Fatalf("the walk leaves out %v, want build-agent alone", same) - } - p := &inventory.Plan{Tiers: [][]string{{"mesh-controller", "build-agent"}}, - Modules: map[string]*inventory.PlanModule{"mesh-controller": {}, "build-agent": {}}} - withhold(p, []string{"build-agent"}, nil) - if len(p.Tiers) != 1 || len(p.Tiers[0]) != 1 || p.Tiers[0][0] != "mesh-controller" { - t.Fatalf("the walk moves %v, want mesh-controller alone", p.Tiers) - } -} - -// TestADependentOfABaseThatCameOutTheSameIsLeftOut is ADR 0267 rule 3 (novox/hq issue 607): a dependent is in a walk -// for its base; when the base comes out the same as the build the mesh holds, the dependent stands on that build -// still, and leaves the walk with it — and so does a dependent of that dependent. One a merge moves itself stays. -func TestADependentOfABaseThatCameOutTheSameIsLeftOut(t *testing.T) { - edges := []inventory.Edge{{From: "app", To: "base", Kind: inventory.EdgeDeclared}, - {From: "app2", To: "app", Kind: inventory.EdgeDeclared}, {From: "own", To: "base", Kind: inventory.EdgeDeclared}} - plan := &inventory.Plan{Modules: map[string]*inventory.PlanModule{"base": {}, "app": {}, "app2": {}, "own": {}}} - same := withDependentsOf(map[string]string{"base": "unchanged: its program is the same"}, plan, - []string{"base", "own"}, edges) - if !strings.Contains(same["app"], "only for base") || !strings.Contains(same["app2"], "only for app") { - t.Fatalf("the dependents of a base that came out the same are not left out: %v", same) - } - if _, left := same["own"]; left { - t.Fatalf("a module its merge moves itself is left out for its base: %v", same) - } - // A dependent of a base that still moves stays. - edges = append(edges, inventory.Edge{From: "app", To: "own", Kind: inventory.EdgeDeclared}) - same = withDependentsOf(map[string]string{"base": "unchanged"}, plan, []string{"base", "own"}, edges) - if _, left := same["app"]; left { - t.Fatalf("a dependent of a base that still moves is left out: %v", same) - } -} - -// Against the store, the path a real merge takes (novox/hq issues 721 and 611): a merge whose check built its one -// module the same as the build the mesh holds is cut into a walk done with nothing to walk, whose record names the -// module unchanged and why — the walk mesh-delivery reads as delivered, never stranded. diff --git a/cmd/mesh-controller/same_source_test.go b/cmd/mesh-controller/same_source_test.go deleted file mode 100644 index b88fd010..00000000 --- a/cmd/mesh-controller/same_source_test.go +++ /dev/null @@ -1,212 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "errors" - "reflect" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A build whose source is unchanged is never a move (novox/hq issue 280): an image is not -// byte-reproducible, so two builds of one source make two digests, and the rule that a rebuild with -// identical artifacts is no move (ADR 0236) never held for one. - -// anImageBuild is a build outcome of an image module: its manifest names the image by the digest made. -func anImageBuild(t *testing.T, module, id, commit, digest, source string, asked time.Time) link.BuildResult { - t.Helper() - image := "registry.invalid:5000/" + module + "/server@sha256:" + digest - manifest, _ := json.Marshal(map[string]any{"module": module, "version": "1", - "resources": []any{map[string]any{"id": "svc", "type": "container", "name": module, "image": image}}}) - return link.BuildResult{ID: link.NewBuildID(asked), Repository: "novox/mesh-catalog", Path: "modules/" + module, - On: "anchor", Module: module, Commit: commit, Manifest: manifest, SourceFingerprint: source, - Made: []link.MadeArtifact{{Name: "server", Kind: "image", Reference: image}}, - Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}} -} - -// shelfNames is whether the module the mesh holds names this digest. -func shelfNames(t *testing.T, inv *inventory.Inventory, module, digest string) bool { - t.Helper() - shelf, err := inv.Catalogue(t.Context()) - if err != nil { - t.Fatal(err) - } - raw, _ := json.Marshal(shelf[module]) - return strings.Contains(string(raw), digest) -} - -// A rebuild of an unchanged source is registered with the artifacts of the build it was first made -// as: no machine is sent a new digest, a module standing on it is handed the same base, and the two -// builds are one to every rule that asks whether a send moves something. A real source change moves. -func TestARebuildOfAnUnchangedSourceKeepsItsArtifacts(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - start := time.Now().Add(-time.Hour) - for i, b := range []link.BuildResult{ - anImageBuild(t, "app", "", "c1aaaaaa", strings.Repeat("a", 64), "src1:same", start), - // Another module's merge rebuilt it: a new commit, a new image digest, the same source. - anImageBuild(t, "app", "", "c2bbbbbb", strings.Repeat("b", 64), "src1:same", start.Add(time.Minute)), - } { - if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil { - t.Fatalf("build %d: %v", i, err) - } - } - if !shelfNames(t, inv, "app", strings.Repeat("a", 64)) || shelfNames(t, inv, "app", strings.Repeat("b", 64)) { - t.Fatal("a rebuild of an unchanged source registered the digest it made, not the one the mesh holds") - } - if src, err := inv.SourceOf(ctx, "app"); err != nil || src.BuiltFrom != "c2bbbbbb" { - t.Fatalf("the module is not at the commit it was rebuilt from: %+v %v", src, err) - } - held, err := inv.Held(ctx) - if err != nil || !strings.HasSuffix(held["app/server"], strings.Repeat("a", 64)) { - t.Fatalf("a module standing on it is handed %q, not the build the mesh holds (%v)", held["app/server"], err) - } - f, err := readMoveFacts(ctx, inv) - if err != nil { - t.Fatal(err) - } - if !f.identical("app", "c1aaaaaa", "c2bbbbbb") { - t.Fatal("two builds of one source are not one build") - } - - // A real change to the source moves: its own digest registered, and not identical. - if _, _, err := takeIn(ctx, inv, anImageBuild(t, "app", "", "c3cccccc", strings.Repeat("c", 64), "src1:changed", - start.Add(2*time.Minute))); err != nil { - t.Fatal(err) - } - if !shelfNames(t, inv, "app", strings.Repeat("c", 64)) { - t.Fatal("a changed source did not register what it made") - } - if f, _ = readMoveFacts(ctx, inv); f.identical("app", "c2bbbbbb", "c3cccccc") { - t.Fatal("a changed source is read as the same build") - } - // And a builder that says no fingerprint is told apart by its artifacts alone, as before. - if _, _, err := takeIn(ctx, inv, anImageBuild(t, "app", "", "c4dddddd", strings.Repeat("d", 64), "", - start.Add(3*time.Minute))); err != nil { - t.Fatal(err) - } - if !shelfNames(t, inv, "app", strings.Repeat("d", 64)) { - t.Fatal("a build with no fingerprint did not register what it made") - } -} - -// The bus rebuilt for another module's merge, its source untouched and its image digest new: no move — -// no push to its machine is held, no bus step is demanded, and `bus upgrade` has nothing to do. A real -// change to the bus's source is the planned step again. -func TestABusRebuiltFromAnUnchangedSourceDemandsNoBusStep(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - start := time.Now().Add(-time.Hour) - bus := func(id, commit, digest, source string, at time.Time) link.BuildResult { - b := anImageBuild(t, "nats", id, commit, digest, source, at) - manifest, _ := json.Marshal(map[string]any{"module": "nats", "version": "2", - "provides": []any{map[string]any{"name": "mesh-bus"}}, - "resources": []any{map[string]any{"id": "svc", "type": "container", "name": "nats", - "image": b.Made[0].Reference}}}) - b.Manifest = manifest - return b - } - if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, bus("", "n1111111", strings.Repeat("a", 64), "src1:bus", start))); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1111111"}); err != nil { - t.Fatal(err) - } - // The wide rebuild: a new commit, a new image digest, the same source. - if _, _, err := takeIn(ctx, inv, bus("", "n2222222", strings.Repeat("b", 64), "src1:bus", start.Add(time.Minute))); err != nil { - t.Fatal(err) - } - if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 { - t.Fatalf("a bus rebuilt from an unchanged source held its machine: %v %v", held, err) - } - wasSnapshot := takeBusSnapshot - t.Cleanup(func() { takeBusSnapshot = wasSnapshot }) - takeBusSnapshot = func(context.Context, string, string) (string, error) { - return "", errors.New("no snapshot is taken for a bus step nobody needs") - } - var sent [][]string - wasSend := sendRollout - sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) { - sent = append(sent, names) - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - if err := busCommand(ctx, []string{"upgrade", "--why", "nothing changed", "--reversible"}); err != nil || len(sent) != 0 { - t.Fatalf("a bus step ran for a bus whose source is unchanged: %v, sent %v", err, sent) - } - if !shelfNames(t, inv, "nats", strings.Repeat("a", 64)) { - t.Fatal("the bus the mesh holds is not the image its machine runs") - } - - // A real change to the bus's source: the planned step. - if _, _, err := takeIn(ctx, inv, bus("", "n3333333", strings.Repeat("c", 64), "src1:bus-2.12", start.Add(2*time.Minute))); err != nil { - t.Fatal(err) - } - held, err := busHeld(ctx, inv, []string{"anchor"}) - if err != nil || !strings.Contains(held["anchor"], "planned step") { - t.Fatalf("a changed bus did not demand its planned step: %v %v", held, err) - } -} - -// A plan's build made from the source every machine running the module runs is not sent and not -// judged: nothing moves. A build of a changed source is sent to its first machine, gated, as before. -func TestAPlanSendsNothingForAnUnchangedSource(t *testing.T) { - judgeLeavesAsTheCore(t) - for _, tc := range []struct { - name string - source string - sent [][]string - }{ - {"unchanged", "src1:app", nil}, - {"changed", "src1:app-changed", [][]string{{"anchor"}}}, - } { - t.Run(tc.name, func(t *testing.T) { - g := aGateMesh(t) - ctx := t.Context() - inv := g.open.inventory - // What anchor and laptop run (c1) was made from src1:app; the plan's build of c2 from the source - // the case says. - for _, b := range []inventory.Build{ - {ID: "build-1s", Module: "app", Commit: "c1", SourceFingerprint: "src1:app", - Asked: time.Now().Add(-30 * time.Second), At: time.Now().Add(-30 * time.Second)}, - {ID: "build-2s", Module: "app", Commit: "c2", SourceFingerprint: tc.source, - Asked: time.Now(), At: time.Now()}, - } { - b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - p := g.plan(t) - p.Modules["app"].Build = "build-2s" - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - advancePlans(ctx, g.open) - if !reflect.DeepEqual(g.sent, tc.sent) { - t.Fatalf("sent %v, want %v", g.sent, tc.sent) - } - p = g.plan(t) - s := p.Modules["app"] - if tc.sent == nil && (s.SentAt == nil || s.Gate != nil || !strings.Contains(s.Why, "no move")) { - t.Fatalf("an unchanged source was not read as no move: %+v", s) - } - if tc.sent != nil && (s.Gate == nil || len(s.First) == 0) { - t.Fatalf("a changed source was not sent to its first machine, gated: %+v", s) - } - }) - } -} diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index 87350b89..0eaedffb 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -95,7 +95,11 @@ type verbArguments struct { // removedVerbs are the controller seat's verbs R4e removed, kept in its table with no input so a caller is answered // by name (ADR 0501 §5, ADR 0503 §3, ADR 0450 §8). var removedVerbs = map[string]bool{"build": true, "rebuild": true, "forget-checks": true, "delivery-plan": true, - "delivery-check": true, "queue": true, "cancel": true, "clear": true, "kill": true, "replay": true, "pause": true, "resume": true} + "delivery-check": true, "queue": true, "cancel": true, "clear": true, "kill": true, "replay": true, "pause": true, "resume": true, + // Release 6 (ADR 0539 §5): a delivery is each machine following its wants, so the verbs that planned, started, stopped, + // retried or closed one are gone. + "plans": true, "plan": true, "push": true, "deliver": true, "delivery-stop": true, "delivery-backups-again": true, + "delivery-walks": true, "delivery-order": true, "upgrade": true} // controllerVerb is this binary's own definition of a verb: what it runs is what it declares, so the // arguments are checked against the table compiled beside argvFor, not a row a newer or older build @@ -351,121 +355,6 @@ func (a *verbArguments) commandLine() ([]string, error) { argv = append(argv, m) } return argv, nil - case "plans": - if err := plainPlanIDs(a.given); err != nil { - return nil, err - } - if r := str("repository"); r != "" { - argv := []string{"plans", "--what-if", r} - if p := str("paths"); p != "" { - argv = append(argv, "--paths", p) - } - if m := str("modules"); m != "" { - argv = append(argv, "--modules", m) - } - if d := str("module-dirs"); d != "" { - argv = append(argv, "--module-dirs", d) - } - // The plan as data, the document delivery-plan answers (novox/hq ADR 0450 §2, issue 1324). - if str("data") == "true" { - argv = append(argv, "--data") - for _, flag := range []string{"head", "base", "removed"} { - if v := str(flag); v != "" { - argv = append(argv, "--"+flag, v) - } - } - } - return argv, nil - } - for _, act := range []string{"stop", "close", "retry", "go"} { - if id := str(act); id != "" { - argv := []string{"plans", act, id} - if act == "retry" { - return argv, nil - } - // Ending a plan by hand says why (novox/hq to-be 45 §7); the command refuses it without. - if w := str("why"); w != "" { - argv = append(argv, "--why", w) - } - if c := str("cause"); c != "" { - argv = append(argv, "--cause", c) - } - return argv, nil - } - } - if id := str("wait"); id != "" { - // Following a plan (novox/hq issue 229): checked here too, so a call asking what no wait can do is - // refused before a command is started. - if _, _, err := planWaitAsked(a.given["until"], a.given["bound"]); err != nil { - return nil, err - } - argv := []string{"plans", "wait", id} - if u := str("until"); u != "" { - argv = append(argv, "--until", u) - } - if b := str("bound"); b != "" { - argv = append(argv, "--bound", b) - } - return argv, nil - } - if id := str("id"); id != "" { - return []string{"plans", id}, nil - } - argv := []string{"plans"} - if n := str("limit"); n != "" { - argv = append(argv, "-n", n) - } - return argv, nil - // The delivery's owner's verbs (novox/hq ADR 0239). - case "delivery-order": - if err := need("members"); err != nil { - return nil, err - } - members, err := membersFlags(str("members")) - if err != nil { - return nil, err - } - return append([]string{"delivery", "order"}, members...), nil - case "deliver": - if err := need("plan"); err != nil { - return nil, err - } - argv := []string{"delivery", "go", str("plan"), "--by", catalogue.DeliverySeat} - if w := str("why"); w != "" { - argv = append(argv, "--why", w) - } - return argv, nil - case "delivery-stop": - if err := need("plan", "why"); err != nil { - return nil, err - } - argv := []string{"delivery", "stop", str("plan"), "--why", str("why")} - if b := str("by"); b != "" { - argv = append(argv, "--by", catalogue.DeliverySeat+" for "+b) - } - // The merges to leave out: the walk's other merges are walked again without them (novox/hq ADR 0299). - if w := str("without"); w != "" { - argv = append(argv, "--without", w) - } - return argv, nil - case "delivery-backups-again": - if err := need("plan", "why"); err != nil { - return nil, err - } - argv := []string{"delivery", "backups-again", str("plan"), "--why", str("why")} - if b := str("by"); b != "" { - argv = append(argv, "--by", catalogue.DeliverySeat+" for "+b) - } - return argv, nil - case "delivery-walks": - argv := []string{"delivery", "walks"} - if id := str("plan"); id != "" { - return append(argv, "--plan", id), nil - } - if n := str("limit"); n != "" { - argv = append(argv, "-n", n) - } - return argv, nil case "try": // A branch tried before its pull request (novox/hq issue 686): asked, or read by the id it answered. if id := str("id"); id != "" { @@ -488,18 +377,6 @@ func (a *verbArguments) commandLine() ([]string, error) { argv = append(argv, "--store-only") } return argv, nil - case "plan": - if err := need("node"); err != nil { - return nil, err - } - if on("files") { - return []string{"plan", str("node"), "--files"}, nil - } - // What a push would change there (novox/hq ADR 0217); diff with files is passed over, and so refused. - if on("diff") { - return []string{"plan", str("node"), "--diff"}, nil - } - return []string{"plan", str("node"), "--json"}, nil case "assign", "unassign": if err := need("node", "module"); err != nil { return nil, err @@ -523,42 +400,6 @@ func (a *verbArguments) commandLine() ([]string, error) { return nil, err } return []string{"unpin", str("node"), str("provision")}, nil - case "push": - // A push to a named machine waits for that machine's own answer, so the call's record says - // how it went: applied, refused with the machine's reason, or which resources failed and why - // (novox/hq issue 629). How long is the call's `wait`, else the controller's setting `push-wait`; - // neither given waits for nothing, as before. The caller is answered before the push starts - // (answersFirst) and the machines are released once sent, so the wait holds nobody; it is what - // `calls` then shows. A sweep (--behind) does not wait: the slowest machine would hold back - // what is said of the others. - // A push through the seat is a push by hand, and says why (novox/hq to-be 45 §7). - if err := need("why"); err != nil { - return nil, fmt.Errorf("%w: a push by hand is a repair, recorded in the hand-act log with why", err) - } - why := []string{"--why", str("why")} - if c := str("cause"); c != "" { - why = append(why, "--cause", c) - } - if n := str("node"); n != "" { - // behind is not read here: given with a machine, it is refused as passed over — naming - // a machine and asking for every machine behind are two requests, and guessing one - // would push a machine nobody named, or not push one somebody did. - wait, err := pushWait(str("wait")) - if err != nil { - return nil, err - } - argv := []string{"push", n, "--wait", wait.String()} - // A running module's data moving is sent only when said, per module (novox/hq ADR 0217). - if m := str("move"); m != "" { - argv = append(argv, "--move", m) - } - return append(argv, why...), nil - } - // No machine: the whole mesh, whether or not behind said so. The command's answer says it - // first, so a caller who meant one machine reads that it was not one. move is not read: a - // machine whose data would move is held and named, and sent by a push that names it. - on("behind") - return append([]string{"push", "--behind", "--wait", "0"}, why...), nil case "hand-act": if err := need("what", "why", "cause"); err != nil { return nil, err @@ -777,35 +618,6 @@ func (a *verbArguments) commandLine() ([]string, error) { argv = append(argv, "--retired") } return argv, nil - case "upgrade": - if on("backlog") { - return []string{"upgrade", "backlog"}, nil - } - if on("release-backlog") { - argv := []string{"upgrade", "release-backlog", "--why", str("why")} - if c := str("cause"); c != "" { - argv = append(argv, "--cause", c) - } - return argv, nil - } - m, policy := str("module"), str("policy") - if m == "" { - if policy != "" { - return nil, errors.New("upgrade takes a policy only for a module") - } - return []string{"upgrade"}, nil - } - argv := []string{"upgrade", m} - if policy != "" { - argv = append(argv, policy) - if on("together") { - argv = append(argv, "--together") - } - if w := str("why"); w != "" { - argv = append(argv, "--why", w) - } - } - return argv, nil case "bus": if !on("upgrade") { return []string{"bus"}, nil @@ -1319,23 +1131,13 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) { // At once, from the summary the serving controller keeps (novox/hq to-be 45 Phase 0). return statusFrom.answer(ctx) } - if !readingVerbs[verb] && !(verb == "plans" && !actsOnAPlan(args)) { + if !readingVerbs[verb] { // Whatever it did, `status` is composed again once it has. defer statusFrom.nudge() } - if verb == "plans" && len(argv) > 1 && argv[1] == "wait" { - // A wait holds its answer up to its bound (novox/hq issue 229), and is answered in full inside it. - asked := "" - if i := slices.Index(argv, "--bound"); i > 0 && i+1 < len(argv) { - asked = argv[i+1] - } - if _, bound, err := planWaitAsked("", asked); err == nil { - link.AnswersWithin(ctx, bound+planWaitStart) - } - } if verb == "try" && len(argv) > 1 && argv[1] == "--id" { // Reading a try holds its answer for the verdict, up to its bound (novox/hq issue 686, 229). - link.AnswersWithin(ctx, tryWait+planWaitStart) + link.AnswersWithin(ctx, tryWait+5*time.Second) } if answer, read := readHere(ctx, argv); read { return answer, nil @@ -1346,13 +1148,6 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) { link.Acknowledge(ctx) } answer, err := runTheVerb(ctx, argv) - // **A plan the verb moved is looked at as it ends** (novox/hq ADR 0406 §5, issue 974): the verb ran as a - // child process, whose saves this process does not hear, so `deliver` after an approval, a stop, a retry - // and a send from a verb would otherwise wait for an unrelated event. Even when it failed: it may have - // kept a plan before it did. - if !readingVerbs[verb] && !(verb == "plans" && !actsOnAPlan(args)) { - askToLook() - } return answer, err } } @@ -1363,16 +1158,6 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) { // verb needs no child process. var runTheVerb = runVerb -// actsOnAPlan is `plans` asked to stop, close or retry one rather than to show them. -func actsOnAPlan(args map[string]any) bool { - for _, act := range []string{"stop", "close", "retry"} { - if v, _ := args[act].(string); strings.TrimSpace(v) != "" { - return true - } - } - return false -} - // inProcess are the verbs answered by this process rather than by a command it runs: `tools` from // the records, `calls` from what this process served. var inProcess = map[string]bool{"tools": true, "calls": true, "doctor": true, diff --git a/cmd/mesh-controller/seatverbs_schema_test.go b/cmd/mesh-controller/seatverbs_schema_test.go index c34ff0f5..90ebbd58 100644 --- a/cmd/mesh-controller/seatverbs_schema_test.go +++ b/cmd/mesh-controller/seatverbs_schema_test.go @@ -56,28 +56,6 @@ var saidOutright = map[string]string{ // what it requires and one argument more; and `node` in particular, for every verb whose schema // does not take a machine. -// The push that was the cause: a machine named is that machine; none named is the whole mesh, and -// naming one beside behind is refused rather than one of the two guessed. -func TestAPushIsOneMachineOrSaysItIsTheWholeMesh(t *testing.T) { - argv, err := argvFor("push", map[string]any{"node": "g1", "why": "w"}) - if err != nil || strings.Join(argv, " ") != "push g1 --wait 0s --why w" { - t.Fatalf("a named push: %v %v", argv, err) - } - for _, args := range []map[string]any{{"why": "w"}, {"behind": "true", "why": "w"}} { - argv, err := argvFor("push", args) - if err != nil || strings.Join(argv, " ") != "push --behind --wait 0 --why w" { - t.Fatalf("a push of the whole mesh %v: %v %v", args, argv, err) - } - } - if _, err := argvFor("push", map[string]any{"node": "g1", "behind": "true", "why": "w"}); err == nil || - !strings.Contains(err.Error(), `"behind"`) { - t.Fatalf("a named push with behind was taken: %v", err) - } - if _, err := argvFor("push", map[string]any{"machine": "g1"}); err == nil || !strings.Contains(err.Error(), `"machine"`) { - t.Fatalf("a push given the machine under another name ran as a push of every machine: %v", err) - } -} - // commandFlags is every flag set this package's commands parse, by the name the set is made with, // and the flags defined on it — read from the source, so a flag added to a command is seen here // without anyone remembering to. diff --git a/cmd/mesh-controller/seatverbs_test.go b/cmd/mesh-controller/seatverbs_test.go index 37193269..06c525b8 100644 --- a/cmd/mesh-controller/seatverbs_test.go +++ b/cmd/mesh-controller/seatverbs_test.go @@ -1,11 +1,8 @@ package main import ( - "context" "errors" "fmt" - "os" - "path/filepath" "strings" "testing" ) @@ -99,62 +96,6 @@ func TestAVerbMissingWhatItNeedsIsRefused(t *testing.T) { } } -// Acts do not block the call. A build is sent, not waited for: the asker reads the build's log by its id. -// A push answers its caller before it starts (answersFirst), so a named push waits for the machine's own -// answer without holding anybody, and the call's record says how it went (novox/hq issue 629). A -// repository given as a forge path is said to be one (issue 176). -func TestActsDoNotBlockTheCall(t *testing.T) { - argv, _ := argvFor("push", map[string]any{"node": "one", "why": "w"}) - if strings.Join(argv, " ") != "push one --wait 0s --why w" || !answersFirst(argv) { - t.Fatalf("a named push with no setting waits, or holds its caller: %v", argv) - } -} - -// A11a (ADR 0501 §5; issue 1851): the build, rebuild and replay --register verbs reach a command that refuses them by -// name, naming the replacement, whatever a table of before asked of them; the command lines refuse the same. -func TestTheRemovedVerbsAreRefusedByName(t *testing.T) { - for _, c := range []struct { - verb string - args map[string]any - said string - }{ - {"build", map[string]any{"repository": "novox/x", "path": "modules/x"}, "`build` is removed"}, - {"build", map[string]any{}, "`build` is removed"}, - {"rebuild", map[string]any{"what": "gitea"}, "`rebuild` is removed"}, - {"replay", map[string]any{"id": "build-1"}, "`replay` is removed"}, - {"queue", map[string]any{}, "`queue` is removed"}, - {"cancel", map[string]any{"id": "build-1"}, "`cancel` is removed"}, - {"clear", map[string]any{"dead": "true"}, "`clear` is removed"}, - {"kill", map[string]any{"id": "build-1"}, "`kill` is removed"}, - {"pause", map[string]any{"node": "ace"}, "`pause` is removed"}, - {"resume", map[string]any{}, "`resume` is removed"}, - // 1318: delivery-plan and forget-checks go too. - {"delivery-plan", map[string]any{"repository": "novox/a", "paths": "x"}, "`delivery-plan` is removed"}, - {"forget-checks", map[string]any{"owner": "novox", "repo": "a", "number": "7"}, "`forget-checks` is removed"}, - } { - argv, err := argvFor(c.verb, c.args) - if err == nil || !strings.Contains(err.Error(), c.said) { - t.Errorf("%s %v: %v %v, want refused saying %q", c.verb, c.args, argv, err, c.said) - } - } - for _, removed := range []string{"build", "rebuild", "forget-checks", "delivery-plan", "queue", "replay"} { - err := removedByName(removed) - if err == nil || !strings.Contains(err.Error(), "`"+removed+"` is removed") || !strings.Contains(err.Error(), "ADR 05") { - t.Errorf("%s: %v", removed, err) - } - } - if err := deliveryCommand(context.Background(), []string{"plan", "--repository", "novox/a"}); err == nil || - !strings.Contains(err.Error(), "`delivery-plan` is removed") { - t.Errorf("delivery plan: %v", err) - } -} - -// What `tools` answers is the seats' records, with each verb's schema. - -// A JSON verb's answer is parsed from what the command wrote to standard output alone; a warning it -// printed beside the document does not take the document away. The test binary stands in for the -// controller: `-test.run` with a name that matches nothing prints `ok` and a warning about no tests. - // `command` is the generic verb: the command line as given, split as a shell would, nothing added // (novox/hq ADR 0154, ADR 0175). It once carried an operator's `node account g14 jochen` too; a node's // accounts are the controller's terminal's alone since ADR 0266 (TestNoVerbSetsANodesAccounts). @@ -285,40 +226,3 @@ func TestAQuotedValueRoundTrips(t *testing.T) { } } } - -// A named push waits as long as the call says, else as long as the controller's setting push-wait says, else -// not at all: no length is the code's own (novox/hq issue 629). -func TestAPushWaitsAsTheCallOrTheSettingSays(t *testing.T) { - file := filepath.Join(t.TempDir(), "push.json") - if err := os.WriteFile(file, []byte(`{"push-wait": "2m"}`), 0o600); err != nil { - t.Fatal(err) - } - t.Setenv(pushFileVar, file) - argv, err := argvFor("push", map[string]any{"node": "one", "why": "w"}) - if err != nil || strings.Join(argv, " ") != "push one --wait 2m0s --why w" { - t.Fatalf("the setting was not the default: %v %v", argv, err) - } - argv, err = argvFor("push", map[string]any{"node": "one", "why": "w", "wait": "30s"}) - if err != nil || strings.Join(argv, " ") != "push one --wait 30s --why w" { - t.Fatalf("the call's wait did not win over the setting: %v %v", argv, err) - } - argv, err = argvFor("push", map[string]any{"node": "one", "why": "w", "wait": "0"}) - if err != nil || strings.Join(argv, " ") != "push one --wait 0s --why w" { - t.Fatalf("a wait of 0 did not wait for nothing: %v %v", argv, err) - } - if _, err := argvFor("push", map[string]any{"node": "one", "why": "w", "wait": "soon"}); err == nil || - !strings.Contains(err.Error(), "no duration") { - t.Fatalf("a wait that is no duration was taken: %v", err) - } - if err := os.WriteFile(file, []byte(`{"push-wait": "later"}`), 0o600); err != nil { - t.Fatal(err) - } - argv, err = argvFor("push", map[string]any{"node": "one", "why": "w"}) - if err != nil || strings.Join(argv, " ") != "push one --wait 0s --why w" { - t.Fatalf("an unreadable setting did not fall back to waiting for nothing: %v %v", argv, err) - } - argv, _ = argvFor("push", map[string]any{"why": "w"}) - if strings.Join(argv, " ") != "push --behind --wait 0 --why w" { - t.Fatalf("a sweep waits: %v", argv) - } -} diff --git a/cmd/mesh-controller/send_key_refused.go b/cmd/mesh-controller/send_key_refused.go deleted file mode 100644 index 5c7c9287..00000000 --- a/cmd/mesh-controller/send_key_refused.go +++ /dev/null @@ -1,62 +0,0 @@ -package main - -import ( - "context" - "fmt" - "sort" - "sync" - - "github.com/novox/mesh-controller/internal/conditions" -) - -// A plan's send key the bus refuses (novox/hq ADR 0433 §4, issue 1157): the look that could not write it says which, and -// a probe of the controller's self-check raises each as a condition naming the plan and the machine. The next look writes -// the key again; the look that writes it leaves it out of the set, and the condition clears. -const ( - kindSendKeyRefused = "send-key-refused" - sendKeyRefusedID = "DL" -) - -var refusedSendKeys = struct { - mu sync.Mutex - by map[silentWait]error -}{} - -// noteRefusedSendKeys sets the keys the newest look could not write, replacing what the look before found. -func noteRefusedSendKeys(refused map[silentWait]error) { - refusedSendKeys.mu.Lock() - defer refusedSendKeys.mu.Unlock() - refusedSendKeys.by = refused -} - -// observeRefusedSendKeys is one condition per plan and machine whose send key was refused. -func observeRefusedSendKeys() []conditions.Observation { - refusedSendKeys.mu.Lock() - waits := make([]silentWait, 0, len(refusedSendKeys.by)) - why := map[silentWait]error{} - for w, err := range refusedSendKeys.by { - waits = append(waits, w) - why[w] = err - } - refusedSendKeys.mu.Unlock() - sort.Slice(waits, func(i, j int) bool { return silentKey(waits[i]) < silentKey(waits[j]) }) - seen := map[[2]string]bool{} - var out []conditions.Observation - for _, w := range waits { - if seen[[2]string{w.Plan, w.Node}] { - continue // a plan sends a machine one tier at a time; one condition says it - } - seen[[2]string{w.Plan, w.Node}] = true - out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: w.Plan, Token: kindSendKeyRefused + "-" + w.Node, - Kind: kindSendKeyRefused, Machine: w.Node, Severity: conditions.Warning, - Summary: fmt.Sprintf("the bus refused the key that watches the delivery %s for a report from %s, so nothing "+ - "names the machine silent if it never reports; the plan writes the key again at its next look", w.Plan, w.Node), - Said: fmt.Sprintf("%s: %v", silentKey(w), why[w])}) - } - return out -} - -// probeSendKeyRefused reads what the newest look of the plans could not write. -func probeSendKeyRefused(context.Context, *doctor) ([]conditions.Observation, error) { - return observeRefusedSendKeys(), nil -} diff --git a/cmd/mesh-controller/settings_walk.go b/cmd/mesh-controller/settings_walk.go deleted file mode 100644 index 08fd5aa4..00000000 --- a/cmd/mesh-controller/settings_walk.go +++ /dev/null @@ -1,281 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "slices" - "sort" - "strings" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/compose" - "github.com/novox/mesh-controller/internal/inventory" -) - -// An approved settings change is sent by a walk at once, and the operator's one Approve is its push (novox/hq ADR -// 0344, issue 641). -// -// On 2026-10-13 the operator approved a betterbird proposal on Telegram and nothing happened: the warrant set the -// layer, and the push stayed a separate act (ADR 0277 §18) that nobody took. Now the warrant that sets the layer -// opens a **settings walk**: the walk ADR 0236 opens for builds waiting for a gate, with a layer in place of the -// builds. Its order is every machine running the module that the layer reaches, the control node last; each is -// sent and its module judged there, from the machine's report on that send, before the next is sent. The send is -// the layer alone: every module there stays at the build its machine was last sent (keepingBuildsForSettings), so a -// build held for a person (ADR 0242) or waiting for a gate stays where it is — a policy of `record` holds a build, -// never a setting — and a send that would move a running module's data stops the walk (ADR 0217 §3). The walk says its start and end on `plan-moved` (ADR 0308), and the holder of the operator's -// channel tells its end once. A failure stops the walk and leaves the layer as approved; it holds back nothing -// else. - -// settingsWalkPrefix names a settings walk's id. -const settingsWalkPrefix = "settings-" - -// settingsWalkFor is what a settings walk says it is for, where a merge's walk says its repository. -func settingsWalkFor(s inventory.PlanSettings) string { - where := s.Node - if where == "" { - where = "the whole mesh" - } - return fmt.Sprintf("the settings of %s on %s", s.Module, where) -} - -// settingsWalkOrder is the machines a layer reaches, in the order a settings walk sends them: every machine -// running the module, or the one the layer names when it runs it there, sorted, the control node last. Pure. -func settingsWalkOrder(node string, running, controllers []string) []string { - var order, last []string - for _, n := range running { - if node != "" && n != node { - continue - } - if slices.Contains(controllers, n) { - last = append(last, n) - } else { - order = append(order, n) - } - } - sort.Strings(order) - sort.Strings(last) - return append(order, last...) -} - -// openSettingsWalk opens the walk that sends an approved layer, and answers the words of what it does: the -// machines it sends, or that no machine runs the module and nothing is sent. Called on the operator's warrant, -// after the layer is set. -var openSettingsWalk = func(ctx context.Context, open *stores, s inventory.PlanSettings) (string, error) { - inv := open.inventory - running, err := inv.Running(ctx, s.Module) - if err != nil { - return "", err - } - controllers, err := inv.Running(ctx, catalogue.ControllerSeatName) - if err != nil { - return "", err - } - order := settingsWalkOrder(s.Node, running, controllers) - if len(order) == 0 { - return fmt.Sprintf("no machine runs %s there, so there is nothing to send", s.Module), nil - } - release, err := inv.HoldPlans(ctx, true) - if err != nil { - return "", err - } - defer release() - now := time.Now().UTC() - p := inventory.Plan{ID: fmt.Sprintf("%s%d", settingsWalkPrefix, now.UnixNano()), Repository: settingsWalkFor(s), - Created: now, State: inventory.PlanRolling, Tiers: [][]string{{s.Module}}, Modules: map[string]*inventory.PlanModule{}, - Release: &inventory.PlanRelease{Order: order, Settings: &s}, - Note: fmt.Sprintf("approved (ask %s): sent to %s, one machine at a time, each judged", s.Ask, - strings.Join(order, ", "))} - if err := inv.SavePlan(ctx, &p); err != nil { - return "", err - } - fmt.Printf("%s: %s — %s\n", p.ID, p.Repository, p.Note) - return fmt.Sprintf("%s sends it to %s, one machine at a time, each judged", p.ID, strings.Join(order, ", ")), nil -} - -// settingsDataMoves is what a settings walk's send to a machine would move of a running module's data: composed as -// that send composes it, every build kept, and compared with what the machine was last sent (ADR 0217 §3). -var settingsDataMoves = func(ctx context.Context, open *stores, node string) ([]dataMove, error) { - keeping := keepingBuildsForSettings(keepingRecorded(ctx), node) - keeping = withInputCache(keeping) - if _, err := generatorsOf(keeping, open); err != nil { - return nil, err - } - if _, _, err := resolveNode(keeping, open, node); err != nil { - return nil, err - } - declared, _, err := composeQuestion(keeping, open, node, compose.Reading, false) - if err != nil { - return nil, err - } - body, err := declared.Body() - if err != nil { - return nil, err - } - return heldMoves(keeping, open.inventory, node, body, nil) -} - -// judgingOn is the open walk, other than the one named, whose gate is judging on a machine; empty when none is. A -// settings walk sends a machine only when no other walk judges there: its send would set aside the declaration -// that walk's gate reads the machine's report against. -var judgingOn = func(ctx context.Context, open *stores, self, node string) (string, error) { - plans, err := open.inventory.OpenPlans(ctx) - if err != nil { - return "", err - } - return judgingIn(plans, self, node), nil -} - -// judgingIn is judgingOn over plans read. Pure. -func judgingIn(plans []inventory.Plan, self, node string) string { - judging := func(g *inventory.PlanGate) bool { - return g != nil && g.Verdict == "" && slices.Contains(g.Machines, node) - } - for _, p := range plans { - if p.ID == self || !p.Open() { - continue - } - if p.Release != nil { - if judging(p.Release.Gate) { - return p.ID - } - continue - } - for _, s := range p.Modules { - if s != nil && judging(s.Gate) { - return p.ID - } - } - } - return "" -} - -// advanceSettingsWalk takes one step of a settings walk: the next machine sent and its module judged there, or the -// machine being judged judged once more; done when every machine is. -func advanceSettingsWalk(ctx context.Context, open *stores, p *inventory.Plan) (bool, error) { - r, s := p.Release, p.Release.Settings - now := time.Now().UTC() - wait := func(note string) (bool, error) { - changed := p.Note != note - p.Note = note - return changed, nil - } - if r.Gate == nil { - heard, err := releaseHeard(ctx, open) - if err != nil { - return false, err - } - for r.Next < len(r.Order) { - node := r.Order[r.Next] - if !heard[node] { - r.Skipped = append(r.Skipped, node) - r.Next++ - continue - } - if id, err := judgingOn(ctx, open, p.ID, node); err != nil { - return false, err - } else if id != "" { - return wait(fmt.Sprintf("waiting before %s: %s is judging there", node, id)) - } - // **A running module's data that would move is a person's word** (ADR 0217 §3): the walk stops before - // that machine, failed and said, and `push --move` sends it. - held, err := settingsDataMoves(ctx, open, node) - if errors.Is(err, errSettingsCannotKeep) { - return failBefore(p, node, err.Error()) - } - if err != nil { - return false, fmt.Errorf("whether %s's data would move could not be read: %w", node, err) - } - if len(held) > 0 { - var said strings.Builder - sayDataHeld(&said, node, held) - return failBefore(p, node, "a running module's data would move there — "+ - strings.Join(strings.Fields(said.String()), " ")) - } - // A build that cannot be kept is refused by the composition above, which is the send's own, and fails the - // walk there: the send flattens its refusals into words, so it is not looked for again here. - sent, grants, err := sendJudged(keepingBuildsForSettings(ctx, node), open, node) - if err != nil { - return false, fmt.Errorf("sending %s %s: %w", node, settingsWalkFor(*s), err) - } - // **The module's build is the one the gate judges** (issue 641 review): kept by this send, it is what a - // crossing send — a merge's walk sending the machine a new build of the module — supersedes, so this gate - // never reads that send's report as its own. - kept := "" - if builds, known, err := open.inventory.SentBuilds(ctx, node); err == nil && known { - kept = builds[s.Module] - } - r.Gate = &inventory.PlanGate{Component: coreComponent(s.Module), Machines: sent, Since: &now, - From: kept, To: kept, Sent: sentNow(ctx, open.inventory, sent), Grants: grants} - p.Note = fmt.Sprintf("sent %s %s (ask %s), every build there kept; judging %s there", node, - settingsWalkFor(*s), s.Ask, s.Module) - if said := grantsSaid(grants); said != "" { - p.Note += "; " + said - } - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil - } - p.State, p.Tier = inventory.PlanDone, len(p.Tiers) - p.Note = fmt.Sprintf("%s (ask %s) delivered on %s", settingsWalkFor(*s), s.Ask, strings.Join(r.Done, ", ")) - if len(r.Done) == 0 { - // Sent nowhere is not delivered: said as the walk's failure, so the operator is not told it landed. - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("%s (ask %s) was sent to no machine", settingsWalkFor(*s), s.Ask) - } - if len(r.Skipped) > 0 { - p.Note += "; not heard from, left as they were, their next send carries it: " + strings.Join(r.Skipped, ", ") - } - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil - } - g := r.Gate - node := r.Order[r.Next] - verdict, err := judgeMoves(ctx, open, g, []judged{{module: s.Module, node: node}}, now) - if err != nil { - return false, err - } - switch verdict { - case "": - return wait(fmt.Sprintf("judging %s on %s: %s", s.Module, node, gateLine(g))) - case inventory.GatePassed: - r.Done = append(r.Done, node) - r.Next++ - r.Gate = nil - return true, nil - case inventory.GateSuperseded: - p.State = inventory.PlanSuperseded - p.Note = fmt.Sprintf("superseded on %s: %s — nothing judged, nothing put back; the layer is set, and the "+ - "machines not yet sent take it at their next send", node, g.Why) - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil - } - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("%s failed its gate on %s: %s — the layer stays as the operator approved it, and the one it "+ - "replaced is in `settings %s` history", settingsWalkFor(*s), node, g.Why, s.Module) + unreached(r) - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil -} - -// failBefore ends a settings walk failed before it sent a machine, saying why, and that the layer stays. -func failBefore(p *inventory.Plan, node, why string) (bool, error) { - r, s := p.Release, p.Release.Settings - p.State = inventory.PlanFailed - p.Note = fmt.Sprintf("%s not sent to %s: %s; the layer stays as the operator approved it", settingsWalkFor(*s), - node, why) + unreached(r) - fmt.Printf("%s: %s\n", p.ID, p.Note) - return true, nil -} - -// unreached is what a stopped settings walk left without the layer, in words: the machines after the one it stopped -// at, and those passed over as away (ADR 0344 §6). Empty when there are none. -func unreached(r *inventory.PlanRelease) string { - var out string - if r.Next+1 < len(r.Order) { - out += "; not sent: " + strings.Join(r.Order[r.Next+1:], ", ") - } - if len(r.Skipped) > 0 { - out += "; not heard from, left as they were: " + strings.Join(r.Skipped, ", ") - } - return out -} diff --git a/cmd/mesh-controller/settings_walk_integration_test.go b/cmd/mesh-controller/settings_walk_integration_test.go deleted file mode 100644 index 5f11ec27..00000000 --- a/cmd/mesh-controller/settings_walk_integration_test.go +++ /dev/null @@ -1,439 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "reflect" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" -) - -func TestAnApprovedLayerOpensTheWalkThatSendsIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "notes", Version: "1", - Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", - "content": "x = ${setting:x}\n"}}}) - if _, err := assign(ctx, open, "laptop", "notes"); err != nil { - t.Fatal(err) - } - // A layer set by hand opens no walk: it still waits for a push (ADR 0344 §7). - if err := atTheTerminal(t, "settings", "set", "notes", `{"x":"0"}`, "--node", "laptop"); err != nil { - t.Fatal(err) - } - if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 0 { - t.Fatalf("a layer set at the terminal opened %d walk(s)", len(plans)) - } - set := setLayerIn(open) - now := time.Now() - values := map[string]any{"x": "1"} - p := settingsProposal{Module: "notes", Values: values, From: "g14/claude-code", At: now, - Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} - words, err := set(ctx, p, "s9", "approved by the operator (ask s9)") - if err != nil { - t.Fatal(err) - } - if !strings.Contains(words, "+ x (ask s9); settings-") || !strings.Contains(words, "sends it to laptop, one machine at a time") { - t.Errorf("the warrant's words: %q", words) - } - plans, err := open.inventory.OpenPlans(ctx) - if err != nil || len(plans) != 1 { - t.Fatalf("open walks: %+v %v", plans, err) - } - w := plans[0] - if w.Release == nil || w.Release.Settings == nil || w.Release.Settings.Module != "notes" || w.Release.Settings.Ask != "s9" || - !reflect.DeepEqual(w.Release.Order, []string{"laptop"}) || w.Repository != "the settings of notes on the whole mesh" || - !reflect.DeepEqual(w.Tiers, [][]string{{"notes"}}) { - t.Fatalf("the settings walk: %+v %+v", w, w.Release) - } - - // A machine's layer where the module does not run: set, and nothing to send. - q := settingsProposal{Module: "notes", Node: "anchor", Values: values, From: "g14/claude-code", At: now, - Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} - words, err = set(ctx, q, "s10", "approved by the operator (ask s10)") - if err != nil || !strings.Contains(words, "no machine runs notes there, so there is nothing to send") { - t.Fatalf("a layer nobody runs: %q %v", words, err) - } - if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 1 { - t.Fatalf("a walk was opened for a machine that does not run the module: %d open", len(plans)) - } - - // A walk that cannot be opened is the act's failure, said with the push that sends it by hand. - was := openSettingsWalk - openSettingsWalk = func(context.Context, *stores, inventory.PlanSettings) (string, error) { - return "", fmt.Errorf("the plans are not to be had") - } - t.Cleanup(func() { openSettingsWalk = was }) - again := map[string]any{"x": "2"} - byHand := map[string]any{"x": "0"} - r := settingsProposal{Module: "notes", Node: "laptop", Values: again, Before: byHand, HadLayer: true, - From: "g14/claude-code", At: now, Digest: layerDigest(again), BeforeDigest: layerDigest(byHand)} - if _, err := set(ctx, r, "s11", "approved"); err == nil || !strings.Contains(err.Error(), "is set, and the walk that "+ - "sends it could not be opened: the plans are not to be had — `push laptop` sends it") { - t.Errorf("an unopened walk: %v", err) - } -} - -func aSettledMesh(t *testing.T) *walkMesh { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - w := &walkMesh{open: open, broken: map[string]bool{}, moves: map[string][]dataMove{}, - heard: map[string]bool{"anchor": true, "laptop": true}} - withConditionsInMemory(t) - was := doctorFrom - doctorFrom = nil - t.Cleanup(func() { doctorFrom = was }) - old := time.Now().Add(-2 * time.Hour) - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c1", Module: "app", Commit: "c1", - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: old, At: old, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-1"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c1", Head: "c1", Asked: old}); err != nil { - t.Fatal(err) - } - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, "app"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - } - wasMoves, wasHeard, wasSend, wasGather, wasData := machineMoves, releaseHeard, sendRollout, gatherGateFacts, settingsDataMoves - settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) { return w.moves[node], nil } - machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { - modules, err := open.inventory.Assigned(ctx, node) - if err != nil { - return nil, err - } - sent, known, err := open.inventory.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - return f.moves(node, modules, sent, known, all), nil - } - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return w.heard, nil } - n := 0 - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - w.sent = append(w.sent, append([]string(nil), names...)) - w.kept = append(w.kept, len(names) == 1 && settingsKeptOn(ctx, names[0])) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - modules, _ := open.inventory.Assigned(ctx, node) - carried := map[string]string{} - for _, m := range modules { - carried[m] = current[m].Commit - } - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { - return nil, err - } - if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { - return nil, err - } - } - return names, nil - } - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return f, err - } - for _, r := range reports { - if w.broken[r.Node] { - r.Outcome = inventory.OutcomeFailed - } - f.reports[r.Node] = r - } - return f, nil - } - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - gateWithin, gateEvery = time.Hour, time.Hour - t.Cleanup(func() { - machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather - settingsDataMoves = wasData - gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound - }) - return w -} - -func TestASettingsWalkSendsOneMachineAtATimeEachJudged(t *testing.T) { - judgeLeavesAsTheCore(t) - w := aSettledMesh(t) - ctx := t.Context() - words, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s1", Change: "~ x"}) - if err != nil || !strings.Contains(words, "sends it to anchor, laptop") { - t.Fatalf("%q %v", words, err) - } - advancePlans(ctx, w.open) - if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: the first machine alone, before it is judged", w.sent) - } - p := w.walk(t) - if p.Release.Gate == nil || p.Release.Next != 0 || !strings.Contains(p.Note, "judging app on anchor") { - t.Fatalf("the walk after its first send: %s %+v", p.Note, p.Release) - } - gateEvery = 0 - for i := 0; i < 6; i++ { - advancePlans(ctx, w.open) - } - if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("sent %v", w.sent) - } - p = w.walk(t) - if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) || - p.Outcome() != inventory.WalkDelivered || !strings.Contains(p.Note, "delivered on anchor, laptop") { - t.Fatalf("the walk is %s: %s %+v", p.State, p.Note, p.Release) - } - // It said its start and its end, so the operator is told once (ADR 0308). - if p.Times == nil || p.Times.Started == nil || p.Times.Ended == nil { - t.Fatalf("the walk's start and end are not marked: %+v", p.Times) - } - // Nothing waits, and no walk of waiting builds opened beside it. - if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r != nil { - t.Fatalf("%+v %v", r, err) - } -} - -func TestAFailedSettingsWalkStopsAndHoldsNothingBack(t *testing.T) { - w := aSettledMesh(t) - ctx := t.Context() - inv := w.open.inventory - w.broken["anchor"] = true - if err := inv.KeepSettings(ctx, "", "app", map[string]any{"x": "1"}); err != nil { - t.Fatal(err) - } - if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s2"}); err != nil { - t.Fatal(err) - } - for i := 0; i < 4; i++ { - advancePlans(ctx, w.open) - } - p := w.walk(t) - if p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed || - !strings.Contains(p.Note, "the settings of app on the whole mesh failed its gate on anchor") || - !strings.Contains(p.Note, "the layer stays as the operator approved it") || !strings.Contains(p.Note, "not sent: laptop") { - t.Fatalf("the walk is %s: %s", p.State, p.Note) - } - if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: anchor alone, nothing to laptop", w.sent) - } - if layer, has, err := inv.Layer(ctx, "", "app"); err != nil || !has || layer["x"] != "1" { - t.Fatalf("the layer after the failure: %v %v %v", layer, has, err) - } - // A build that waits for a gate is still walked: the failed settings walk holds nothing back. - w.broken["anchor"] = false - now := time.Now().Add(-time.Minute) - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { - t.Fatal(err) - } - if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil { - t.Fatalf("the waiting build is held back by a failed settings walk: %+v %v", r, err) - } -} - -func TestASettingsWalkSendsTheLayerAloneEveryBuildKept(t *testing.T) { - w := aSettledMesh(t) - ctx := t.Context() - inv := w.open.inventory - gateEvery = 0 - if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s3"}); err != nil { - t.Fatal(err) - } - for i := 0; i < 4; i++ { - advancePlans(ctx, w.open) - } - if !reflect.DeepEqual(w.kept, []bool{true, true}) || w.walk(t).State != inventory.PlanDone { - t.Fatalf("the sends kept every build: %v; the walk: %s", w.kept, w.walk(t).Note) - } - // What such a send keeps, on the store: a rolled-out build waiting there is kept at the one last sent. - now := time.Now().Add(-time.Minute) - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { - t.Fatal(err) - } - kept, err := recordedKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor") - if err != nil || kept["app"] != "c1" { - t.Fatalf("a settings send to anchor keeps %v (%v), want app at c1", kept, err) - } - if kept, _ := recordedKept(ctx, w.open, "anchor"); kept["app"] != "" { - t.Fatalf("an ordinary send keeps a rolled-out build: %v", kept) - } - // And so the ungated build waiting on anchor is no move of the settings send's: nothing for a gate to see. - f, err := readMoveFacts(ctx, inv) - if err != nil { - t.Fatal(err) - } - waiting, _ := machineMoves(ctx, w.open, f, "anchor", false) - if len(waiting) != 1 || waiting[0].Module != "app" { - t.Fatalf("anchor waits for %+v, want app's c2", waiting) - } - if moved, err := withoutKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor", waiting); err != nil || len(moved) != 0 { - t.Fatalf("a settings send to anchor would move %+v (%v)", moved, err) - } -} - -func TestASettingsWalkStopsBeforeADataMoveAndPassesOverAMachineAway(t *testing.T) { - w := aSettledMesh(t) - ctx := t.Context() - gateEvery = 0 - w.moves["laptop"] = []dataMove{{Container: "app-server", Inside: "/data", From: "/srv/a", To: "/srv/b"}} - if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s4"}); err != nil { - t.Fatal(err) - } - for i := 0; i < 4; i++ { - advancePlans(ctx, w.open) - } - p := w.walk(t) - if p.State != inventory.PlanFailed || !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) || - !strings.Contains(p.Note, "not sent to laptop: a running module's data would move there") || - !strings.Contains(p.Note, "push laptop --move app") { - t.Fatalf("sent %v; the walk is %s: %s", w.sent, p.State, p.Note) - } - - v := aSettledMesh(t) - v.heard = map[string]bool{"laptop": true} - gateEvery = 0 - if _, err := openSettingsWalk(ctx, v.open, inventory.PlanSettings{Module: "app", Ask: "s5"}); err != nil { - t.Fatal(err) - } - for i := 0; i < 4; i++ { - advancePlans(ctx, v.open) - } - p = v.walk(t) - if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Skipped, []string{"anchor"}) || - !strings.Contains(p.Note, "delivered on laptop; not heard from, left as they were, their next send carries it: anchor") { - t.Fatalf("the walk with anchor away is %s: %s", p.State, p.Note) - } - - u := aSettledMesh(t) - u.heard = map[string]bool{} - if _, err := openSettingsWalk(ctx, u.open, inventory.PlanSettings{Module: "app", Ask: "s6"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, u.open) - if p := u.walk(t); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed || - !strings.Contains(p.Note, "was sent to no machine") || len(u.sent) != 0 { - t.Fatalf("a walk that reached nobody is %s: %s", p.State, p.Note) - } -} - -func TestACrossingSendSupersedesASettingsGate(t *testing.T) { - judgeLeavesAsTheCore(t) - w := aSettledMesh(t) - ctx := t.Context() - inv := w.open.inventory - if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s7"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, w.open) - p := w.walk(t) - if g := p.Release.Gate; g == nil || g.To != "c1" || g.From != "c1" { - t.Fatalf("the settings gate judges no build: %+v", p.Release.Gate) - } - // Another walk sends anchor app's c2 while the settings gate judges there. - now := time.Now().Add(-time.Minute) - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, w.open, "anchor"), "d-merge", map[string]string{"app": "c2"}); err != nil { - t.Fatal(err) - } - if _, err := inv.RecordDoing(ctx, nodeID(t, w.open, "anchor"), inventory.Doing{Node: "anchor", - Outcome: inventory.OutcomeApplied, Declared: "d-merge", Applied: 1, At: time.Now()}); err != nil { - t.Fatal(err) - } - gateEvery = 0 - advancePlans(ctx, w.open) - p = w.walk(t) - if p.State != inventory.PlanSuperseded || p.Release.Gate == nil || p.Release.Gate.Verdict != inventory.GateSuperseded || - !strings.Contains(p.Note, "superseded on anchor") || len(p.Release.Done) != 0 { - t.Fatalf("the settings walk after a crossing send is %s: %s %+v", p.State, p.Note, p.Release) - } - if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: laptop was sent after a superseded judging", w.sent) - } -} - -func TestASettingsWalkFailsOnARefusalThatCannotClear(t *testing.T) { - w := aSettledMesh(t) - ctx := t.Context() - inv := w.open.inventory - // What a send to a machine never sent can keep: nothing, and said as the refusal the walk fails on. - if _, err := inv.AddNode(ctx, "fresh"); err != nil { - t.Fatal(err) - } - if _, err := recordedKept(keepingBuildsForSettings(ctx, "fresh"), w.open, "fresh"); !errors.Is(err, errSettingsCannotKeep) { - t.Fatalf("a settings send to a machine never sent: %v", err) - } - w.heard = map[string]bool{"laptop": true} - settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) { - return nil, fmt.Errorf("%w: what %s was last sent is not known", errSettingsCannotKeep, node) - } - if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s8"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, w.open) - p := w.walk(t) - if p.State != inventory.PlanFailed || len(w.sent) != 0 || - !strings.Contains(p.Note, "not sent to laptop: the settings cannot be sent with every build kept: what laptop was last sent is not known") || - !strings.Contains(p.Note, "not heard from, left as they were: anchor") { - t.Fatalf("the walk is %s: %s", p.State, p.Note) - } - // It is not open, so a build waiting for a gate is walked. - now := time.Now().Add(-time.Minute) - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { - t.Fatal(err) - } - if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil { - t.Fatalf("the waiting build is held by the failed settings walk: %+v %v", r, err) - } -} diff --git a/cmd/mesh-controller/settings_walk_test.go b/cmd/mesh-controller/settings_walk_test.go deleted file mode 100644 index 2d9e0c18..00000000 --- a/cmd/mesh-controller/settings_walk_test.go +++ /dev/null @@ -1,152 +0,0 @@ -package main - -import ( - "context" - "reflect" - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq ADR 0344, issue 641: the operator's one Approve on a settings proposal is the act that makes it take -// effect. The warrant sets the layer and opens a settings walk, which sends it to the machines it reaches one at a -// time, judging the module on each; the ask says so before the Approve, and nothing names a push. - -// The order: every machine running the module, or the one the layer names, the control node last. -func TestASettingsWalkGoesToTheMachinesTheLayerReachesTheControlNodeLast(t *testing.T) { - running := []string{"shanks", "novox", "g14", "ace"} - for _, c := range []struct { - node string - want []string - }{ - {"", []string{"ace", "g14", "shanks", "novox"}}, - {"g14", []string{"g14"}}, - {"novox", []string{"novox"}}, - {"elsewhere", nil}, - } { - if got := settingsWalkOrder(c.node, running, []string{"novox"}); !reflect.DeepEqual(got, c.want) { - t.Errorf("a layer on %q goes to %v, want %v", c.node, got, c.want) - } - } -} - -// A settings walk sends a machine only when no other walk judges there. -func TestASettingsWalkWaitsWhileAnotherWalkJudgesOnTheMachine(t *testing.T) { - judging := &inventory.PlanGate{Machines: []string{"g14"}} - passed := &inventory.PlanGate{Machines: []string{"g14"}, Verdict: inventory.GatePassed} - plans := []inventory.Plan{ - {ID: "settings-1", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: judging}}, - {ID: "release-2", State: inventory.PlanRolling, Release: &inventory.PlanRelease{Gate: passed}}, - {ID: "plan-3", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{"app": {Gate: judging}}}, - {ID: "plan-4", State: inventory.PlanDone, Modules: map[string]*inventory.PlanModule{"app": {Gate: judging}}}, - } - if id := judgingIn(plans, "settings-1", "g14"); id != "plan-3" { - t.Errorf("on g14 %q judges, want plan-3 (itself, a passed gate and an ended walk are not judging)", id) - } - if id := judgingIn(plans[1:2], "x", "g14"); id != "" { - t.Errorf("a passed gate judges: %q", id) - } - if id := judgingIn(plans, "x", "shanks"); id != "" { - t.Errorf("on shanks %q judges", id) - } -} - -// The ask says, before the Approve, that the layer is sent at once and judged, and that its end is told as every -// walk's is; neither the ask nor the proposer's answer names a push any more. -func TestTheAskSaysAnApprovedLayerIsSentAtOnce(t *testing.T) { - for _, c := range []struct { - in proposeInput - where string - had bool - }{ - {proposeInput{module: "mounts", node: "shanks", values: mountsSources}, "shanks", false}, - {proposeInput{module: "notes", values: map[string]any{"x": "1"}}, "every machine running notes", true}, - {proposeInput{module: "notes", node: "laptop", clear: true}, "laptop", true}, - } { - r := newProposerRig(t) - if c.had { - r.before, r.had = map[string]any{"x": "0"}, true - } - words, err := r.pr.propose(context.Background(), c.in) - if err != nil { - t.Fatal(err) - } - q := r.askSent(t) - if err := q.Check(r.now); err != nil { - t.Fatalf("the ask is refused: %v", err) - } - // At a glance (ADR 0345): the summary says it is sent at once by a walk, one machine at a time; the rest of - // what Approve does is the Details', which the router notes after the answers. - if !strings.Contains(q.Explanation, "sent at once by a walk to "+c.where+", one machine at a time, each judged") || - !strings.Contains(q.Details, "and sent at once to "+c.where+", one machine at a time, each judged, "+ - "every build there kept; its end is told as every walk's is") { - t.Errorf("the ask does not say the layer is sent at once to %s:\n%s", c.where, q.Explanation) - } - approve := q.Options[0] - if approve.Label != "Approve" || !strings.Contains(approve.Does, "sent to "+c.where+" at once") { - t.Errorf("Approve does %q", approve.Does) - } - if !strings.Contains(words, "a walk sends it to "+c.where+" at once") { - t.Errorf("the proposer's answer:\n%s", words) - } - for _, said := range []string{q.Explanation, q.Whole, approve.Does, words} { - if strings.Contains(said, "push") { - t.Errorf("a push is still named, as if a second act were needed:\n%s", said) - } - } - } -} - -// ---- on the real stores ----------------------------------------------------------------------------- - -// The warrant sets the layer and opens the walk that sends it, naming the machines it reaches; a layer for a -// machine that does not run the module opens none and says so. - -// walkMesh is a mesh where `app` runs c1 on anchor and laptop and nothing waits for a gate: every send applied at -// once, recorded, and judged from the reports, with a machine named broken failing what it was sent. -type walkMesh struct { - open *stores - sent [][]string - broken map[string]bool - // kept says, per send, whether it kept every build of its machine; moves is the data a machine's send would move. - kept []bool - moves map[string][]dataMove - heard map[string]bool -} - -func (w *walkMesh) walk(t *testing.T) inventory.Plan { - t.Helper() - plans, err := w.open.inventory.RecentPlans(t.Context(), 10) - if err != nil { - t.Fatal(err) - } - for _, p := range plans { - if p.Release != nil && p.Release.Settings != nil { - return p - } - } - t.Fatal("no settings walk") - return inventory.Plan{} -} - -// The walk sends the first machine alone and judges the module there before it sends the next, and ends -// delivered on both; nothing waited for a gate, so each send carried the layer alone. - -// A machine that fails what it was sent stops the walk there: the next is not sent, the layer stays, and the -// failure holds back no walk of waiting builds. - -// Each send of a settings walk keeps every build of its machine: the layer goes, and a build held for a person or -// waiting for a gate stays where it is (ADR 0344 §3). - -// A machine whose send would move a running module's data stops the walk there, failed and said with the push that -// sends it on a person's word (ADR 0217 §3); one not heard from is passed over and named; and a walk that reached -// no machine is not said delivered. - -// The gate keeps the module's build its send kept, so a crossing send — a merge's walk sending the machine a new -// build of the module while the settings gate judges there — supersedes it, and its report is never read as the -// settings send's (issue 641 review, finding 1). - -// A send refusal that cannot clear by itself — what the machine was last sent is not known, or a build it runs is -// gone — fails the walk with why, rather than keeping it open and holding the walk of waiting builds; and a -// failure names the machines passed over as away (issue 641 review, findings 2 and 3). diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index 7ee34b89..dbcf95b6 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -58,8 +58,6 @@ const ( controlNodeUrgentAfter = 30 * time.Minute // reportAtLeast is the least a machine is given to report a send (S2). reportAtLeast = 2 * time.Minute - // tierAtLeast is the least a plan's tier is given (S3), the bound `status` calls a plan late at. - tierAtLeast = planWaitBound // loopDeafAfter is how long the event loop may take nothing while its consumers hold some (S4). loopDeafAfter = 2 * time.Minute // callDefault is the bound of a verb that declares none (S7); push's and build's are longer. @@ -102,20 +100,6 @@ var signalsTable = []signalRow{ newest: func(f *signalFacts) time.Time { return newestOf(f.machines, func(m machineFacts) time.Time { return m.lastHeard }) }}, - {Row: "S2", Signal: "report after a send", Emitter: "node-engine", Trigger: "each declaration sent", - Bound: "max(2 min, 3 × that machine's last apply duration), provisional; not while the machine is silent or asleep", - Kind: "sent-not-reported", Severity: conditions.Warning, Phase: 1, - needs: func(f *signalFacts) error { return f.machinesErr }, watch: watchReports, - newest: func(f *signalFacts) time.Time { - return newestOf(f.machines, func(m machineFacts) time.Time { return m.reportedAt }) - }}, - {Row: "S3", Signal: "plan tier progress", Emitter: "controller's plan", Trigger: "each tier entered", - Bound: "max(30 min, 3 × the p90 of that repository's measured tiers), provisional", - Kind: "stalled", Severity: conditions.Warning, Phase: 1, - needs: func(f *signalFacts) error { return f.plansErr }, watch: watchPlans, - newest: func(f *signalFacts) time.Time { - return newestOf(f.plans, func(p planFacts) time.Time { return p.entered }) - }}, {Row: "S4", Signal: "the controller's event loop takes a message", Emitter: "controller", Trigger: "while its consumers have pending messages", Bound: "2 min", Kind: "controller-deaf", Severity: conditions.Urgent, Phase: 1, @@ -189,58 +173,6 @@ var signalsTable = []signalRow{ newest: func(f *signalFacts) time.Time { return newestOf(f.handActs, func(a link.HandAct) time.Time { return a.At }) }}, - {Row: "S16", Signal: "a walk waiting for its delivery's word is let go", Emitter: "mesh-delivery, through deliver", - Trigger: "each merge whose walk waits (novox/hq ADR 0239)", - Bound: "30 min, then urgent after 4 h — raised by the controller whatever mesh-delivery says of itself, " + - "naming `plans go` as the way on", - Kind: kindWalkWaiting, Severity: conditions.Warning, Phase: 3, - needs: func(f *signalFacts) error { return f.plansErr }, watch: watchWaits, - newest: func(f *signalFacts) time.Time { - return newestOf(f.waits, func(w waitFacts) time.Time { return w.since }) - }}, - {Row: "S18", Signal: "a batch of merges is cut into its walk", Emitter: "controller's merge window", - Trigger: "each batch (novox/hq ADR 0276)", - Bound: "a minute past merge-window-at-most, while no walk is open: the controller failed to cut it", - Kind: kindBatchNotCut, Severity: conditions.Warning, Phase: 3, - needs: func(f *signalFacts) error { return f.plansErr }, watch: watchBatchesNotCut, - newest: func(f *signalFacts) time.Time { - return newestOf(f.batches, func(b batchFacts) time.Time { return b.atMost }) - }}, - {Row: "S19", Signal: "a batch waiting behind an open walk is cut when it ends", Emitter: "controller's merge window", - Trigger: "each batch closed while a walk is open (novox/hq ADR 0276)", - Bound: "the walk's bound, a tier's bound for each of its tiers, from when the batch closed: naming the walk", - Kind: kindBatchBehindWalk, Severity: conditions.Warning, Phase: 3, - needs: func(f *signalFacts) error { return f.plansErr }, watch: watchBatchesBehind, - newest: func(f *signalFacts) time.Time { - return newestOf(f.batches, func(b batchFacts) time.Time { return b.closed }) - }}, - {Row: "S22", Signal: "the last member of a delivery group merges", Emitter: "mesh-delivery's group, held by the controller's merge window", - Trigger: "each batch held for a group with a member still to merge (novox/hq ADR 0418)", - Bound: "merge-window-at-most from the group's first merge: naming the member and why it did not merge; " + - "nothing of the group is delivered while it waits", - Kind: kindGroupNotMerged, Severity: conditions.Warning, Phase: 3, - needs: func(f *signalFacts) error { return f.plansErr }, watch: watchGroupsNotMerged, - newest: func(f *signalFacts) time.Time { - return newestOf(f.batches, func(b batchFacts) time.Time { return b.created }) - }}, - {Row: "S23", Signal: "a delivery plan that failed is asked of the operator", Emitter: "controller's plan", - Trigger: "each delivery plan in state failed, not a rollout of held updates and not one a person stopped (novox/hq ADR 0449)", - Bound: "none: raised when the plan fails, naming the modules that failed, their machines and the failure's own " + - "words, with Retry when `plans retry` would run and the refusal's words when it would not; kept a week " + - "from the plan's last change, and ended when the plan is no longer failed", - Kind: kindPlanFailed, Severity: conditions.Warning, Phase: 3, - needs: func(f *signalFacts) error { return f.failedErr }, watch: watchPlansFailed, - newest: func(f *signalFacts) time.Time { - return newestOf(f.failed, func(p failedPlanFacts) time.Time { return p.since }) - }}, - {Row: "S24", Signal: "a move of a recorded module that waits for a person's push is asked of the operator", - Emitter: "controller's plan", Trigger: "each machine with a move of a recorded module held for a person (novox/hq ADR 0377, issue 1732)", - Bound: "none: raised on the first look that finds the machine holding such a move, naming the modules and the " + - "machine, with Release (the machine's push) as the answer; ended when the machine no longer waits, whoever " + - "pushed it. A hold a person chose (a module set to record with a why, the bus) is not asked.", - Kind: kindMovesHeld, Severity: conditions.Warning, Phase: 3, - needs: func(f *signalFacts) error { return f.heldErr }, watch: watchMovesHeld, - newest: func(f *signalFacts) time.Time { return time.Time{} }}, {Row: "S25", Signal: "a module left out of a send because the artifact store lacks an image it names", Emitter: "controller's send", Trigger: "each send whose declaration names an image of the store that the store does not hold (novox/hq issue 1742)", Bound: "none: raised on the first look at a registered build that names such an image, naming the module, each " + @@ -249,34 +181,6 @@ var signalsTable = []signalRow{ Kind: kindImageMissing, Severity: conditions.Warning, Phase: 3, needs: func(f *signalFacts) error { return f.imagesErr }, watch: watchMissingImages, newest: func(f *signalFacts) time.Time { return time.Time{} }}, - {Row: "S20", Signal: "a send refused for the assignment generation it was composed from", - Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)", - Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " + - "machine applied; cleared an hour after the last", - Kind: kindOlderGeneration, Severity: conditions.Warning, Phase: 2, - needs: func(f *signalFacts) error { return f.refusedSendsErr }, watch: watchGenerationRefusals, - newest: func(f *signalFacts) time.Time { - return newestOf(f.refusedSends, func(s inventory.Send) time.Time { - if s.RefusedAt == nil { - return time.Time{} - } - return *s.RefusedAt - }) - }}, - {Row: "S17", Signal: "a send held for the bus's planned step is told to a person", Emitter: "controller's plan", - Trigger: "each send refused because it would replace the bus outside its step (novox/hq issue 336)", - Bound: "none: raised at the first refusal, for the operator, naming what waits, the bus build from and to, " + - "since when and the mesh-controller.bus call; cleared once the bus's machine has been sent the build the mesh holds", - Kind: kindBusStepWaiting, Severity: conditions.Warning, Phase: 3, - needs: func(f *signalFacts) error { - if f.plansErr != nil { - return f.plansErr - } - return f.busErr - }, watch: watchBusWaits, - newest: func(f *signalFacts) time.Time { - return newestOf(f.bus.waits, func(w busWaitFacts) time.Time { return w.since }) - }}, {Row: "S21", Signal: "a clean apply", Emitter: "node-engine", Trigger: "each report of a declaration refused or applied only in part (novox/hq issue 233)", Bound: "one apply interval (5 min) past the first report of the same failure, judged by time while the machine " + "is heard — the node-engine says a re-apply only when how it went changed; not while it is silent or " + @@ -393,105 +297,6 @@ func watchReports(f *signalFacts) []conditions.Observation { return out } -func watchPlans(f *signalFacts) []conditions.Observation { - var out []conditions.Observation - for _, p := range f.plans { - // Held only by the bus's planned step (S17, novox/hq issue 336): a wait - // for a person, said as itself, not a walk running late. - if p.bus { - continue - } - in := f.now.Sub(p.entered) - if in <= p.bound { - continue - } - out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: "stalled", - Headline: "Delivery of " + repoName(p.repository) + " is stuck halfway", - Explanation: fmt.Sprintf("Its walk across the machines has been at the same step for %s, longer than "+ - "usual. Nothing is lost, and the mesh keeps it where it is.", humanDuration(f.now.Sub(p.entered))), - Resolved: "Delivery of " + repoName(p.repository) + " moves again", - Severity: conditions.Warning, - Summary: fmt.Sprintf("the plan for %s %s has been at tier %d of %d since %s (bound %s): %s", - p.repository, short(p.commit), p.tier+1, p.tiers, p.entered.UTC().Format("2006-01-02 15:04 MST"), - ago(p.bound), p.waiting), - Said: fmt.Sprintf("at tier %d for %s: %s", p.tier+1, ago(in), p.waiting)}) - } - return out -} - -// kindPlanFailed is S23's kind: plan..plan-failed. -const kindPlanFailed = "plan-failed" - -// planFailedWithin is how long a failed plan is kept as a condition from its last change (ADR 0449 §1 names no -// bound; a failed plan stays failed for ever, and a condition for every one the mesh has ever had is no question to -// the operator). A plan failed longer ago than this is left to `plans`. -const planFailedWithin = 7 * 24 * time.Hour - -// watchPlansFailed is S23: a delivery plan that failed, said to the operator at once with a Retry answer when -// `plans retry` would run it (novox/hq ADR 0449 §1). Never a retry of its own: the answer is the operator's (§2). -// The condition ends when the plan is no longer failed, because it is no longer among the facts. -func watchPlansFailed(f *signalFacts) []conditions.Observation { - var out []conditions.Observation - for _, p := range f.failed { - if f.now.Sub(p.since) > planFailedWithin { - continue - } - headline, explanation, needs, actions, resolved := planFailedWords(p) - out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.id, Kind: kindPlanFailed, - Also: p.machines(), Severity: conditions.Warning, Resolver: conditions.ResolverOperator, - Summary: fmt.Sprintf("the delivery plan %s of %s %s failed: %s", p.id, p.repository, short(p.commit), - p.failureSaid()), - Said: p.failureSaid() + p.refusalSaid(), - Headline: headline, Explanation: explanation, Needs: needs, Actions: actions, Resolved: resolved}) - } - return out -} - -// kindWalkWaiting is S16's kind: plan..waiting. -const kindWalkWaiting = "waiting" - -// watchWaits is S16: a walk that has waited for its delivery's word past its bound, said by the controller -// itself — a delivery's owner that is up and never says go (a bug, a delivery stuck in its own table) would -// otherwise leave a merge waiting for ever with nothing open (novox/hq ADR 0239). -func watchWaits(f *signalFacts) []conditions.Observation { - var out []conditions.Observation - for _, w := range f.waits { - in := f.now.Sub(w.since) - if in <= waitBound { - continue - } - severity := conditions.Warning - if in > waitUrgentAfter { - severity = conditions.Urgent - } - out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: w.id, Kind: kindWalkWaiting, - Severity: severity, - Summary: fmt.Sprintf("the walk of %s %s has waited %s for %s's word to start: `mesh-delivery.show` for the "+ - "deliveries that landed as %s says why; `plans go %s --why …` starts it by hand", w.repository, - short(w.commit), ago(in), w.awaits, mergesWords(w), w.id), - Said: fmt.Sprintf("waiting since %s for %s", w.since.UTC().Format(time.RFC3339), w.awaits), - Headline: deliveryName(w.modules, w.repository) + " waiting to start", - Explanation: walkWaitingWords(w, in, severity), - Needs: waitingNeeds(severity), - Actions: waitingActions(w.id, severity), - Resolved: deliveryName(w.modules, w.repository) + " no longer waiting"}) - } - return out -} - -// mergesWords is the merges a waiting walk answers (novox/hq ADR 0276): every delivery it carries, not one -// commit that a supersession may already have ended (issue 362). Its own commit for a walk naming none. -func mergesWords(w waitFacts) string { - if len(w.merges) == 0 { - return short(w.commit) - } - var out []string - for _, m := range w.merges { - out = append(out, m.Repository+"@"+short(m.Commit)) - } - return readableList(out) -} - func watchLoop(f *signalFacts) []conditions.Observation { if f.loop.pending == 0 { return nil diff --git a/cmd/mesh-controller/signals_test.go b/cmd/mesh-controller/signals_test.go index bbb0f09a..08976a0b 100644 --- a/cmd/mesh-controller/signals_test.go +++ b/cmd/mesh-controller/signals_test.go @@ -30,8 +30,6 @@ func calm(now time.Time) *signalFacts { machines: []machineFacts{{name: "anchor", control: true, lastHeard: now.Add(-10 * time.Second), sentAt: now.Add(-time.Hour), reportedCurrent: true, reportedAt: now.Add(-59 * time.Minute), lastApply: 20 * time.Second, tools: true, toolsHeard: now.Add(-10 * time.Second), toolsEvery: time.Minute}}, - plans: []planFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee00", tier: 0, tiers: 2, - entered: now.Add(-time.Minute), bound: 30 * time.Minute, waiting: "building"}}, loop: loopFacts{took: now.Add(-time.Second), pending: 1}, selfCheck: selfCheckFacts{last: now.Add(-time.Minute), every: 5 * time.Minute}, lostConsumers: map[string]bool{}, epochs: map[int64]inventory.Epoch{}, @@ -55,18 +53,6 @@ var suppressions = map[string]suppression{ inside: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute + time.Second) }, past: func(f *signalFacts) { f.machines[0].lastHeard = f.now.Add(-3*time.Minute - time.Second) }, }, - "S2": { - inside: func(f *signalFacts) { - f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-110*time.Second), false - }, - past: func(f *signalFacts) { - f.machines[0].sentAt, f.machines[0].reportedCurrent = f.now.Add(-121*time.Second), false - }, - }, - "S3": { - inside: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-29 * time.Minute) }, - past: func(f *signalFacts) { f.plans[0].entered = f.now.Add(-31 * time.Minute) }, - }, "S4": { inside: func(f *signalFacts) { f.loop.took = f.now.Add(-119 * time.Second) }, past: func(f *signalFacts) { f.loop.took = f.now.Add(-121 * time.Second) }, @@ -116,57 +102,11 @@ var suppressions = map[string]suppression{ past: func(f *signalFacts) { f.facts.taken = f.now.Add(-49 * time.Hour) }, }, // A walk waiting for its delivery's word for 30 minutes (novox/hq ADR 0239). - "S16": { - inside: func(f *signalFacts) { - f.waits = []waitFacts{{id: "plan-2", repository: "novox/app", commit: "c0ffee11", awaits: "mesh-delivery", - since: f.now.Add(-29 * time.Minute)}} - }, - past: func(f *signalFacts) { - f.waits = []waitFacts{{id: "plan-2", repository: "novox/app", commit: "c0ffee11", awaits: "mesh-delivery", - since: f.now.Add(-31 * time.Minute)}} - }, - }, // A batch still assembling past its maximum with no walk open (novox/hq ADR 0276): a minute's grace. - "S18": { - inside: func(f *signalFacts) { - f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanAssembling, grouped: "novox/app@c0ffee11", - atMost: f.now.Add(-59 * time.Second), closed: f.now.Add(-59 * time.Second)}} - }, - past: func(f *signalFacts) { - f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanAssembling, grouped: "novox/app@c0ffee11", - atMost: f.now.Add(-61 * time.Second), closed: f.now.Add(-61 * time.Second)}} - }, - }, // A batch held for a group whose last member has not merged by the window's maximum after the first (ADR 0418). - "S22": { - inside: func(f *signalFacts) { - f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanAssembling, grouped: "novox/app@c0ffee11", - holds: map[string]string{"feat/g": "group feat/g has a member still to merge: novox/two (ready)"}, - created: f.now.Add(-9 * time.Minute), atMostWindow: 10 * time.Minute}} - }, - past: func(f *signalFacts) { - f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanAssembling, grouped: "novox/app@c0ffee11", - holds: map[string]string{"feat/g": "group feat/g has a member still to merge: novox/two (ready)"}, - created: f.now.Add(-11 * time.Minute), atMostWindow: 10 * time.Minute}} - }, - }, // A failed delivery plan is said when it fails and kept a week from its last change (novox/hq ADR 0449). - "S23": { - inside: func(f *signalFacts) { - f.failed = []failedPlanFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee11", modules: []string{"app"}, - since: f.now.Add(-planFailedWithin - time.Second)}} - }, - past: func(f *signalFacts) { - f.failed = []failedPlanFacts{{id: "plan-1", repository: "novox/app", commit: "c0ffee11", modules: []string{"app"}, - since: f.now.Add(-time.Second)}} - }, - }, // A machine holding a move of a recorded module for a person is asked at its first look (novox/hq issue 1732). // Inside: nothing waits there. - "S24": { - inside: func(f *signalFacts) { f.held = nil }, - past: func(f *signalFacts) { f.held = []heldFacts{aHeldMailu()} }, - }, // A registered build naming an image the artifact store lacks is raised at the first look (novox/hq issue 1742). "S25": { inside: func(f *signalFacts) { f.images = nil }, @@ -175,37 +115,10 @@ var suppressions = map[string]suppression{ }, }, // A batch queued behind an open walk past that walk's bound, named. - "S19": { - inside: func(f *signalFacts) { - f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanQueued, grouped: "novox/app@c0ffee11", - closed: f.now.Add(-59 * time.Minute), behind: "plan-1", walkBound: time.Hour}} - }, - past: func(f *signalFacts) { - f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanQueued, grouped: "novox/app@c0ffee11", - closed: f.now.Add(-61 * time.Minute), behind: "plan-1", walkBound: time.Hour}} - }, - }, // A send refused because it would replace the bus outside its planned step: said at its first refusal, // whatever the bound (novox/hq issue 336). Inside: a new bus build waits, and no send was refused for it. - "S17": { - inside: func(f *signalFacts) { - f.bus = busFacts{module: "nats", to: "32307bd1bbbb", from: map[string]string{"anchor": "88135ad0aaaa"}, - machines: []string{"anchor"}} - }, - past: func(f *signalFacts) { - f.bus = busFacts{module: "nats", to: "32307bd1bbbb", from: map[string]string{"anchor": "88135ad0aaaa"}, - machines: []string{"anchor"}, waits: []busWaitFacts{{plan: "plan-1", repository: "novox/app", - commit: "c0ffee001122", modules: []string{"app"}, since: f.now.Add(-time.Second)}}} - }, - }, // A send refused by its machine for the generation it was composed from (novox/hq issue 234): said at // once, naming its sender, for an hour after. Inside: the last such refusal was more than an hour ago. - "S20": { - inside: func(f *signalFacts) { - f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-61 * time.Minute))} - }, - past: func(f *signalFacts) { f.refusedSends = []inventory.Send{refusedSend(f.now.Add(-time.Second))} }, - }, // A machine whose package manager does not answer (novox/hq issue 233): said once its first report of the // failure is older than one apply interval while it is heard. Inside: exactly five minutes old. "S21": { @@ -526,7 +439,7 @@ func TestABlindWatchdogSaysSoAndClearsNothing(t *testing.T) { for _, c := range open { keys = append(keys, c.Key) } - for _, want := range []string{"machine.anchor.silent", "probe.S1.failed", "probe.S2.failed", "probe.S11.failed"} { + for _, want := range []string{"machine.anchor.silent", "probe.S1.failed", "probe.S11.failed"} { if !slices.Contains(keys, want) { t.Errorf("%s is not open while the machines cannot be read: %v", want, keys) } @@ -608,21 +521,6 @@ func TestALeaseLostOrMissingIsSaid(t *testing.T) { } } -// **A walk waiting four hours for its delivery's word is urgent** (S16, novox/hq ADR 0239), and says how on. -func TestAWalkWaitingFourHoursIsUrgentAndNamesTheWayOn(t *testing.T) { - now := time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) - f := calm(now) - f.waits = []waitFacts{{id: "plan-2", repository: "novox/app", commit: "c0ffee11", awaits: "mesh-delivery", - since: now.Add(-4*time.Hour - time.Minute)}} - got := watchWaits(f) - if len(got) != 1 || got[0].Severity != conditions.Urgent || got[0].Key() != "plan.plan-2.waiting" { - t.Fatalf("four hours of waiting raised %+v", got) - } - if !strings.Contains(got[0].Summary, "plans go plan-2") || !strings.Contains(got[0].Summary, "mesh-delivery") { - t.Fatalf("it does not say how on: %q", got[0].Summary) - } -} - // **A drill is a person's deliberate test, never a repair** — the log of 2026-10-07: two drills of ADR // 0240 recorded through `hand-act record --cause drill` before `hand-act drill` existed raised // `mesh.hand-acts.drill.healer-wanted`. A drill through its own verb never counts, the two recorded diff --git a/cmd/mesh-controller/silent_mid_tier_integration_test.go b/cmd/mesh-controller/silent_mid_tier_integration_test.go deleted file mode 100644 index 4d403b24..00000000 --- a/cmd/mesh-controller/silent_mid_tier_integration_test.go +++ /dev/null @@ -1,207 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "fmt" - "strings" - "sync" - "testing" - "time" - - "github.com/nats-io/nats.go" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -// 1160/AC5, live, with the store and a real bus and no ticker: a plan of two tiers whose first tier has a machine that -// never reports, and whose second tier waits on the first (the plan's own order), passes its first tier after the bound, -// the silent key's expiry the only trigger, and sends its second tier to the machine that reported alone (issue 1160). -func TestAPlanPassesATierWhoseMachineNeverReportsAfterTheBoundAndSendsTheNextToTheOthers(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - - // A quarter of an hour of silence is two seconds here. - wasSilent := applySilentAfter - applySilentAfter = 2 * time.Second - t.Cleanup(func() { applySilentAfter = wasSilent }) - - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - legacy, err := js.Conn().JetStream() - if err != nil { - t.Fatal(err) - } - checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy} - t.Cleanup(func() { checkEvents = nil }) - sentHeard := make(chan *nats.Msg, 32) - sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeySent), sentHeard) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - - wasSaved := inventory.PlanSaved - inventory.PlanSaved = servingPlanSaved(ctx, checkEvents, open) - t.Cleanup(func() { inventory.PlanSaved = wasSaved }) - - // The build machine and the machines, as far as the plan meets them: an ask is counted, a send is recorded and kept. - var mu sync.Mutex - var sends [][]string - asksRecorded(t) // each module is taken from a done key as the walk reaches it - // Except tier 1's module b, whose key is not built until the test says so: a walk that takes it in the same look goes on - // through tier 1's send and overwrites the note of the move before the test reads it (see away_integration_test.go). - heldB := holdKeys(t, "b") - digests := 0 - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - mu.Lock() - defer mu.Unlock() - sends = append(sends, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - builds := map[string]string{} - for _, m := range []string{"a", "b"} { - builds[m] = current[m].Commit - } - for _, node := range names { - digests++ - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), fmt.Sprintf("d-%s-%d", node, digests), builds); err != nil { - return nil, err - } - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - - register := func(m, commit string, asked time.Time) { - t.Helper() - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit}, - inventory.Source{Repository: "novox/" + m, Seat: "git", Path: "modules/" + m, BuiltFrom: commit, Head: commit, - Asked: asked}); err != nil { - t.Fatal(err) - } - } - record := func(id, m, commit string, at time.Time) { - t.Helper() - b := inventory.Build{ID: id, Module: m, Commit: commit, Repository: "novox/" + m, Path: "modules/" + m, Asked: at, At: at} - b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit}) - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - old := time.Now().Add(-2 * time.Hour) - for _, m := range []string{"a", "b"} { - record("build-"+m+"-1", m, "c1", old) - register(m, "c1", old) - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, m); err != nil { - t.Fatal(err) - } - } - } - for _, n := range []string{"anchor", "laptop"} { - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"a": "c1", "b": "c1"}); err != nil { - t.Fatal(err) - } - } - record("build-a-2", "a", "c2", time.Now().Add(-time.Minute)) - register("a", "c2", time.Now().Add(-time.Minute)) - - // A machine's report on what it was sent, heard as the controller hears it. - listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}} - seq := int64(0) - reports := func(node string) { - t.Helper() - last, found, err := inv.SentTo(ctx, node) - if err != nil || !found { - t.Fatalf("%s was sent nothing: %v", node, err) - } - seq++ - body, _ := json.Marshal(map[string]any{"node": node, "applied": []string{"x.server"}, "declared": last.Digest, - "report_sequence": seq}) - var report link.Report - if err := json.Unmarshal(body, &report); err != nil { - t.Fatal(err) - } - if _, err := listener.Heard(ctx, report); err != nil { - t.Fatal(err) - } - } - plan := func() inventory.Plan { - t.Helper() - p, err := inv.PlanByID(ctx, "plan-1160") - if err != nil { - t.Fatal(err) - } - return p - } - - built := time.Now().UTC() - p := inventory.Plan{ID: "plan-1160", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", Created: time.Now().UTC(), - State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, - Delivery: &inventory.PlanDelivery{Order: []inventory.Edge{{From: "b", To: "a", Kind: edgeChangeOrder}}}, - Modules: map[string]*inventory.PlanModule{ - "a": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-a-2"}, - "b": {}, - }} - first, stopFirst := context.WithCancel(ctx) - defer stopFirst() - waitFirst := startRolloutEvents(first, open, js) - // Ends with the test before the cleanups restore what the look reads, which run after it (issue 1338). - t.Cleanup(func() { stopFirst(); waitFirst() }) - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - eventually(t, "tier 0 sent to both machines", func() bool { - mu.Lock() - defer mu.Unlock() - return len(sends) == 1 - }) - reports("anchor") - time.Sleep(time.Second) - if got := plan().Tier; got != 0 { - t.Fatalf("1160/AC5: the plan went on to tier %d before the bound, with laptop not heard from", got) - } - // The note of the move to tier 1 is read in the step that sees it: later steps of the tier say their own. - var moved string - eventually(t, "tier 1 passed once laptop's key expired", func() bool { - if q := plan(); q.Tier == 1 { - moved = q.Note - return true - } - return false - }) - if !strings.Contains(moved, "laptop") || strings.Contains(moved, "anchor") { - t.Errorf("1160/AC5: the plan says %q; want laptop named silent and not anchor", moved) - } - if b := plan().Modules["b"]; b == nil || !b.Key { - t.Errorf("tier 1's module is not taken from its key: %+v", b) - } - // Tier 1's module is taken from its key once that is built, which the walk's look does by itself. - heldB.Store(false) - askToLook() - eventually(t, "tier 1 sent to anchor alone", func() bool { - mu.Lock() - defer mu.Unlock() - return len(sends) == 2 && strings.Join(sends[1], ",") == "anchor" - }) -} diff --git a/cmd/mesh-controller/silent_mid_tier_test.go b/cmd/mesh-controller/silent_mid_tier_test.go deleted file mode 100644 index f0d64c99..00000000 --- a/cmd/mesh-controller/silent_mid_tier_test.go +++ /dev/null @@ -1,85 +0,0 @@ -package main - -import ( - "slices" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// A machine silent past the bound at a tier between the first and the last is named silent and left as it was, as the -// last tier's is (novox/hq ADR 0282 §1, ADR 0360 §4-5, issue 1160): one rule, walkReports, read by the walk's end and by -// the gates between tiers. - -func silentWalk(now time.Time) (inventory.Plan, []inventory.Reported) { - sent := now.Add(-time.Minute) - long := now.Add(-inventory.ApplySilentAfter - time.Minute) - at := now.Add(-10 * time.Second) - p := inventory.Plan{Tier: 0, Tiers: [][]string{{"engine"}, {"app"}}, Modules: map[string]*inventory.PlanModule{ - "engine": {Rest: map[string]inventory.SentDeclaration{ - "anchor": {Digest: "d-anchor", At: sent}, - "g14": {Digest: "d-g14", At: long}, - "fresh": {Digest: "d-fresh", At: sent}}}, - "app": {}}} - reports := []inventory.Reported{{Node: "anchor", Outcome: inventory.OutcomeApplied, At: &at, Declared: "d-anchor"}} - return p, reports -} - -func needsOf(running ...string) []gateNeed { - return []gateNeed{{module: "engine", running: running, since: func(string) time.Time { return time.Time{} }}} -} - -// 1160/AC1: at a middle tier the machine silent past the bound is named, not waited for, and the plan goes on with the -// machines that reported; one still inside the bound is waited for. -func TestAMiddleTierGateNamesTheSilentMachineAndGoesOnWithThoseThatReported(t *testing.T) { - now := time.Now().UTC() - p, reports := silentWalk(now) - silent := silentNodes(p, reports, now, nil) - waiting, quiet, lost := settleTierGate(needsOf("anchor", "g14"), reports, silent) - if len(waiting) != 0 || !slices.Equal(quiet, []string{"g14"}) || lost { - t.Fatalf("1160/AC1: waiting %v, silent %v, lost %v; want g14 named silent and nothing waited for", waiting, quiet, lost) - } - waiting, quiet, _ = settleTierGate(needsOf("anchor", "g14", "fresh"), reports, silent) - if len(waiting) != 1 || waiting[0] != "engine on fresh" || !slices.Equal(quiet, []string{"g14"}) { - t.Fatalf("1160/AC1: a machine inside the bound is still waited for: waiting %v, silent %v", waiting, quiet) - } -} - -// 1160/AC2: the walk's end and the gate read the same silent machines. -func TestTheGateAndTheWalksEndShareOneRuleOfSilence(t *testing.T) { - now := time.Now().UTC() - p, reports := silentWalk(now) - _, _, ended := walkReports(p, reports, now, nil) - got := silentNodes(p, reports, now, nil) - if len(got) != len(ended) || !got["g14"] || !slices.Equal(ended, []string{"g14"}) { - t.Fatalf("1160/AC2: the gate reads %v silent, the walk's end %v", got, ended) - } -} - -// 1160/AC3: a later tier is not sent to a machine silent. -func TestALaterTierIsNotSentToASilentMachine(t *testing.T) { - got := withoutSilent([]string{"anchor", "g14"}, map[string]bool{"g14": true}) - if !slices.Equal(got, []string{"anchor"}) { - t.Fatalf("1160/AC3: a later tier goes to %v; want anchor alone", got) - } - in := []string{"anchor", "g14"} - withoutSilent(in, map[string]bool{"g14": true}) - if !slices.Equal(in, []string{"anchor", "g14"}) { - t.Fatalf("1160/AC3: the machines running it were changed: %v", in) - } -} - -// 1160/AC4: every machine the gate waits on silent: the plan is lost, not gone on as if delivered. -func TestAGateWhoseEveryMachineIsSilentIsLost(t *testing.T) { - now := time.Now().UTC() - p, reports := silentWalk(now) - reports = nil - p.Modules["engine"].Rest = map[string]inventory.SentDeclaration{ - "g14": {Digest: "d", At: now.Add(-time.Hour)}, "laptop": {Digest: "d", At: now.Add(-time.Hour)}} - silent := silentNodes(p, reports, now, nil) - _, quiet, lost := settleTierGate(needsOf("g14", "laptop"), reports, silent) - if !lost || !slices.Equal(quiet, []string{"g14", "laptop"}) { - t.Fatalf("1160/AC4: lost %v, silent %v; want both named and the gate lost", lost, quiet) - } -} diff --git a/cmd/mesh-controller/states_announced.go b/cmd/mesh-controller/states_announced.go index f4f05611..0d15efb4 100644 --- a/cmd/mesh-controller/states_announced.go +++ b/cmd/mesh-controller/states_announced.go @@ -191,17 +191,7 @@ func composeAnnouncedReads(ctx context.Context, open *stores) { if holder == "" || !behind { return } - held, err := heldMachines(ctx, open, []string{holder}) - if err != nil { - fmt.Printf("states: whether %s may be sent its new user list cannot be read: %v\n", holder, err) - return - } - if why, isHeld := held[holder]; isHeld { - fmt.Printf("states: the user list the states announced compose is not sent to %s, which holds the bus: %s — "+ - "`push %s` sends it\n", holder, strings.Join(why, "; "), holder) - return - } - if _, err := sendRollout(ctx, open, []string{holder}); err != nil { + if _, err := sendToEach(ctx, open, []string{holder}); err != nil { fmt.Printf("states: the user list the states announced compose could not be sent to %s: %v\n", holder, err) return } diff --git a/cmd/mesh-controller/status.go b/cmd/mesh-controller/status.go index 32c9ac25..89504e45 100644 --- a/cmd/mesh-controller/status.go +++ b/cmd/mesh-controller/status.go @@ -152,18 +152,6 @@ func printStatus(asked answers) error { printPending(asked.pending, asked.pendingUnread) - if open, late := openPlans(asked.plans, time.Now(), asked.tierBounds); len(open) > 0 { - fmt.Printf("%d plan(s) open", len(open)) - if late > 0 { - fmt.Printf(", %d in a tier past its bound", late) - } - fmt.Println(":") - for _, p := range open { - fmt.Printf(" %s\n", planLineWith(p, time.Now(), asked.tierBounds.of(p.Repository))) - } - fmt.Println() - } - if len(behind) > 0 { var names []string for m := range behind { @@ -212,10 +200,6 @@ func printStatus(asked answers) error { fmt.Printf("%d machine(s) have never been sent anything:\n %s\n", len(never), strings.Join(never, ", ")) } - for _, n := range sortedKeysOf(stringsKeys(asked.recordedWaits)) { - fmt.Printf(" %s is behind, waiting for a person: %s — `push %s` sends it\n", n, - strings.Join(asked.recordedWaits[n], "; "), n) - } fmt.Printf("\n `push --behind` sends them\n\n") } @@ -496,13 +480,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) { out.lastSentUnread = err.Error() err = nil } - out.plans, err = inv.RecentPlans(ctx, 5) - if err != nil { - return answers{}, err - } - // Whether the build seat takes work, for a plan waiting on it (novox/hq ADR 0219). - // And how long a tier may take, the bound a plan is late at (novox/hq issue 296). - out.tierBounds = readTierBounds(ctx, inv, time.Now()) // And how many repairs were done by hand this week (novox/hq to-be 45 §7) — where there is a bus // to read the log from; a process with none has no log to count. if _, onBus := broker.BusAddress(); onBus == nil { @@ -540,29 +517,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) { if err != nil { return answers{}, err } - // And, of those, what waits for a person's push (novox/hq ADR 0377 §6): a recorded module's move that went out - // with its walk raises nothing here. - if len(out.waiting) > 0 { - f, err := readMoveFacts(ctx, inv) - if err != nil { - return answers{}, err - } - for _, m := range out.waiting { - if m.Never { - continue - } - held, err := recordedWaiting(ctx, inv, f, m.Node) - if err != nil { - return answers{}, err - } - if len(held) > 0 { - if out.recordedWaits == nil { - out.recordedWaits = map[string][]string{} - } - out.recordedWaits[m.Node] = held - } - } - } out.reported, err = inv.LastReports(ctx) if err != nil { return answers{}, err diff --git a/cmd/mesh-controller/status_summary.go b/cmd/mesh-controller/status_summary.go index 790ae212..c52f3c41 100644 --- a/cmd/mesh-controller/status_summary.go +++ b/cmd/mesh-controller/status_summary.go @@ -210,11 +210,6 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e if news { l.summary.nudge() } - // A machine's account of an apply is looked at at once (novox/hq ADR 0406 §5, issue 974): it ends the wait for the - // machine, and the look is what deletes its key, knowing the report was on that send. - if err == nil && report.Declared != "" && report.Superseded == "" && report.Rekey == nil { - askToLook() - } // What it says of its long-running resources (novox/hq ADR 0240): kept, and its modules' conditions // raised or cleared from it. Only from an account of the machine the store took. if err == nil && report.Health != nil && report.Superseded == "" && report.Rekey == nil && report.Node != "" && diff --git a/cmd/mesh-controller/tier_send_integration_test.go b/cmd/mesh-controller/tier_send_integration_test.go deleted file mode 100644 index 5ab4a20f..00000000 --- a/cmd/mesh-controller/tier_send_integration_test.go +++ /dev/null @@ -1,408 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "fmt" - "reflect" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" -) - -func aTierMesh(t *testing.T, modules ...string) *tierMesh { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - tm := &tierMesh{open: open, modules: modules} - tm.keeper, _ = withConditionsInMemory(t) - was := doctorFrom - doctorFrom = &doctor{open: open, keeper: tm.keeper, teller: &conditions.Told{}} - t.Cleanup(func() { doctorFrom = was }) - - register := func(m, commit string, asked time.Time) { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m, BuiltFrom: commit, - Head: commit, Asked: asked}); err != nil { - t.Fatal(err) - } - } - first := map[string]string{} - for _, m := range modules { - for i, commit := range []string{"c1", "c2"} { - at := time.Now().Add(-2 * time.Hour) - if commit == "c2" { - at = time.Now().Add(-time.Minute) - } - b := inventory.Build{ID: fmt.Sprintf("build-%s-%d", m, i+1), Module: m, Commit: commit, - Repository: "novox/mesh-catalog", Path: "modules/" + m, Asked: at, At: at} - b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit}) - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - register(m, "c1", time.Now().Add(-2*time.Hour)) - first[m] = "c1" - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, m); err != nil { - t.Fatal(err) - } - } - } - for _, n := range []string{"anchor", "laptop"} { - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", first); err != nil { - t.Fatal(err) - } - } - for _, m := range modules { - register(m, "c2", time.Now().Add(-time.Minute)) - } - - n := 0 - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - tm.sent = append(tm.sent, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - builds := map[string]string{} - for _, m := range tm.modules { - builds[m] = current[m].Commit - } - for _, node := range names { - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, builds); err != nil { - return nil, err - } - if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { - return nil, err - } - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - - wasGather := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}, judged: true, - open: append([]conditions.Condition(nil), tm.conds...)} - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return f, err - } - for _, r := range reports { - f.reports[r.Node] = r - f.served[r.Node] = served{runtime: true, tools: map[string]bool{}} - } - return f, nil - } - t.Cleanup(func() { gatherGateFacts = wasGather }) - - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. - gateWithin, gateEvery = time.Hour, time.Hour - t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) - - built := time.Now().UTC() - states := map[string]*inventory.PlanModule{} - for _, m := range modules { - states[m] = &inventory.PlanModule{State: "built", BuiltAt: &built, Commit: "c2", Build: "build-" + m + "-2"} - } - plan := inventory.Plan{ID: "plan-tier", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", - Created: built, State: inventory.PlanBuilding, Tiers: [][]string{modules}, Modules: states} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - return tm -} - -func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - var modules []string - for i := 1; i <= 12; i++ { - modules = append(modules, fmt.Sprintf("m%02d", i)) - } - tm := aTierMesh(t, modules...) - ctx := t.Context() - - advancePlans(ctx, tm.open) - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: the first machine once, for the whole tier", tm.sent) - } - p := tm.plan(t) - lead := p.Modules["m01"] - if lead.Gate == nil || len(lead.Gate.Carried) != len(modules) || lead.GatedBy != "" { - t.Fatalf("the gate on the first module does not judge the whole send: %+v", lead.Gate) - } - for _, m := range modules { - s := p.Modules[m] - if s.FirstAt == nil || !reflect.DeepEqual(s.First, []string{"anchor"}) || s.Previous != "c1" { - t.Fatalf("%s's first send is not recorded, or what anchor ran before is not c1: %+v", m, s) - } - if m != "m01" && s.GatedBy != "m01" { - t.Fatalf("%s is not gated by the send's gate: %+v", m, s) - } - } - - gateEvery = 0 - for i := 0; i < 6; i++ { - advancePlans(ctx, tm.open) - } - p = tm.plan(t) - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("sent %v: one send per machine for the tier", tm.sent) - } - if p.State != inventory.PlanDone { - t.Fatalf("the plan is %s: %s", p.State, p.Note) - } - for _, m := range modules { - if g := p.Modules[m].Gate; g == nil || g.Verdict != inventory.GatePassed { - t.Fatalf("%s's gate: %+v", m, g) - } - if v, found, err := tm.open.inventory.GateOf(ctx, "build-"+m+"-2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("%s's pass was not kept: %+v %v %v", m, v, found, err) - } - } -} - -func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - tm := aTierMesh(t, "app1", "app2", broker.RuntimeModule) - ctx := t.Context() - inv := tm.open.inventory - // The node tools are held inside their witness window, which the node-engine keeps while they settle: their - // one reading does not count yet, so the core behind raised meanwhile is theirs (novox/hq ADR 0351 decision 4). - gateWitnessWindow = time.Hour - - advancePlans(ctx, tm.open) - if len(tm.sent) != 1 { - t.Fatalf("sent %v", tm.sent) - } - tm.raise("anchor", kindCoreBehind, "anchor runs the node tools it was last sent, not yet reported applied, "+ - "and no plan is rolling them out") - gateEvery = 0 - advancePlans(ctx, tm.open) - g := tm.plan(t).Modules["app1"].Gate - if g.Verdict != "" || !reflect.DeepEqual(g.Failing, []string{broker.RuntimeModule}) { - t.Fatalf("inside the settle window the gate is %+v: only the node tools wait on it", g) - } - - gateBound = -time.Second - advancePlans(ctx, tm.open) - p := tm.plan(t) - g = p.Modules["app1"].Gate - if p.State != inventory.PlanFailed || g.Verdict != inventory.GateFailed || - !reflect.DeepEqual(g.Failing, []string{broker.RuntimeModule}) { - t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, g) - } - current, _ := inv.CurrentBuilds(ctx) - if current[broker.RuntimeModule].Commit != "c1" { - t.Fatalf("the node tools were not put back: %s", current[broker.RuntimeModule].Commit) - } - for _, m := range []string{"app1", "app2"} { - if current[m].Commit != "c2" { - t.Fatalf("%s was put back for the node tools' condition: %s", m, current[m].Commit) - } - if failed, _ := inv.GateFailed(ctx, "build-"+m+"-2"); failed { - t.Fatalf("%s's build was marked failed for the node tools' condition", m) - } - } - // Healthy for the passes asked, the module the gate is kept on keeps its own pass (novox/hq issue 318 - // review): it is not blamed, and not left without a verdict. - if !strings.Contains(p.Note, "app1 passed on its own and is kept") { - t.Fatalf("the plan blames the module its gate was kept on, or leaves it unjudged: %s", p.Note) - } - if v, found, _ := inv.GateOf(ctx, "build-app1-2"); !found || v.Verdict != inventory.GatePassed { - t.Fatalf("app1's own pass was not kept: %+v", v) - } -} - -func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - tm := aTierMesh(t, "app1", "app2", "app3") - ctx := t.Context() - inv := tm.open.inventory - advancePlans(ctx, tm.open) - tm.raise("anchor", "disk-full", "anchor's root file system is full") - gateEvery, gateBound = 0, -time.Second - advancePlans(ctx, tm.open) - p := tm.plan(t) - g := p.Modules["app1"].Gate - if p.State != inventory.PlanFailed || !strings.Contains(g.Why, "anchor as a whole") || - !reflect.DeepEqual(g.Failing, []string{"app1", "app2", "app3"}) { - t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, g) - } - current, _ := inv.CurrentBuilds(ctx) - for _, m := range []string{"app1", "app2", "app3"} { - if current[m].Commit != "c1" { - t.Fatalf("%s was not put back with the send: %s", m, current[m].Commit) - } - } - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"anchor"}}) { - t.Fatalf("sent %v: the first send and one send putting all of it back, and nothing to laptop", tm.sent) - } - if !strings.Contains(p.Note, "put back app1 to c1, app2 to c1, app3 to c1 on anchor, in one send") { - t.Fatalf("the plan does not say what was put back: %s", p.Note) - } -} - -func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - inv := g.open.inventory - // An earlier send carried c2 to anchor before the plan's own. - if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-early", map[string]string{"app": "c2"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, g.open) - if p := g.plan(t); p.Modules["app"].Previous != "c2" { - t.Fatalf("what anchor ran before the send: %+v", p.Modules["app"]) - } - g.health["anchor"] = healthNotYet - gateEvery, gateBound = 0, -time.Second - advancePlans(ctx, g.open) - - p := g.plan(t) - gate := p.Modules["app"].Gate - if p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed || gate.Rollback != gateUnchanged || - !strings.Contains(p.Note, "app was left as it was") { - t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, gate) - } - if failed, _ := inv.GateFailed(ctx, "build-2"); failed { - t.Fatal("a build its send changed nothing of was marked failed") - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { - t.Fatalf("the module was moved: %s", current["app"].Commit) - } - open, _ := g.keeper.Open(ctx) - for _, c := range open { - if c.Kind == kindRollbackFailed || c.Kind == kindRolledBack { - t.Fatalf("said as a rollback: %+v", c) - } - } - - everyKeyDone(t) // the retried module is taken from a done key - g.health["anchor"] = healthGood - said, err := retryPlan(ctx, g.open, "plan-gate") - if err != nil { - t.Fatalf("a plan stopped at a gate that judged no build was not retried: %v", err) - } - p = g.plan(t) - s := p.Modules["app"] - if p.State != inventory.PlanBuilding || s.State != "asked" || !s.Key || s.FirstAt != nil || s.Gate != nil { - t.Fatalf("retried as %q: the plan is %s, app %+v", said, p.State, s) - } -} - -func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - for _, order := range [][]string{{broker.RuntimeModule, "app1"}, {"app1", broker.RuntimeModule}} { - t.Run("gate kept on "+order[0], func(t *testing.T) { - tm := aTierMesh(t, order...) - ctx := t.Context() - inv := tm.open.inventory - tierFacts := gatherGateFacts - var seen []string - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f, err := tierFacts(ctx, open, component) - current, _ := open.inventory.CurrentBuilds(ctx) - seen = append(seen, current[broker.RuntimeModule].Commit) - if current[broker.RuntimeModule].Commit == "c2" { - f.rolledBack["anchor"] = []lease.Rollback{{Component: lease.ComponentNodeTools, - Outcome: lease.OutcomeRolledBack, From: "c2", To: "c1", At: time.Now(), Why: "the node tools did not answer"}} - } - return f, err - } - // app1, judged as the core, counts a reading at most every 100 ms: it cannot have its three in the - // judgings that find the node tools broken, and is judged to its own pass after them (the hold the - // settle time gave before novox/hq ADR 0352 counted readings from the send). - gateEvery, gateWithin, gateBound = 100*time.Millisecond, time.Hour, 10*time.Second - for i := 0; i < 20 && len(seen) < 2; i++ { - advancePlans(ctx, tm.open) - } - if len(seen) < 2 || seen[0] != "c2" || seen[1] != "c1" { - t.Fatalf("the node tools' registered build at each judging: %v; want c2, then c1 at once", seen) - } - if failed, _ := inv.GateFailed(ctx, "build-"+broker.RuntimeModule+"-2"); !failed { - t.Fatal("the node tools' build is not marked failed at once") - } - if p := tm.plan(t); p.State == inventory.PlanFailed { - t.Fatalf("the plan failed at once: %s; want it judging app1 first", p.Note) - } - for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); { - advancePlans(ctx, tm.open) - if tm.plan(t).State == inventory.PlanFailed { - break - } - time.Sleep(20 * time.Millisecond) - } - p := tm.plan(t) - if p.State != inventory.PlanFailed { - t.Fatalf("the plan is %s: %s; want failed, for the node tools", p.State, p.Note) - } - if v, found, _ := inv.GateOf(ctx, "build-app1-2"); !found || v.Verdict != inventory.GatePassed { - t.Fatalf("app1's verdict: %+v; want its own pass (plan: %s)", v, p.Note) - } - if current, _ := inv.CurrentBuilds(ctx); current["app1"].Commit != "c2" || current[broker.RuntimeModule].Commit != "c1" { - t.Fatalf("registered app1 %s, node tools %s; want c2 and c1", current["app1"].Commit, - current[broker.RuntimeModule].Commit) - } - }) - } -} - -func TestASendCarryingTheToolRunnerWaitsForItsWitnessWindow(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - tm := aTierMesh(t, broker.RuntimeModule, "app1") - ctx := t.Context() - // One reading counted per advance, as every test with a gate left open does: a gate judged with no time - // between readings would count a pass at every look and never let an advance rest. - gateEvery, gateWitnessWindow = time.Hour, time.Hour - for i := 0; i < 6; i++ { - advancePlans(ctx, tm.open) - } - p := tm.plan(t) - lead := p.Modules[broker.RuntimeModule].Gate - if lead == nil { - lead = p.Modules["app1"].Gate - } - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != 1 { - t.Fatalf("passed inside the witness window: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead) - } - gateEvery, gateWitnessWindow = 0, 0 - for i := 0; i < 6; i++ { - advancePlans(ctx, tm.open) - } - p = tm.plan(t) - if p.State != inventory.PlanDone || !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent) - } - for _, m := range []string{broker.RuntimeModule, "app1"} { - if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < 1 || - !strings.Contains(g.Why, "the tool runner's rule")) { - t.Fatalf("%s's gate: %+v; want passed on one reading by the tool runner's rule", m, g) - } - } -} diff --git a/cmd/mesh-controller/tier_send_test.go b/cmd/mesh-controller/tier_send_test.go deleted file mode 100644 index 188ec090..00000000 --- a/cmd/mesh-controller/tier_send_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A plan's tier is sent to each machine once, and judged there by one gate (novox/hq issue 281). -// -// On 2026-10-06 a merge rebuilt about seventy modules in one tier, and the plan sent the first machine -// one module at a time — one whole declaration per module, seconds apart. The node-engine set aside one -// declaration for the next, the node tools never settled, the mesh said the machine's core was behind, -// and the gate failed the module it happened to be judging for that machine-level condition; it then -// could not put the module back, having read as "the build it ran before" the very build an earlier send -// of the same tier had already carried there. - -// tierMesh is a mesh with every module of one tier running on anchor and laptop at build c1, a newer -// build c2 of each registered, a plan whose one tier built them all, and every send recorded and -// answered: the machine applies what it is sent and reports it. A judging reads the store's reports, the -// node tools as answering everywhere, and the conditions the test opens. -type tierMesh struct { - open *stores - modules []string - sent [][]string - conds []conditions.Condition - keeper *conditions.Keeper -} - -func (tm *tierMesh) plan(t *testing.T) inventory.Plan { - t.Helper() - p, err := tm.open.inventory.PlanByID(t.Context(), "plan-tier") - if err != nil { - t.Fatal(err) - } - return p -} - -// raise opens a condition about a machine itself, raised now: after the send. -func (tm *tierMesh) raise(machine, kind, summary string) { - tm.conds = append(tm.conds, conditions.Condition{Key: "machine." + machine + "." + kind, Kind: kind, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: machine, Machine: machine}, - Raised: time.Now().Add(time.Second), Summary: summary, Source: "D10"}) -} - -// The incident's tier, small: a dozen modules whose first machine is the same are sent there in one -// send, judged by one gate, and the rest of their machines are sent once — two sends, not two dozen. - -// The core behind about the node tools, while the node tools are themselves among what was sent and -// settling, is the node tools' — inside their settle window, the gate's bound — and no failure of the -// modules sent with them: at the bound only the node tools are put back. - -// A machine-level condition about anything else holds the machine back as a whole: everything the send -// moved there fails together, once, with that reason — not one module for it. - -// The machine-level conditions, sorted: one naming a module is that module's; the core behind is the -// core's when the send moved it and nobody's when it did not; anything else is the machine's as a whole. -func TestTheConditionsAboutAMachineItself(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - since := time.Now().Add(-time.Minute) - about := func(kind, id string) conditions.Condition { - return conditions.Condition{Key: "machine." + id + "." + kind, Kind: kind, Raised: time.Now(), - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: id, Machine: "anchor"}} - } - f := gateFacts{judged: true, open: []conditions.Condition{about(kindCoreBehind, "anchor")}} - if w := aboutTheMachine("anchor", []string{"app", "blueman"}, since, f); w.whole != "" || len(w.on) != 0 || len(w.facts.open) != 0 { - t.Errorf("the core behind, with no core component sent, counted against what was sent: %+v", w) - } - if w := aboutTheMachine("anchor", []string{"app", broker.RuntimeModule}, since, f); w.whole != "" || - w.on[broker.RuntimeModule] == "" || w.on["app"] != "" { - t.Errorf("the core behind, with the node tools sent, is not theirs alone: %+v", w) - } - f.open = []conditions.Condition{about("x", "anchor.app")} - if w := aboutTheMachine("anchor", []string{"app", "other"}, since, f); w.on["app"] == "" || w.on["other"] != "" || w.whole != "" { - t.Errorf("a condition naming a module is not that module's: %+v", w) - } - f.open = []conditions.Condition{about("unreachable", "anchor")} - if w := aboutTheMachine("anchor", []string{"app"}, since, f); !strings.Contains(w.whole, "anchor as a whole") { - t.Errorf("a condition about the machine is not the machine's as a whole: %+v", w) - } - f.open[0].Raised = since.Add(-time.Hour) - if w := aboutTheMachine("anchor", []string{"app"}, since, f); w.whole != "" || len(w.facts.open) != 1 { - t.Errorf("a condition older than the send counted: %+v", w) - } -} - -// A module its first send changed nothing of — an earlier send already carried this very build there — -// fails a gate as no verdict on its build: left as it was, never marked failed, nothing said as a -// rollback that failed; and `plans retry` asks it again rather than refusing a build that was never -// judged (the incident's "NOT put back: no build kept"). - -// **In a plan's tier path, a broken module is put back at once too** (novox/hq issue 318 review): the node -// tools' witness reverts them on the first machine. Whether the gate is kept on them or on the module beside -// them, they are registered back and marked in the judging that finds them broken, the plan goes on judging -// the other module to its own pass, and only then fails. - -// **A send carrying the tool runner waits for its witness window** (issue 524 review, novox/hq ADR 0351 -// decision 4): the leaf beside the node tools has its one reading at once, but the send is judged by the tool -// runner's rule: one reading, not before the node-engine's witness window for it has ended. diff --git a/cmd/mesh-controller/unseen_integration_test.go b/cmd/mesh-controller/unseen_integration_test.go deleted file mode 100644 index e15a2acb..00000000 --- a/cmd/mesh-controller/unseen_integration_test.go +++ /dev/null @@ -1,19 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "strings" - "testing" -) - -func TestAPushNamingNoMachineIsRefusedUnlessItSaysAll(t *testing.T) { - err := pushCommand(context.Background(), nil) - if err == nil || !strings.Contains(err.Error(), "--all") { - t.Fatalf("a bare push was not refused: %v", err) - } - if err := pushCommand(context.Background(), []string{"--all", "--behind"}); err == nil { - t.Fatal("--all with --behind was accepted") - } -} diff --git a/cmd/mesh-controller/unseen_test.go b/cmd/mesh-controller/unseen_test.go index 6d8ad915..30bb7413 100644 --- a/cmd/mesh-controller/unseen_test.go +++ b/cmd/mesh-controller/unseen_test.go @@ -148,10 +148,6 @@ func TestTheVerbsCarryReadReplaceAndMove(t *testing.T) { {"settings", map[string]any{"module": "plex", "node": "home"}, []string{"settings", "show", "plex", "--node", "home"}}, {"settings", map[string]any{"module": "plex", "history": "true"}, []string{"settings", "show", "plex", "--history"}}, {"settings", map[string]any{"module": "plex", "values": "{}", "replace": "true"}, []string{"settings", "set", "plex", "{}", "--replace"}}, - {"push", map[string]any{"node": "home", "move": "plex", "why": "w"}, - []string{"push", "home", "--wait", "0s", "--move", "plex", "--why", "w"}}, - {"push", map[string]any{"why": "w"}, []string{"push", "--behind", "--wait", "0", "--why", "w"}}, - {"plan", map[string]any{"node": "home", "diff": "true"}, []string{"plan", "home", "--diff"}}, } { got, err := argvFor(c.verb, c.args) if err != nil || !reflect.DeepEqual(got, c.want) { diff --git a/cmd/mesh-controller/upgrades.go b/cmd/mesh-controller/upgrades.go index d3155a4f..fe2a4e39 100644 --- a/cmd/mesh-controller/upgrades.go +++ b/cmd/mesh-controller/upgrades.go @@ -3,7 +3,6 @@ package main import ( "context" "errors" - "flag" "fmt" "github.com/novox/mesh-controller/internal/gosource" "regexp" @@ -38,54 +37,8 @@ type following struct { // are built on. So "record it" is the absence of an action, and the only thing this has to decide // is whether to act. func (f following) Upgraded(ctx context.Context, u link.Upgraded) error { - inv := f.open.inventory - - // The store read first, and an outage there said as one, so the announcement is held and asked - // again (novox/hq issue 083). Only here: a push that fails further down is not asked again. - decision, err := inv.UpgradeOf(ctx, u.Module) - if err != nil { - return notNow(err) - } - on, err := inv.Running(ctx, u.Module) - if err != nil { - return notNow(err) - } - if len(on) == 0 { - fmt.Printf("%s moved to %s; no machine runs it\n", u.Module, shortCommit(u.Commit)) - return nil - } - if !decision.RollOut { - // Named rather than counted, and said even though nothing happens: an upgrade that was - // deliberately not rolled out and an upgrade that was never noticed look identical in a - // log that only speaks when it acts. - fmt.Printf("%s moved to %s; %s %s behind it, and this mesh records upgrades rather than "+ - "rolling them out — `push --behind` when you want them\n", - u.Module, shortCommit(u.Commit), readableList(on), isAre(len(on))) - return nil - } - - // **A plan that holds the module rolls it out, and this does not** (novox/hq issue 249, ADR - // 0218). A merge's plan builds the module and sends it one machine first, the rest once that one - // has applied it; this announcement arrives as the build registers, and sending here too — one - // machine after another without waiting for any to apply — put the new bundle on every machine in - // the same minute, whatever the plan was waiting for. A move no plan answers (a build asked by - // hand) is still this handler's. - if plans, err := inv.OpenPlans(ctx); err != nil { - return notNow(err) - } else if id := rolledOutByAPlan(plans, u.Module); id != "" { - // Said with its remedy: a plan that ends without sending it — failed, or closed by hand — leaves - // these machines behind, which `status` lists and `push --behind` sends (novox/hq issue 249). - fmt.Printf("%s moved to %s; %s rolls it out to %s — if that plan ends without sending it, "+ - "`status` lists them as behind and `push --behind` sends it\n", - u.Module, shortCommit(u.Commit), id, readableList(on)) - return nil - } - // **No plan holds it: a release plan sends it** (novox/hq ADR 0236). A build asked outside a plan — a - // `rebuild`, a `replay` registered — was sent here one machine after another without any judging; it - // now waits for a gate like any other build, and the release plan walks it through the machines, one - // at a time, each judged. - fmt.Printf("%s moved to %s outside a plan; a release plan sends it to %s, one machine first and then the rest at once, each judged "+ - "(`upgrade backlog` lists what waits)\n", u.Module, shortCommit(u.Commit), readableList(on)) + // Nothing is sent: each machine follows the wants the assigner writes for the new fingerprint (ADR 0539 §5). + fmt.Printf("%s moved to %s\n", u.Module, shortCommit(u.Commit)) return nil } @@ -99,17 +52,6 @@ func askAgainOnGrants(err error) error { return err } -// rolledOutByAPlan is the open plan that will send a module's machines its new build — one holding -// the module that has not finished sending it — or empty when none will (novox/hq issue 249). -func rolledOutByAPlan(plans []inventory.Plan, module string) string { - for _, p := range plans { - if s, holds := p.Modules[module]; p.Open() && holds && (s == nil || (s.SentAt == nil && s.State != "failed")) { - return p.ID - } - } - return "" -} - // readableList names machines the way a sentence does, because this is read by a person deciding // whether an upgrade went where they expected. func readableList(names []string) string { @@ -138,118 +80,6 @@ func isAre(n int) string { return "are" } -// upgradeCommand says what should happen when a module's current version moves (ADR 0162 §3, ADR -// 0235): every module's policy and where it comes from; one module's; or a person's choice for one. -func upgradeCommand(ctx context.Context, args []string) error { - // What waits for a gate, and releasing it by a person's word (ADR 0236). - if len(args) > 0 && (args[0] == "backlog" || args[0] == "release-backlog") { - return backlogCommand(ctx, args[0], args[1:]) - } - set := flag.NewFlagSet("upgrade", flag.ContinueOnError) - together := set.Bool("together", false, - "send every machine running it at once, instead of one machine first") - why := set.String("why", "", "why this choice — required for record; kept and said with the policy") - positionals, err := parseAround(set, args) - if err != nil { - return err - } - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - inv := open.inventory - - if len(positionals) == 0 { - all, err := inv.Upgrades(ctx) - if err != nil { - return err - } - names := make([]string, 0, len(all)) - for n := range all { - names = append(names, n) - } - sort.Strings(names) - for _, n := range names { - u := all[n] - line := fmt.Sprintf("%-28s %-9s %s", n, u.Policy(), u.From) - if u.Why != "" { - line += ": " + u.Why - } - fmt.Println(line) - } - return nil - } - module := positionals[0] - if len(positionals) == 1 { - decision, err := inv.UpgradeOf(ctx, module) - if err != nil { - return err - } - fmt.Println(sayUpgrade(module, decision)) - return nil - } - - decision := inventory.Upgrade{Why: strings.TrimSpace(*why), By: link.Caller()} - switch positionals[1] { - case "roll-out", "roll": - decision.RollOut, decision.Together = true, *together - case "record": - if *together { - // Refused rather than ignored: --together only means anything for a roll-out, and - // accepting it here would store a preference that never applies and looks like it does. - return errors.New("`--together` says how to roll out, so it cannot be given with " + - "`record`, which is the choice not to") - } - if decision.Why == "" { - return fmt.Errorf("holding %s back from every merge is a choice a person reads later: --why "+ - "(novox/hq ADR 0236). Nothing was changed", module) - } - case "default": - if err := inv.ClearUpgradeOf(ctx, module); err != nil { - return err - } - decision, err := inv.UpgradeOf(ctx, module) - if err != nil { - return err - } - fmt.Println(sayUpgrade(module, decision)) - return nil - default: - return fmt.Errorf("upgrade roll-out|record|default — not %q", positionals[1]) - } - if err := inv.SetUpgradeOf(ctx, module, decision); err != nil { - return err - } - decision, err = inv.UpgradeOf(ctx, module) - if err != nil { - return err - } - fmt.Println(sayUpgrade(module, decision)) - return nil -} - -func sayUpgrade(module string, u inventory.Upgrade) string { - from := " (" + u.From - if u.By != "" { - from += ", " + u.By - } - if u.Why != "" { - from += ": " + u.Why - } - from += ")" - if !u.RollOut { - return fmt.Sprintf("when %s moves, the mesh records it and the machines running it are "+ - "reported as behind until a person pushes them%s", module, from) - } - if u.Together { - return fmt.Sprintf("when %s moves, every machine running it is sent the new version "+ - "together%s", module, from) - } - return fmt.Sprintf("when %s moves, one machine running it is sent the new version first and judged at "+ - "the gate; the rest follow once it passes, and a build that fails is put back there%s", module, from) -} - // Announceable is every build this mesh recorded, in the shape the builder announces one. // // **The catalogue asks for this when it starts, and the answer is the graph's foundation** @@ -316,15 +146,8 @@ func unsaved(err error) error { // SourceMoved is the forge announcing a merge. It is put into the open batch (novox/hq ADR 0276), which // becomes one walk when its merge window closes: hearMerge, and cutBatchesHeld in batches.go. func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error { - if err := f.hearMerge(ctx, m, time.Now().UTC()); err != nil { - return err - } - // And the merges of that trunk it did not hear, in the trunk's order (novox/hq ADR 0457): a catch-up that leaves - // something for later is no failure of this merge's hearing, and is never a reason to hear it again. - if !f.oneShot { - trunk := &caughtTrunk{Owner: m.Owner, Repo: m.Repo, Branch: m.Base} - catchUpBeside(func() { f.catchUp(context.WithoutCancel(ctx), trunk) }) - } + // A merge plans nothing here any more: mesh-delivery merges on the Approve, the indexer writes the fingerprints and + // the builder builds them (ADR 0539 §5, §17). The controller holds no batch for it. return nil } @@ -764,35 +587,6 @@ func packagesSaid(read []inventory.ReadRepository, m link.SourceMoved) bool { return false } -// staleIn is the modules whose recorded build source a plan has overtaken (novox/hq ADR 0267): a plan still -// working that has yet to build one, or a plan made after that build which never built it — failed, stopped -// or superseded. What such a module is built from is changing, or changed without a build to say so: a merge -// that added an import to it, and a later one changing only what that import names, would otherwise move -// nothing. Each is read whole, as before, until a build of it works again. -// -// Each is answered with the plan that overtook it, for saying why it is read whole. -func staleIn(read map[string][]inventory.ReadRepository, plans []inventory.Plan) map[string]string { - stale := map[string]string{} - for name, rs := range read { - var since time.Time - for _, r := range rs { - if r.Built.After(since) { - since = r.Built - } - } - for _, p := range plans { - s, in := p.Modules[name] - if !in || (s != nil && (s.State == "built" || s.State == planDeleted)) { - continue - } - if p.Open() || p.Created.After(since) { - stale[name] = p.ID - } - } - } - return stale -} - // lookedAt is when a merge was last acted on for a module that packages another repository's source: its // newest build, or the newest plan that built it or is still building it, whichever is later. A build asked // after a merge clones that repository with the merge in it, so an older merge is history for it; a plan @@ -813,66 +607,10 @@ func lookedAt(name string, read []inventory.ReadRepository, plans []inventory.Pl return at } -// readForPlanning is what each module's build read, as the planner maps a change onto it — for a merge -// acting now, the merge gate, a pull request's check, a delivery's order and the what-if alike, so planning -// and gating cannot disagree (novox/hq ADR 0238): the build sources the newest trunk builds said, but for -// the modules a plan has overtaken (staleIn), and with when each was last looked at (lookedAt). +// readForPlanning is what each module's build read, as a change is mapped onto it — for a merge's reach, a pull request's +// check and the facts alike (novox/hq ADR 0238), so they cannot disagree. No plan overtakes it any more (ADR 0539 §17). func readForPlanning(ctx context.Context, inv *inventory.Inventory) (map[string][]inventory.ReadRepository, error) { - read, err := inv.ReadRepositories(ctx) - if err != nil { - return nil, err - } - // Every plan since the oldest build whose source is recorded: one made after a module's build can have - // overtaken it, however long ago, so no window of recent plans would do. - var oldest time.Time - for _, rs := range read { - for _, r := range rs { - if !r.Built.IsZero() && (oldest.IsZero() || r.Built.Before(oldest)) { - oldest = r.Built - } - } - } - plans, err := inv.PlansSince(ctx, oldest) - if err != nil { - return nil, err - } - return planningView(read, plans), nil -} - -// planningView is readForPlanning over what was read, so a test can hand it records. -func planningView(read map[string][]inventory.ReadRepository, plans []inventory.Plan) map[string][]inventory.ReadRepository { - stale := staleIn(read, plans) - out := make(map[string][]inventory.ReadRepository, len(read)) - for name, rs := range read { - looked := lookedAt(name, rs, plans) - var commits []string - for _, p := range plans { - if st, in := p.Modules[name]; in && p.Commit != "" && (p.Open() || (st != nil && st.State == "built")) { - // Every commit the walk carries (novox/hq ADR 0276): a batch's walk answers one per repository. - for _, c := range p.Carried() { - commits = append(commits, c.Commit) - } - } - } - var kept []inventory.ReadRepository - overtaken, isStale := stale[name] - for _, r := range rs { - if isStale { - if r.Own { - continue - } - r.Paths = nil - } - r.Looked, r.LookedAt = looked, commits - kept = append(kept, r) - } - if isStale { - kept = append(kept, inventory.ReadRepository{Own: true, Whole: "plan " + overtaken + " has not built it yet", - Looked: looked, LookedAt: commits}) - } - out[name] = kept - } - return out + return inv.ReadRepositories(ctx) } // whatTheMergeTouched narrows the modules built from a repository to the ones the merge changed: **a @@ -989,9 +727,9 @@ type mergeReach struct { Already int // Added are the directories the change holds a new module in; Unread the changed files no build reads. Added, Unread []string - // Plan is what moves and everything that follows it along the catalogue's dependencies, tiered — - // what a merge would build. Empty when nothing moves. - Plan inventory.Plan + // Following are the modules that stand on the moved ones along the catalogue's dependencies: what a merge would + // build after them. + Following []string } // Moved are the modules the merge moves itself, by name: touched, deleted and packaging. @@ -1008,36 +746,7 @@ func (r mergeReach) Moved() []string { // Dependents are the modules the plan builds after the moved ones because they stand on them. func (r mergeReach) Dependents() []string { - moved := map[string]bool{} - for _, name := range r.Moved() { - moved[name] = true - } - var out []string - for name := range r.Plan.Modules { - if !moved[name] { - out = append(out, name) - } - } - sort.Strings(out) - return out -} - -// reachOfMerge is what a merge of these changed files into this branch would move and build, without -// acting: the merge handler's own judgement and the release plan's dependency walk. -func reachOfMerge(m link.SourceMoved, entries []inventory.Entry, read map[string][]inventory.ReadRepository, - edges []inventory.Edge) mergeReach { - from, packaging, already := mergeCandidates(m, entries, read) - touched, added, unread := touchedBy(from, entries, m, read) - kept, deleted := splitDeleted(touched, m) - r := mergeReach{Touched: kept, Deleted: deleted, Packaging: packaging, Already: already, Added: added, Unread: unread} - var building []string - for _, e := range append(append([]inventory.Entry{}, kept...), packaging...) { - building = append(building, e.Manifest.Module) - } - if len(building) > 0 { - r.Plan = planOfMerge(m, building, edges) - } - return r + return r.Following } // saidModuleDirs is the directories the announcer says hold a module at the merge commit (novox/hq @@ -1167,3 +876,63 @@ func deletedAtSource(failed string) bool { return strings.Contains(failed, "has no "+moduleManifestFile+" at ") && strings.Contains(failed, "so there is nothing saying what it is") } + +// reachOfMerge is what a merge of these changed files into this branch would move, without acting: a pull request's +// reach, as the check reckons it. +func reachOfMerge(m link.SourceMoved, entries []inventory.Entry, read map[string][]inventory.ReadRepository, + edges []inventory.Edge) mergeReach { + from, packaging, already := mergeCandidates(m, entries, read) + touched, added, unread := touchedBy(from, entries, m, read) + kept, deleted := splitDeleted(touched, m) + r := mergeReach{Touched: kept, Deleted: deleted, Packaging: packaging, Already: already, Added: added, Unread: unread} + moved := r.Moved() + if len(moved) > 0 { + in := map[string]bool{} + for _, name := range moved { + in[name] = true + } + for _, name := range reachableFrom(moved, edges) { + if !in[name] { + r.Following = append(r.Following, name) + } + } + } + return r +} + +// reachableFrom is the moved modules plus everything that depends on them, through every layer: +// what a merge rebuilds. Along the code and build edges only: a module *built by* the build machine +// is not changed by a new build machine, so a built-by edge orders and gates a plan and never +// widens it — the first plan of 2026-10-01 took the whole catalogue along for a controller change. +func reachableFrom(moved []string, edges []inventory.Edge) []string { + in := map[string]bool{} + for _, m := range moved { + in[m] = true + } + for grew := true; grew; { + grew = false + for _, e := range edges { + // Built-by and worker-of order a plan; neither widens it. A new build machine changes + // nothing it builds, and a new controller changes nothing about the holder it orders. A + // packages edge, read from a record made before novox/hq ADR 0267, widens nothing either: + // a shared file moves each module whose build source holds it, directly. + if e.Kind == inventory.EdgeBuiltBy || e.Kind == inventory.EdgeWorkerOf || e.Kind == inventory.EdgePackages || + e.Kind == edgeChangeOrder { + continue + } + if in[e.To] && !in[e.From] { + in[e.From] = true + grew = true + } + } + } + out := make([]string, 0, len(in)) + for m := range in { + out = append(out, m) + } + sort.Strings(out) + return out +} + +// edgeChangeOrder is the edge kind a change order draws. +const edgeChangeOrder = "change-order" diff --git a/cmd/mesh-controller/urgent_unhealthy_test.go b/cmd/mesh-controller/urgent_unhealthy_test.go deleted file mode 100644 index a1be7af0..00000000 --- a/cmd/mesh-controller/urgent_unhealthy_test.go +++ /dev/null @@ -1,93 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A module may declare its unhealthy urgent (novox/hq ADR 0355): its module-unhealthy condition is then raised -// urgent; without the field it stays a warning; a wait for a new login stays a warning whatever the module says. - -// judgedTwice judges one machine's statement twice, over a catalogue in which mail declares its unhealthy -// urgent and notes says nothing, and returns what is open. -func judgedTwice(t *testing.T, unhealthy map[string][]inventory.ResourceHealth) []conditions.Condition { - t.Helper() - ctx := t.Context() - k, _ := withConditionsInMemory(t) - was := readHoldingFor - t.Cleanup(func() { readHoldingFor = was }) - readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) { - return &holding{ctx: context.Background(), open: open, providers: map[string]providerLookup{}, - shelf: map[string]catalogue.Manifest{ - "mail": {Module: "mail", Version: "1", Unhealthy: &catalogue.UnhealthyPolicy{ - Severity: catalogue.UnhealthyUrgent, Why: "mail is refused or stuck for everyone with a mailbox"}}, - "notes": {Module: "notes", Version: "1"}, - }}, nil - } - streaks := map[string]int{} - for look := 1; look <= 2; look++ { - for m := range unhealthy { - streaks[m] = look - } - if err := judgeModuleHealth(ctx, nil, k, "anchor", unhealthy, streaks, time.Now()); err != nil { - t.Fatal(err) - } - } - open, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - return open -} - -func containerDown(module string) inventory.ResourceHealth { - return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module, - State: link.StateUnhealthy, Reason: "down"} -} - -func TestAModuleThatDeclaresItsUnhealthyUrgentIsRaisedUrgent(t *testing.T) { - open := judgedTwice(t, map[string][]inventory.ResourceHealth{"mail": {containerDown("mail")}}) - c := conditionOf(open, "module.mail.anchor.unhealthy") - if c == nil { - t.Fatalf("mail's unhealthy condition was not raised: %+v", open) - } - if c.Severity != conditions.Urgent { - t.Errorf("mail declares its unhealthy urgent, and it was raised %s", c.Severity) - } -} - -func TestAModuleThatSaysNothingOfItsUnhealthyStaysAWarning(t *testing.T) { - open := judgedTwice(t, map[string][]inventory.ResourceHealth{"notes": {containerDown("notes")}}) - c := conditionOf(open, "module.notes.anchor.unhealthy") - if c == nil { - t.Fatalf("notes' unhealthy condition was not raised: %+v", open) - } - if c.Severity != conditions.Warning { - t.Errorf("notes says nothing of its unhealthy, and it was raised %s", c.Severity) - } -} - -// Only the module-unhealthy condition changes (ADR 0355 §4): a wait for a new login (ADR 0254) of a module that -// declares its unhealthy urgent stays a warning. -func TestAWaitForANewLoginStaysAWarningWhenTheModuleDeclaresItsUnhealthyUrgent(t *testing.T) { - open := judgedTwice(t, map[string][]inventory.ResourceHealth{"mail": {relogin("mail", "operator"), - userUnit("mail", "operator")}}) - c := conditionOf(open, "module.mail.anchor.relogin-needed") - if c == nil { - t.Fatalf("mail's relogin-needed condition was not raised: %+v", open) - } - if c.Severity != conditions.Warning { - t.Errorf("a wait for a new login was raised %s; want a warning", c.Severity) - } - if conditionOf(open, "module.mail.anchor.unhealthy") != nil { - t.Errorf("a wait for a new login raised mail's unhealthy condition beside it: %+v", open) - } -} diff --git a/cmd/mesh-controller/verb_looks_test.go b/cmd/mesh-controller/verb_looks_test.go deleted file mode 100644 index ced2f5c3..00000000 --- a/cmd/mesh-controller/verb_looks_test.go +++ /dev/null @@ -1,56 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "sync/atomic" - "testing" - "time" -) - -// A plan moved by a seat verb is looked at as the verb ends (issue 974, review 1): the verb runs as a child process -// whose saves the serving controller does not hear, so `deliver`, which lets a walk go after the operator's approval, -// would leave its first tier waiting for an unrelated event. -func TestAVerbThatMayMoveAPlanHasTheServingControllerLookWhenItEnds(t *testing.T) { - var looks, failures atomic.Int32 - l := lookerCounting(&failures, &looks, nil) - lookerMu.Lock() - theLooker = l - lookerMu.Unlock() - t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - - was := runTheVerb - runTheVerb = func(context.Context, []string) (verbAnswer, error) { return verbAnswer{}, nil } - t.Cleanup(func() { runTheVerb = was }) - handlers, _, err := seatToolHandlers() - if err != nil { - t.Fatal(err) - } - for _, c := range []struct { - verb string - args string - look bool - }{ - {"deliver", `{"plan":"p"}`, true}, - {"plans", `{"stop":"plan-1","why":"w"}`, true}, - {"plans", `{}`, false}, - {"nodes", `{}`, false}, - } { - before := looks.Load() - h := handlers[c.verb] - if h == nil { - t.Fatalf("no verb %s", c.verb) - } - if _, err := h(ctx, []byte(c.args)); err != nil { - t.Fatalf("%s %s: %v", c.verb, c.args, err) - } - time.Sleep(100 * time.Millisecond) - if got := looks.Load() > before; got != c.look { - t.Errorf("%s %s: look asked = %v; want %v", c.verb, c.args, got, c.look) - } - } -} diff --git a/cmd/mesh-controller/waiting_provider_test.go b/cmd/mesh-controller/waiting_provider_test.go deleted file mode 100644 index 684921c2..00000000 --- a/cmd/mesh-controller/waiting_provider_test.go +++ /dev/null @@ -1,386 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A provider waiting for the operator holds its consumers, and a wait beside another wait is said (novox/hq -// issue 405, found in the review of ADR 0283's controller change). -// -// The shapes are those of issue 386 (mounts waiting for smb-password-games: waitingResource, passwordWait) and of -// the openrazer wait on the workstation of 2026-10-11 (relogin, userUnit): an account in its group whose session -// began before it was, and its unit failed in that account's own service manager. - -// waitingDatabase is a provider whose only part not healthy waits for the operator's secret: a database's -// process stated waiting, as the node-engine states a tool check answering waits (ADR 0283 decision 2). Its wait -// names the part that waits by the provision it gives. -func waitingDatabase() inventory.ResourceHealth { - return waitingDatabaseFor(inventory.Wait{Part: "postgres-database", Secret: "licence", - What: "the licence key of the database"}) -} - -func waitingDatabaseFor(waits ...inventory.Wait) inventory.ResourceHealth { - return inventory.ResourceHealth{Module: "db", Resource: "db.server", Kind: "process", Target: "db.service", - State: link.StateWaiting, Check: "tool", Reason: "the database waits for its licence", Waits: waits} -} - -// backupsWait is a wait of the same provider for another part than the one its consumers need. -var backupsWait = inventory.Wait{Part: "the nightly backups", Secret: "backup-key", What: "the key of the backups"} - -// failingConsumer is a consumer whose check that needs the database fails. -func failingConsumer(module string) inventory.ResourceHealth { - return inventory.ResourceHealth{Module: module, Resource: module + ".web", Kind: "container", Target: module, - State: link.StateUnhealthy, Reason: "http /health on web: answered 500", Check: "http", Needs: "postgres-database"} -} - -// dbSecrets is what the database's waits name: own secrets issued outside the mesh, so a wait for one checks out -// while nobody gave it (ADR 0283 decision 3, novox/hq issue 450). -var dbSecrets = catalogue.OwnSecrets{ - "licence": {Path: "/s/licence", IssuedBy: catalogue.IssuedOutside}, - "backup-key": {Path: "/s/backup-key", IssuedBy: catalogue.IssuedOutside}, -} - -// holdingOf is one reading of the record without a store: the newest statements, the open conditions, the -// catalogue, what was given on the provider's machine (nothing), and each consumer's provider already looked up, -// as providerFor memoises it. -func holdingOf(open []conditions.Condition, healths map[string]inventory.NodeHealth, bound map[[3]string]catalogue.Chosen) *holding { - h := &holding{ctx: context.Background(), healths: healths, open: open, providers: map[string]providerLookup{}, - shelf: map[string]catalogue.Manifest{"db": {Module: "db", Version: "1", OwnSecrets: dbSecrets, - Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}}}, - waits: operatorWaitFacts{given: map[string]map[string]time.Time{"db@anchor": {}}}} - for k, p := range bound { - h.providers[k[0]+"\x00"+k[1]+"\x00"+k[2]] = providerLookup{p, true} - } - return h -} - -var theDatabase = catalogue.Chosen{Node: "anchor", Module: "db"} - -var shopOnTheDatabase = map[[3]string]catalogue.Chosen{{"laptop", "shop", "postgres-database"}: theDatabase} - -func shopFailingBeside(provider ...inventory.ResourceHealth) map[string]inventory.NodeHealth { - return map[string]inventory.NodeHealth{ - "anchor": {Node: "anchor", Resources: provider}, - "laptop": {Node: "laptop", Resources: []inventory.ResourceHealth{failingConsumer("shop")}}, - } -} - -// (1) A provider whose only part not healthy waits for the operator holds the findings of the consumers of the -// waiting part under it (ADR 0240 rule 5); a consumer of another part, or one whose part cannot be matched, is -// its own (ADR 0283 decision 1: a module that waits says nothing else of it is wrong). -func TestAProviderWaitingForTheOperatorHoldsOnlyTheConsumersOfTheWaitingPart(t *testing.T) { - shop := []inventory.ResourceHealth{failingConsumer("shop")} - unhealthyDB := waitingDatabase() - unhealthyDB.State, unhealthyDB.Waits, unhealthyDB.Reason = link.StateUnhealthy, nil, "its tool check: refused" - healthyDB := waitingDatabase() - healthyDB.State, healthyDB.Waits = link.StateHealthy, nil - byCredential := holdingOf(nil, shopFailingBeside(waitingDatabaseFor(inventory.Wait{Part: "the server", - Secret: "licence", What: "the licence key"})), shopOnTheDatabase) - byCredential.shelf["db"] = catalogue.Manifest{Module: "db", Version: "1", OwnSecrets: dbSecrets, Provides: []catalogue.Offer{{ - Name: "postgres-database", Credential: &catalogue.OfferCredential{Own: "licence"}}}} - for _, c := range []struct { - name string - hold *holding - held bool - onlyWaits bool - uncovered bool - }{ - {"the waiting part is the provision", holdingOf(nil, shopFailingBeside(waitingDatabase()), shopOnTheDatabase), true, true, false}, - {"the wait is for the provision's shared credential", byCredential, true, true, false}, - {"the wait is for another part", holdingOf(nil, shopFailingBeside(waitingDatabaseFor(backupsWait)), shopOnTheDatabase), false, false, true}, - {"only the needs-operator condition, its parts not said", holdingOf( - []conditions.Condition{{Key: needsOperatorKey("db", "anchor"), Kind: kindNeedsOperator}}, - shopFailingBeside(), shopOnTheDatabase), false, false, true}, - {"an unhealthy provider holds as before", holdingOf(nil, shopFailingBeside(unhealthyDB), shopOnTheDatabase), true, false, false}, - {"a healthy provider holds nothing", holdingOf(nil, shopFailingBeside(healthyDB), shopOnTheDatabase), false, false, false}, - } { - by, held := c.hold.heldWith("laptop", "shop", shop) - if held != c.held || (held && by.provider != theDatabase) || (len(by.waits) > 0) != c.onlyWaits { - t.Errorf("%s: held %v under %v with waits %v; want held %v, only waits %v", c.name, held, by.provider, - by.waits, c.held, c.onlyWaits) - } - if _, uncovered := c.hold.waitingUncovered("laptop", "shop", shop); uncovered != c.uncovered { - t.Errorf("%s: said as a provider waiting for another part %v; want %v", c.name, uncovered, c.uncovered) - } - } -} - -// (1) The consumer's first-node gate under a provider that only waits for the operator passes as a wait for a -// person (ADR 0254), carrying the wait (ADR 0283 decision 4); under an unhealthy provider it waits, as before. -func TestAConsumersGateUnderAWaitingProviderPassesCarryingTheWait(t *testing.T) { - now := time.Now() - since := now.Add(-time.Minute) - for _, c := range []struct { - name string - provider inventory.ResourceHealth - want health - says []string - }{ - {"a provider that only waits", waitingDatabase(), healthPerson, - []string{"waits on db on anchor, which waits for you", "the licence key of the database", "nox secret ask anchor db licence"}}, - {"an unhealthy provider", func() inventory.ResourceHealth { - r := waitingDatabase() - r.State, r.Waits, r.Reason = link.StateUnhealthy, nil, "its tool check: refused" - return r - }(), healthWaiting, []string{"waits on db on anchor, which is unhealthy"}}, - } { - healths := shopFailingBeside(c.provider) - for m, h := range healths { - h.HeardAt = now - healths[m] = h - } - f := gateFacts{now: now, health: healths, heldOn: heldReadings(holdingOf(nil, healths, shopOnTheDatabase))} - h, why := moduleHealthWord("shop", "laptop", since, f) - if h != c.want { - t.Errorf("%s: the consumer's gate reads %v %q; want %v", c.name, h, why, c.want) - continue - } - for _, s := range c.says { - if !strings.Contains(why, s) { - t.Errorf("%s: the gate's reading %q does not say %q", c.name, why, s) - } - } - } -} - -// judgeBoth judges the provider's statement and then the consumer's, two looks each, over a record that changes -// as the statements do. -func judgeBoth(t *testing.T, k *conditions.Keeper, provider []inventory.ResourceHealth, - byProvider, byConsumer map[string]inventory.NodeHealth) []conditions.Condition { - t.Helper() - ctx := t.Context() - was := readHoldingFor - t.Cleanup(func() { readHoldingFor = was }) - healths := byProvider - readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) { - return holdingOf(open, healths, shopOnTheDatabase), nil - } - for look := 1; look <= 2; look++ { - healths = byProvider - if err := judgeModuleHealth(ctx, nil, k, "anchor", map[string][]inventory.ResourceHealth{"db": provider}, - map[string]int{"db": look}, time.Now()); err != nil { - t.Fatal(err) - } - } - for look := 1; look <= 2; look++ { - healths = byConsumer - if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}}, - map[string]int{"shop": look}, time.Now()); err != nil { - t.Fatal(err) - } - } - open, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - return open -} - -func conditionOf(open []conditions.Condition, key string) *conditions.Condition { - for i, c := range open { - if c.Key == key { - return &open[i] - } - } - return nil -} - -// (1) The consumer raises nothing of its own; the provider's needs-operator condition lists who waits on it and -// stays a warning (ADR 0283 decision 5: never escalated). The consumer's statement arrives after the provider's, -// so it is the consumer's judging (sayWaiters) that lists it at the provider — no store involved. -func TestAWaitingProvidersConditionListsWhoWaitsOnItAndStaysAWarning(t *testing.T) { - k, _ := withConditionsInMemory(t) - open := judgeBoth(t, k, []inventory.ResourceHealth{waitingDatabase()}, - map[string]inventory.NodeHealth{"anchor": {Node: "anchor", Resources: []inventory.ResourceHealth{waitingDatabase()}}}, shopFailingBeside(waitingDatabase())) - provider := conditionOf(open, needsOperatorKey("db", "anchor")) - if len(open) != 1 || provider == nil { - t.Fatalf("a provider waiting for the operator and a consumer of its waiting part raised %v; want the "+ - "provider's needs-operator alone", openKeysOf(open)) - } - if said := provider.Evidence[0].Said; !strings.Contains(said, "shop on laptop") { - t.Fatalf("the provider's needs-operator does not list shop on laptop as waiting on it: %s", said) - } - if provider.Severity != conditions.Warning { - t.Fatalf("the provider's needs-operator became %s with a consumer waiting; it stays a warning", provider.Severity) - } - if provider.Resolver != conditions.ResolverOperator || !strings.Contains(provider.Needs, "desk prompt") { - t.Fatalf("the provider's condition: %+v", provider) - } -} - -// (1) A consumer of another part than the one waiting is raised on its own, and says its provider waits. -func TestAConsumerOfAnotherPartIsRaisedOnItsOwn(t *testing.T) { - k, _ := withConditionsInMemory(t) - backups := waitingDatabaseFor(backupsWait) - open := judgeBoth(t, k, []inventory.ResourceHealth{backups}, shopFailingBeside(backups), shopFailingBeside(backups)) - consumer := conditionOf(open, moduleUnhealthyKey("shop", "laptop")) - if consumer == nil || conditionOf(open, needsOperatorKey("db", "anchor")) == nil { - t.Fatalf("raised %v; want shop's own unhealthy beside db's needs-operator", openKeysOf(open)) - } - if said := consumer.Evidence[0].Said; !strings.Contains(said, "db on anchor waits for you, but not for anything shop is known to need, so shop's fault is said on its own") { - t.Fatalf("the consumer's condition does not say its provider waits: %s", said) - } -} - -// relogin and userUnit are person_wait_test.go's; mounts is said here with the same account and unit beside its -// wait for the password. -func reloginBesideAWait() []inventory.ResourceHealth { - return []inventory.ResourceHealth{relogin("mounts", "operator"), userUnit("mounts", "operator"), - waitingResource(passwordWait)} -} - -// (2) A module with a checked wait beside a relogin-needed account says both: the new login, and the act the -// operator owes it. -func TestAWaitBesideAReloginIsSaid(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - for look := 1; look <= 2; look++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": reloginBesideAWait()}, - map[string]int{"mounts": look}, time.Now()); err != nil { - t.Fatal(err) - } - } - got, open := needsOperatorOpen(t, k) - if got == nil { - t.Fatalf("a wait for the operator beside a relogin is not said: %v", openKeysOf(open)) - } - if !strings.Contains(got.Summary, "smb-password-games") || got.Severity != conditions.Warning { - t.Fatalf("the needs-operator beside the relogin: %+v", got) - } - relogged := false - for _, c := range open { - relogged = relogged || c.Key == reloginKey("mounts", "workstation") - } - if !relogged { - t.Fatalf("the relogin is no longer said beside the wait: %v", openKeysOf(open)) - } - // The login done, the wait stays said and the relogin clears. - if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}}, - map[string]int{"mounts": 3}, time.Now()); err != nil { - t.Fatal(err) - } - got, open = needsOperatorOpen(t, k) - if got == nil || len(open) != 1 { - t.Fatalf("after the new login: %v", openKeysOf(open)) - } -} - -// (2) The same beside a directory used as found. -func TestAWaitBesideADirectoryUsedAsFoundIsSaid(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - found := foundDirectory("mounts", time.Now().Add(-time.Hour)) - for look := 1; look <= 2; look++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{ - "mounts": {found, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil { - t.Fatal(err) - } - } - got, open := needsOperatorOpen(t, k) - if got == nil { - t.Fatalf("a wait for the operator beside a directory used as found is not said: %v", openKeysOf(open)) - } - asFound := false - for _, c := range open { - asFound = asFound || c.Key == usedAsFoundKey("mounts", "workstation") - } - if !asFound { - t.Fatalf("the directory used as found is no longer said beside the wait: %v", openKeysOf(open)) - } -} - -// (2) Beside a fault of its own, the wait is said too, and the fault's words do not count the waiting part among -// what fails. -func TestAWaitBesideAFaultIsSaidAndTheFaultIsTheFaultAlone(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - down := inventory.ResourceHealth{Module: "mounts", Resource: "mounts.apply", Kind: "process", - Target: "mesh-mounts-apply.service", State: link.StateUnhealthy, Reason: "down"} - for look := 1; look <= 2; look++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{ - "mounts": {down, waitingResource(passwordWait)}}, map[string]int{"mounts": look}, time.Now()); err != nil { - t.Fatal(err) - } - } - got, open := needsOperatorOpen(t, k) - if got == nil { - t.Fatalf("a wait for the operator beside a fault is not said: %v", openKeysOf(open)) - } - var fault *conditions.Condition - for i, c := range open { - if c.Key == moduleUnhealthyKey("mounts", "workstation") { - fault = &open[i] - } - } - if fault == nil { - t.Fatalf("the fault is not said: %v", openKeysOf(open)) - } - if strings.Contains(fault.Summary, "mounts.watch") { - t.Fatalf("the fault's summary counts the waiting part as failing: %q", fault.Summary) - } -} - -func openKeysOf(cs []conditions.Condition) []string { - var out []string - for _, c := range cs { - out = append(out, c.Key) - } - return out -} - -// A consumer raised on its own, whose provider then waits for the operator for the part it needs, is cleared saying -// which the provider is: it waits for you, not that it is unhealthy (novox/hq issue 405 review). -func TestAConsumerHeldOnceItsProviderWaitsSaysWhichItIs(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - start := time.Now().Add(-time.Second) - healthy := waitingDatabase() - healthy.State, healthy.Waits = link.StateHealthy, nil - healths := shopFailingBeside(healthy) - was := readHoldingFor - t.Cleanup(func() { readHoldingFor = was }) - readHoldingFor = func(_ context.Context, _ *inventory.Inventory, open []conditions.Condition) (*holding, error) { - return holdingOf(open, healths, shopOnTheDatabase), nil - } - shop := map[string][]inventory.ResourceHealth{"shop": {failingConsumer("shop")}} - for look := 1; look <= 2; look++ { - if err := judgeModuleHealth(ctx, nil, k, "laptop", shop, map[string]int{"shop": look}, time.Now()); err != nil { - t.Fatal(err) - } - } - if open, _ := k.Open(ctx); conditionOf(open, moduleUnhealthyKey("shop", "laptop")) == nil { - t.Fatalf("shop beside a healthy provider is not raised: %v", openKeysOf(open)) - } - healths = shopFailingBeside(waitingDatabase()) - if err := judgeModuleHealth(ctx, nil, k, "laptop", shop, map[string]int{"shop": 3}, time.Now()); err != nil { - t.Fatal(err) - } - var why string - for deadline := time.Now().Add(2 * time.Second); why == "" && time.Now().Before(deadline); { - events, err := k.HistorySince(ctx, start) - if err != nil { - t.Fatal(err) - } - for _, e := range events { - if e.Key == moduleUnhealthyKey("shop", "laptop") && e.Change == conditions.ChangeCleared { - why = e.Why - } - } - if why == "" { - time.Sleep(10 * time.Millisecond) - } - } - if !strings.Contains(why, "waits on db on anchor, which waits for you") { - t.Fatalf("shop's clearing says %q; want it to say db on anchor waits for you", why) - } -} diff --git a/cmd/mesh-controller/walk_declared.go b/cmd/mesh-controller/walk_declared.go deleted file mode 100644 index 528ebc47..00000000 --- a/cmd/mesh-controller/walk_declared.go +++ /dev/null @@ -1,163 +0,0 @@ -package main - -// The machines whose declaration a walk's merges change (novox/hq ADR 0505 §2): beside what the walk sends, a merge -// can change what another machine is to run — an assignment, a provision or a requirement added or removed — and a -// problem raised there as the walk runs, or as it settles, is the walk's as much as one on a machine it sends to. The -// router holds what a walk touches (ADR 0308 §1); these are touches of their own, marked `declared`. -// -// **Compared as the mesh compares whether a machine is up to date.** Each machine's declaration is composed as a push -// would compose it, with the number, epoch and generation it was last sent, and its digest held against the one it was -// last sent (wouldSend, inventory.Waiting): byte for byte what it was sent when nothing changed, so a secret, a sequence -// or a lease's epoch never makes a machine differ by itself. A machine whose declaration cannot be worked out now and -// was sent one before is taken as changed: not known is changed, never "unchanged" (a merge that leaves a machine -// unable to compose is the case this exists for, the 11:03 message of ADR 0505's context). -// -// **Said again, not waited for.** Composing every machine takes seconds; the walk's start and end are said at once as -// before, and the serving controller, which hears every `plan-moved` — its own and a verb's — says a start or an end -// again with the declared touches added, marked `declared_read`, which it never reads twice. The router merges a walk -// said again (messenger walks.go). - -import ( - "context" - "encoding/json" - "fmt" - "os" - "sort" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// walkDeclaredRead marks a walk's start or end said again with its declared touches: read once. -const walkDeclaredRead = "declared_read" - -// declaredWithin bounds the comparison of every machine for one walk's start or end. -const declaredWithin = time.Minute - -// changedDeclarations is every machine whose declaration differs from what it was last sent, and every machine sent -// one before whose declaration cannot be worked out now. A seam a test replaces. -var changedDeclarations = func(ctx context.Context, open *stores) ([]string, error) { - nodes, err := open.inventory.Nodes(ctx) - if err != nil { - return nil, err - } - ctx = withInputCache(ctx) - would, err := wouldSend(ctx, open, nodes) - if err != nil { - return nil, err - } - behind, err := open.inventory.Waiting(ctx, would) - if err != nil { - return nil, err - } - sent := map[string]bool{} - for _, n := range nodes { - if s, known, err := open.inventory.SentTo(ctx, n.Name); err == nil && known && s.Digest != "" { - sent[n.Name] = true - } - } - return changedOf(nodes, would, behind, sent), nil -} - -// changedOf is changedDeclarations over what was read: the machines behind, and those sent one before that could not -// be worked out. Pure; sorted. -func changedOf(nodes []inventory.Node, would map[string]string, behind []inventory.Machine, sent map[string]bool) []string { - set := map[string]bool{} - for _, m := range behind { - set[m.Node] = true - } - for _, n := range nodes { - if _, worked := would[n.Name]; !worked && sent[n.Name] { - set[n.Name] = true - } - } - out := make([]string, 0, len(set)) - for n := range set { - out = append(out, n) - } - sort.Strings(out) - return out -} - -// withDeclared is a walk's start or end said again with the machines whose declaration changed added as declared -// touches: false when the body says no start or end, was said again already, or names no machine it does not touch. -func withDeclared(body []byte, changed []string) ([]byte, bool, error) { - said, wanted, err := toSayAgain(body) - if err != nil || !wanted { - return nil, false, err - } - var touches []walkTouch - if raw := said["touches"]; raw != nil && string(raw) != "null" { - if err := json.Unmarshal(raw, &touches); err != nil { - return nil, false, fmt.Errorf("its touches are not a list of {machine, modules}: %w", err) - } - } - added := declaredTouches(touches, changed) - if len(added) == 0 { - return nil, false, nil - } - touches = append(touches, added...) - sort.Slice(touches, func(i, j int) bool { return touches[i].Machine < touches[j].Machine }) - if said["touches"], err = json.Marshal(touches); err != nil { - return nil, false, err - } - said[walkDeclaredRead] = json.RawMessage(`true`) - out, err := json.Marshal(said) - return out, err == nil, err -} - -// toSayAgain reads a plan-moved: whether it says a walk's start or end not said again yet. -func toSayAgain(body []byte) (map[string]json.RawMessage, bool, error) { - var said map[string]json.RawMessage - if err := json.Unmarshal(body, &said); err != nil { - return nil, false, fmt.Errorf("a plan-moved that is not an object: %w", err) - } - var phase string - _ = json.Unmarshal(said["phase"], &phase) - return said, (phase == inventory.WalkStarted || phase == inventory.WalkEnded) && said[walkDeclaredRead] == nil, nil -} - -// declaredTouches are the machines changed that a walk's touches do not name, each a declared touch. Pure; sorted. -func declaredTouches(touches []walkTouch, changed []string) []walkTouch { - named := map[string]bool{} - for _, t := range touches { - named[t.Machine] = true - } - var out []walkTouch - for _, n := range changed { - if !named[n] { - out = append(out, walkTouch{Machine: n, Modules: []string{}, Declared: true}) - named[n] = true - } - } - sort.Slice(out, func(i, j int) bool { return out[i].Machine < out[j].Machine }) - return out -} - -// sayDeclared says a walk's start or end again with its declared touches, when it has any. What cannot be read is -// said on the controller's log: the walk is then held as before, never less. -func sayDeclared(ctx context.Context, bus link.Bus, open *stores, body []byte) { - if _, wanted, _ := toSayAgain(body); !wanted { - return // no start or end, or said again already: nothing to compare - } - comparing, cancel := context.WithTimeout(ctx, declaredWithin) - defer cancel() - changed, err := changedDeclarations(comparing, open) - if err != nil { - fmt.Fprintf(os.Stderr, "plan-moved: which machines' declarations the walk changes cannot be read: %v\n", err) - return - } - again, ok, err := withDeclared(body, changed) - if err != nil || !ok { - if err != nil { - fmt.Fprintf(os.Stderr, "plan-moved: the walk cannot be said again with its declared machines: %v\n", err) - } - return - } - stating, stop := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) - defer stop() - if err := bus.PublishSeatEvent(stating, link.MeshControllerSeat, link.KeyPlanMoved, again); err != nil { - fmt.Fprintf(os.Stderr, "plan-moved: the walk could not be said again with its declared machines: %v\n", err) - } -} diff --git a/cmd/mesh-controller/walk_declared_test.go b/cmd/mesh-controller/walk_declared_test.go deleted file mode 100644 index 2dd47889..00000000 --- a/cmd/mesh-controller/walk_declared_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package main - -import ( - "encoding/json" - "reflect" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq ADR 0505 §2: the machines whose declaration a walk's merges change are said as declared touches. - -// The machines behind what they were last sent, and those sent one before that cannot be worked out now, are changed; -// one never sent and not worked out is not. -func TestAMachineBehindOrNoLongerComposableIsChanged(t *testing.T) { - nodes := []inventory.Node{{Name: "ace"}, {Name: "g14"}, {Name: "novox"}, {Name: "shanks"}, {Name: "new"}} - would := map[string]string{"ace": "d1", "g14": "d2", "novox": "d3"} - behind := []inventory.Machine{{Node: "ace"}} - sent := map[string]bool{"ace": true, "g14": true, "novox": true, "shanks": true} - if got := changedOf(nodes, would, behind, sent); !reflect.DeepEqual(got, []string{"ace", "shanks"}) { - t.Fatalf("changed: %v, want ace (behind) and shanks (no longer composes)", got) - } -} - -// A walk's start said by plan-moved (walk_said.go's body), said again with the changed machines it does not touch as -// declared touches, once. -func TestAWalksStartIsSaidAgainWithTheMachinesItsMergesChange(t *testing.T) { - body, err := json.Marshal(planMovedBody{Plan: inventory.Plan{ID: "plan-1", State: inventory.PlanBuilding}, - walkSaid: &walkSaid{Walk: "plan-1", Phase: inventory.WalkStarted, BudgetClass: "leaf", BoundSeconds: 600, - Touches: []walkTouch{{Machine: "novox", Modules: []string{"gitea"}}}}}) - if err != nil { - t.Fatal(err) - } - again, ok, err := withDeclared(body, []string{"novox", "shanks"}) - if err != nil || !ok { - t.Fatalf("not said again: %v %v", ok, err) - } - var said struct { - ID string `json:"id"` - Phase string `json:"phase"` - Touches []walkTouch `json:"touches"` - DeclaredRead bool `json:"declared_read"` - } - if err := json.Unmarshal(again, &said); err != nil { - t.Fatal(err) - } - want := []walkTouch{{Machine: "novox", Modules: []string{"gitea"}}, {Machine: "shanks", Modules: []string{}, Declared: true}} - if said.ID != "plan-1" || said.Phase != inventory.WalkStarted || !said.DeclaredRead || !reflect.DeepEqual(said.Touches, want) { - t.Fatalf("said again: %+v", said) - } - if _, ok, _ := withDeclared(again, []string{"ace"}); ok { - t.Fatalf("a start said again was said a third time") - } - if _, ok, _ := withDeclared(body, []string{"novox"}); ok { - t.Fatalf("said again with nothing it does not touch") - } - moved, _ := json.Marshal(inventory.Plan{ID: "plan-2", State: inventory.PlanBuilding}) - if _, ok, _ := withDeclared(moved, []string{"shanks"}); ok { - t.Fatalf("a move inside a walk was said as its start") - } -} diff --git a/cmd/mesh-controller/walk_from_keys_integration_test.go b/cmd/mesh-controller/walk_from_keys_integration_test.go deleted file mode 100644 index a63213ab..00000000 --- a/cmd/mesh-controller/walk_from_keys_integration_test.go +++ /dev/null @@ -1,186 +0,0 @@ -//go:build integration - -package main - -import ( - "github.com/nats-io/nats.go" - "reflect" - "strings" - "testing" - "time" - - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A6, A7 whole against the store and a real bus (ADR 0503 §6, ADR 0498 check §7; issue 1851): a walk whose modules the -// builder built goes from asked to built with no build asked, its keys taken in off the plans' hold, then sends its tier -// to the first machine as before; a key still building holds the tier, said, and moves on when the builder's key is done. -func TestAWalkTakesItsTierFromTheBuildersKeysAndSendsIt(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - tm := aTierMesh(t, "app1", "app2") - ctx := t.Context() - inv := tm.open.inventory - noBuildAsked(t) - b := aBuilderOnTheBus(t) - - // The walk carries c3 of the catalogue; its tier is unasked. - p := tm.plan(t) - p.Commit = "c3c3c3c3c3c3c3c3" - for _, m := range []string{"app1", "app2"} { - p.Modules[m] = &inventory.PlanModule{} - } - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - fp1, fp2 := fpOf('1'), fpOf('2') - m1 := b.indexed(t, "app1", "c3", "novox/mesh-catalog", "modules/app1", p.Commit, fp1) - m2 := b.indexed(t, "app2", "c3", "novox/mesh-catalog", "modules/app2", p.Commit, fp2) - b.built(t, "app1", "c3", fp1, m1, sdkdelivery.BuildDone) - b.built(t, "app2", "c3", fp2, m2, sdkdelivery.BuildBuilding) - - advancePlans(ctx, tm.open) - becomes(t, 10*time.Second, "app1 taken from its key", func() bool { - advancePlans(ctx, tm.open) - return tm.plan(t).Modules["app1"].State == "built" - }) - p = tm.plan(t) - if s := p.Modules["app2"]; s.State != "asked" || !s.Key || !strings.Contains(s.Why, "building on ace") { - t.Fatalf("A7: app2, whose key is building, is %+v", s) - } - if len(tm.sent) != 0 { - t.Fatalf("A7: the tier was sent while app2 waits on its key: %v", tm.sent) - } - if s := p.Modules["app1"]; !strings.Contains(s.Made, "taken from the builder's key") || s.Fingerprint != fp1 { - t.Fatalf("A6: plans does not say app1 was taken from the builder's key: %+v", s) - } - - b.built(t, "app2", "c3", fp2, m2, sdkdelivery.BuildDone) - becomes(t, 10*time.Second, "the tier sent to the first machine", func() bool { - advancePlans(ctx, tm.open) - return len(tm.sent) > 0 - }) - p = tm.plan(t) - if !reflect.DeepEqual(tm.sent[0], []string{"anchor"}) { - t.Fatalf("A6: the tier taken from keys was sent %v, not to the first machine", tm.sent) - } - current, err := inv.CurrentBuilds(ctx) - if err != nil { - t.Fatal(err) - } - for _, m := range []string{"app1", "app2"} { - if current[m].Commit != p.Commit || !strings.HasPrefix(p.Modules[m].Build, "key-") { - t.Fatalf("A6: %s stands on %+v (%s), not the key's build at %s", m, current[m], p.Modules[m].Build, p.Commit) - } - } -} - -// A7: a failed key fails the walk with the builder's words, and nothing is sent. -func TestAFailedKeyFailsTheWalkInTheBuildersWords(t *testing.T) { - tm := aTierMesh(t, "app1") - ctx := t.Context() - inv := tm.open.inventory - noBuildAsked(t) - b := aBuilderOnTheBus(t) - p := tm.plan(t) - p.Commit = "c4c4c4c4c4c4c4c4" - p.Modules["app1"] = &inventory.PlanModule{} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - fp := fpOf('4') - manifest := b.indexed(t, "app1", "c4", "novox/mesh-catalog", "modules/app1", p.Commit, fp) - b.built(t, "app1", "c4", fp, manifest, sdkdelivery.BuildFailed) - for i := 0; i < 3; i++ { - advancePlans(ctx, tm.open) - } - p = tm.plan(t) - if p.State != inventory.PlanFailed || !strings.Contains(p.Note, "digests differ: ace sha256:1, g14 sha256:2") { - t.Fatalf("A7: the walk is %s: %q", p.State, p.Note) - } - if len(tm.sent) != 0 { - t.Fatalf("A7: a failed walk sent %v", tm.sent) - } -} - -// C2's criterion (issue 1851, R4g): a merge whose walk moves the builder, a build module and the controller takes every one -// from the builder's referenced done key, and no ask reaches any subject of the seat node-build-agent. A fake builder writes -// the keys on a real bus. -func TestTheBuilderABuildModuleAndTheControllerAreTakenFromKeysAndNothingIsAskedOfBuildAgent(t *testing.T) { - names := []string{sdkdelivery.BuilderModule, "mesh-build-go", "mesh-controller"} - tm := aTierMesh(t, names...) - ctx := t.Context() - inv := tm.open.inventory - b := aBuilderOnTheBus(t) - asked := make(chan string, 16) - sub, err := b.js.Conn().Subscribe("mesh.seat.node-build-agent.>", func(m *nats.Msg) { asked <- m.Subject }) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - - p := tm.plan(t) - p.Commit = "c5c5c5c5c5c5c5c5" - for _, m := range names { - p.Modules[m] = &inventory.PlanModule{} - } - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - for i, m := range names { - fp := fpOf(byte('a' + i)) - manifest := b.indexed(t, m, "c5", "novox/mesh-catalog", "modules/"+m, p.Commit, fp) - b.built(t, m, "c5", fp, manifest, sdkdelivery.BuildDone) - } - becomes(t, 10*time.Second, "the three taken from their keys", func() bool { - advancePlans(ctx, tm.open) - q := tm.plan(t) - for _, m := range names { - if s := q.Modules[m]; s == nil || s.State != "built" { - return false - } - } - return true - }) - p = tm.plan(t) - for _, m := range names { - if s := p.Modules[m]; !s.Key || !strings.HasPrefix(s.Build, "key-") || !strings.Contains(s.Made, "taken from the builder's key") { - t.Errorf("%s was not taken from the builder's referenced key: %+v", m, s) - } - } - select { - case subject := <-asked: - t.Fatalf("an ask reached the retired build seat: %s", subject) - case <-time.After(300 * time.Millisecond): - } -} - -// C5's criterion: after a plan that moves the builder, its record names the builder's key and its fingerprint. -func TestTheBuildersRecordNamesItsKeyAndFingerprint(t *testing.T) { - tm := aTierMesh(t, sdkdelivery.BuilderModule) - ctx := t.Context() - inv := tm.open.inventory - b := aBuilderOnTheBus(t) - p := tm.plan(t) - p.Commit = "c6c6c6c6c6c6c6c6" - p.Modules[sdkdelivery.BuilderModule] = &inventory.PlanModule{} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - fp := fpOf('9') - manifest := b.indexed(t, sdkdelivery.BuilderModule, "c6", "novox/mesh-catalog", "modules/"+sdkdelivery.BuilderModule, p.Commit, fp) - b.built(t, sdkdelivery.BuilderModule, "c6", fp, manifest, sdkdelivery.BuildDone) - becomes(t, 10*time.Second, "the builder taken from its key", func() bool { - advancePlans(ctx, tm.open) - return tm.plan(t).Modules[sdkdelivery.BuilderModule].State == "built" - }) - id := tm.plan(t).Modules[sdkdelivery.BuilderModule].Build - record, found, err := inv.BuildByID(ctx, id) - if err != nil || !found { - t.Fatalf("the builder's record %q: %v %v", id, found, err) - } - if !strings.HasPrefix(record.ID, "key-") || record.SourceFingerprint != fp || record.Module != sdkdelivery.BuilderModule { - t.Fatalf("the builder's record names %+v, not its key and fingerprint %s", record, fp) - } -} diff --git a/cmd/mesh-controller/walk_keys_scaffold_integration_test.go b/cmd/mesh-controller/walk_keys_scaffold_integration_test.go deleted file mode 100644 index d957e9c0..00000000 --- a/cmd/mesh-controller/walk_keys_scaffold_integration_test.go +++ /dev/null @@ -1,137 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "sync" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - "github.com/novox/mesh-controller/internal/builderkeys" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A walk takes every module from the builder's done key and asks no build (R4g, ADR 0501, ADR 0503 §6). The tests of what -// a walk does around its builds (its tiers, sends, stops, retries, backups and waits for its delivery's word) need the -// builds to happen without a builder, so this stands one in: every module has a done key at once, and the take-in of a key -// records the build and registers the module as the real one does, in the caller's turn. The tests that are of the take-in -// itself (key_takein, walk_from_keys, first_builds) use a builder on a bus of their own (builder_on_the_bus_test.go). - -// keysAllDone is the indexer and the builder as a walk reads them when every module is built at every commit. -type keysAllDone struct { - mu sync.Mutex - paths [][3]string // repository, path and commit of each key taken in, in order - named []string // the same, as repository#id -} - -// fpOfModuleAt is the fingerprint a module has at a commit in these tests: one for each pair. -func fpOfModuleAt(module, commit string) string { - sum := sha256.Sum256([]byte(module + "@" + commit)) - return "src2:" + hex.EncodeToString(sum[:]) -} - -func (k *keysAllDone) ModuleAt(_ context.Context, module, commit string) (sdkdelivery.Module, error) { - return sdkdelivery.Module{FP2: fpOfModuleAt(module, commit), Commit: commit, Repository: "novox/mesh-catalog", - Path: "modules/" + module}, nil -} - -func (k *keysAllDone) Build(_ context.Context, module, fp string) (builderkeys.Key, error) { - digest := "sha256:" + fp[len("src2:"):] - return builderkeys.Key{Module: module, Fingerprint: fp, Found: true, Revision: 1, - Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "ace", Artifacts: []sdkdelivery.BuiltArtifact{{ - Name: "a", Kind: "archive", Digest: digest, Reference: "artifact-store://" + module + "/a/blobs/" + digest}}}}, nil -} - -// take is the take-in of a key as the walk's look starts it: the build recorded and the module registered at the commit the -// walk carries. -func (k *keysAllDone) take(ctx context.Context, open *stores, module string, trunk sdkdelivery.Module, id string) { - inv := open.inventory - defer func() { - keyTakes.Lock() - delete(keyTakes.running, id) - keyTakes.taken[id] = true - keyTakes.Unlock() - askToLook() - }() - m := catalogue.Manifest{Module: module, Version: "1"} - if shelf, err := inv.Catalogue(ctx); err == nil { - if held, ok := shelf[module]; ok { - m = held - } - } - src, err := inv.SourceOf(ctx, module) - if err != nil { - src = inventory.Source{Repository: trunk.Repository, Seat: "git", Ref: "main", Path: keyPath(trunk)} - } - src.BuiltFrom, src.Head = trunk.Commit, trunk.Commit - raw, _ := json.Marshal(m) - if err := inv.RecordBuild(ctx, inventory.Build{ID: id, Module: module, Repository: src.Repository, Commit: trunk.Commit, - Path: src.Path, On: "ace", At: time.Now().UTC(), Manifest: raw, SourceFingerprint: trunk.FP2}); err != nil { - keepKeyWait(id, "the build could not be recorded: "+err.Error()) - return - } - if err := inv.RegisterModule(ctx, m, src); err != nil { - keepKeyWait(id, "the module could not be registered: "+err.Error()) - return - } - k.mu.Lock() - defer k.mu.Unlock() - k.paths = append(k.paths, [3]string{src.Repository, src.Path, trunk.Commit}) - k.named = append(k.named, src.Repository+"#"+id) -} - -// everyKeyDone has the walks of a test take every module from a done key, and says which were taken. -func everyKeyDone(t *testing.T) *keysAllDone { - t.Helper() - k := &keysAllDone{} - reset := func() { - keyTakes.Lock() - keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{} - keyTakes.Unlock() - } - reset() - wasKeys, wasTake, wasBackground := openPlanKeys, takeKeyIn, inBackground - openPlanKeys = func() (planKeyReads, func(), error) { return k, func() {}, nil } - takeKeyIn = k.take - inBackground = func(work func()) { work() } - t.Cleanup(func() { - openPlanKeys, takeKeyIn, inBackground = wasKeys, wasTake, wasBackground - reset() - }) - return k -} - -// asksRecorded has the walks of a test take every module from a done key, and says each as repository#build id. -func asksRecorded(t *testing.T) *[]string { - t.Helper() - return &everyKeyDone(t).named -} - -// asksWithPaths is asksRecorded saying each as repository, path and the commit the walk carried. -func asksWithPaths(t *testing.T) *[][3]string { - t.Helper() - return &everyKeyDone(t).paths -} - -// asksOfEveryModule once made every module the builder's own, the one ask left; a walk asks nothing now. -func asksOfEveryModule(t *testing.T) { t.Helper() } - -// noBuildAsked once failed a test that asked build-agent for a build; nothing can now, so it says nothing. -func noBuildAsked(t *testing.T) { t.Helper() } - -// twoTiers registers two modules, b standing on a, each with a source a plan asks. -func twoTiers(t *testing.T, open *stores) { - t.Helper() - for _, name := range []string{"a", "b"} { - if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: name, Version: "1"}, - inventory.Source{Repository: "novox/" + name, Seat: "git", Ref: "main", BuiltFrom: "c0ffee", Head: "c0ffee"}); err != nil { - t.Fatal(err) - } - } -} diff --git a/cmd/mesh-controller/walk_resumed_silence_1886_test.go b/cmd/mesh-controller/walk_resumed_silence_1886_test.go deleted file mode 100644 index bb566c7f..00000000 --- a/cmd/mesh-controller/walk_resumed_silence_1886_test.go +++ /dev/null @@ -1,173 +0,0 @@ -package main - -import ( - "slices" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// Issue 1886, the resume of R4e (plan-1792707174627079192): tier 0 was sent at 00:23:28, the walk was stopped at -// 02:01 and resumed at 07:43, and tier 1 was sent at 07:49:54. 35 seconds later the gate named ace silent "within -// 15m0s", and at 07:51:30 shanks, in the same second shanks reported it applied: the plan failed as lost. The walk -// judged each machine on every send it had made it, tier 0's of 00:23 too, narrowed to every module the walk carried -// there — so tier 0's send was held against the templates of tier 1's modules it never carried, and silence was -// measured from 00:23. A machine is judged on the walk's last send to it: that send carries its whole declaration. - -func r4eResumed() (inventory.Plan, []inventory.Reported, inventory.ProvLookup, time.Time) { - tier0 := time.Date(2026, 10, 22, 22, 23, 28, 0, time.UTC) - tier1 := time.Date(2026, 10, 23, 5, 49, 54, 0, time.UTC) - now := time.Date(2026, 10, 23, 5, 51, 30, 0, time.UTC) - t0, t1 := tier0, tier1 - p := inventory.Plan{ID: "plan-1792707174627079192", State: inventory.PlanRolling, Tier: 1, - Tiers: [][]string{{"pacman"}, {"mesh-build-image"}, {"gitea"}}, - Modules: map[string]*inventory.PlanModule{ - "pacman": {State: "built", SentAt: &t0, Rest: map[string]inventory.SentDeclaration{ - "shanks": {Digest: "d0-shanks", Sequence: 900, At: tier0, - Provisioned: map[string]string{"pacman": "pac-1", "mesh-build-image": "img-old"}}, - "ace": {Digest: "d0-ace", Sequence: 1100, At: tier0, - Provisioned: map[string]string{"pacman": "pac-1", "mesh-build-image": "img-old"}}}}, - "mesh-build-image": {State: "built", SentAt: &t1, Rest: map[string]inventory.SentDeclaration{ - "shanks": {Digest: "d1-shanks", Sequence: 971, At: tier1, - Provisioned: map[string]string{"pacman": "pac-1", "mesh-build-image": "img-new"}}, - "ace": {Digest: "d1-ace", Sequence: 1212, At: tier1, - Provisioned: map[string]string{"pacman": "pac-1", "mesh-build-image": "img-new"}}}}, - "gitea": {}}} - shanksAt := now - aceAt := time.Date(2026, 10, 23, 0, 57, 50, 0, time.UTC) - reports := []inventory.Reported{ - // shanks reported on tier 1's send in the second the gate looked. - {Node: "shanks", Outcome: inventory.OutcomeApplied, At: &shanksAt, Declared: "d1-shanks", Current: true, - ReportedSequence: 971}, - // ace's last report is from before tier 1's send, on a declaration sequenced after tier 0's. - {Node: "ace", Outcome: inventory.OutcomeApplied, At: &aceAt, Declared: "d-ace-0257", ReportedSequence: 1150}, - } - says := map[string]inventory.ProvReport{ - "shanks/pacman": {Status: inventory.ProvApplied, Templates: "pac-1", At: now}, - "shanks/mesh-build-image": {Status: inventory.ProvApplied, Templates: "img-new", At: now}, - "ace/pacman": {Status: inventory.ProvApplied, Templates: "pac-1", At: aceAt}, - // ace's client had not placed the old image's files (issue 1884), and has not yet the new one's. - "ace/mesh-build-image": {Status: inventory.ProvWaiting, Templates: "img-old", At: aceAt}, - } - prov := func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok } - return p, reports, prov, now -} - -func TestAResumedWalkJudgesEachMachineOnItsLastSend(t *testing.T) { - p, reports, prov, now := r4eResumed() - waiting, failed, silent := walkReports(p, reports, now, prov) - if len(silent) != 0 { - t.Fatalf("1886: %v named silent within %s of tier 1's send, from tier 0's send of 00:23", silent, applySilentAfter) - } - if len(failed) != 0 || !slices.Equal(waiting, []string{"ace"}) { - t.Fatalf("1886: waiting %v, failed %v; want ace waited for (sent 96 s ago) and shanks reported", waiting, failed) - } - // The gate the plan failed at: mesh-build-image runs on ace and shanks, which tier 2 needs. - needs := []gateNeed{{module: "mesh-build-image", running: []string{"ace", "shanks"}, - since: func(string) time.Time { return *p.Modules["mesh-build-image"].SentAt }}} - w, quiet, lost := settleTierGate(needs, reports, silentNodes(p, reports, now, prov)) - if lost || len(quiet) != 0 || !slices.Equal(w, []string{"mesh-build-image on ace"}) { - t.Fatalf("1886: the gate says waiting %v, silent %v, lost %v; want mesh-build-image on ace waited for", w, quiet, lost) - } -} - -// The machine silent on the walk's last send to it is still silent once the bound has passed since that send. -func TestAMachineSilentOnItsLastSendIsStillSilent(t *testing.T) { - p, reports, prov, now := r4eResumed() - later := now.Add(applySilentAfter + time.Minute) - _, _, silent := walkReports(p, reports, later, prov) - if !slices.Equal(silent, []string{"ace"}) { - t.Fatalf("1886: silent %v past the bound from tier 1's send; want ace", silent) - } -} - -// A walk failed only because every machine its gate waited on was silent, whose tiers are all sent, is carried on by -// `plans retry` once those machines reported (issue 1886): it failed on no build and no rollout, so the retry of failed -// builds and of stopped rollouts refused it with "nothing in tier 1 … failed to build or stopped rolling out". -func r4eLost() inventory.Plan { - p, _, _, _ := r4eResumed() - p.State = inventory.PlanFailed - p.Note = "tier 1 built; " + silentGateWords + " 15m0s: shanks" - // mesh-tools is built and recorded, never sent, as R4e's was. - p.Tiers[1] = append(p.Tiers[1], "mesh-tools") - p.Modules["mesh-tools"] = &inventory.PlanModule{State: "built"} - return p -} - -func TestRetryCarriesOnAWalkLostToSilenceOnceItsMachinesReported(t *testing.T) { - p, reports, prov, now := r4eResumed() - p = r4eLost() - if !lostToSilence(p) { - t.Fatalf("1886: the plan R4e failed as is not read as lost to silence: %q", p.Note) - } - if err := retryRefusal(p, []inventory.Plan{p}, ownPathOf(nil)); err != nil { - t.Fatalf("1886: a walk lost to silence was refused a retry: %v", err) - } - reports[1].Current, reports[1].Declared, reports[1].ReportedSequence = true, "d1-ace", 1212 - reports[1].At = &now - says := map[string]inventory.ProvReport{} - for _, n := range []string{"ace", "shanks"} { - says[n+"/pacman"] = inventory.ProvReport{Status: inventory.ProvApplied, Templates: "pac-1", At: now} - says[n+"/mesh-build-image"] = inventory.ProvReport{Status: inventory.ProvApplied, Templates: "img-new", At: now} - } - prov = func(node, module string) (inventory.ProvReport, bool) { r, ok := says[node+"/"+module]; return r, ok } - said, err := resumeLost(&p, silentNodes(p, reports, now.Add(time.Hour), prov)) - if err != nil { - t.Fatalf("1886: the resume of a walk whose machines reported refused: %v", err) - } - if p.State != inventory.PlanBuilding || p.Tier != 1 { - t.Fatalf("1886: resumed walk is %s at tier %d; want building at tier 1, for the next look to pass its gate", p.State, p.Tier) - } - if !strings.Contains(p.Note, "resumed by hand") || !strings.Contains(said, "resumed at tier 2 of 3") { - t.Fatalf("1886: note %q, said %q", p.Note, said) - } -} - -// Every machine its tier was sent to still silent: the gate would fail it again, so the retry says so instead. -func TestRetryRefusesAWalkLostToSilenceWhileEveryMachineIsStillSilent(t *testing.T) { - p := r4eLost() - if _, err := resumeLost(&p, map[string]bool{"ace": true, "shanks": true}); err == nil || !strings.Contains(err.Error(), "ace, shanks") { - t.Fatalf("1886: a walk whose machine is still silent was resumed: %v", err) - } - if p.State != inventory.PlanFailed { - t.Fatalf("1886: a refused resume changed the plan to %s", p.State) - } -} - -// A walk lost to silence with something of its tier never sent is not this: its send is the look's to make. -func TestALostWalkWithATierUnsentIsNotLostToSilence(t *testing.T) { - p := r4eLost() - p.Modules["mesh-build-image"].State = "asked" - if lostToSilence(p) { - t.Fatal("1886: a tier with a module still asked was read as lost to silence") - } - p = r4eLost() - p.Note = "tier 1 built; mesh-build-image failed to build" - if lostToSilence(p) { - t.Fatal("1886: a plan failed by a build was read as lost to silence") - } -} - -// A walk lost to silence is refused its resume beside a newer open plan of its repository and branch, which supersedes -// it, and beside a related open plan: one plan at a time (ADR 0531). -func TestRetryRefusesAWalkLostToSilenceBesideANewerOrARelatedOpenPlan(t *testing.T) { - p := r4eLost() - created := time.Date(2026, 10, 22, 22, 15, 9, 0, time.UTC) - p.Repository, p.Branch, p.Commit, p.Created = "novox/mesh-sdk", "main", "7aacbc16", created - newerOpen := inventory.Plan{ID: "plan-newer", Repository: "novox/mesh-sdk", Branch: "main", Commit: "d00dfeed", - Created: created.Add(time.Hour), State: inventory.PlanBuilding, Tiers: [][]string{{"zsh"}}, - Modules: map[string]*inventory.PlanModule{"zsh": {State: "asked"}}} - if err := retryRefusal(p, []inventory.Plan{p, newerOpen}, ownPathOf(nil)); err == nil || - !strings.Contains(err.Error(), "plan-newer supersedes it") { - t.Errorf("1886: a walk lost to silence was resumed under a newer open plan of its branch: %v", err) - } - beside := inventory.Plan{ID: "plan-beside", Repository: "novox/mesh-sdk", Branch: "release", Commit: "bbadbb2a", - Created: created.Add(time.Hour), State: inventory.PlanBuilding, Tiers: [][]string{{"mesh-build-image"}}, - Modules: map[string]*inventory.PlanModule{"mesh-build-image": {State: "asked"}}} - if err := retryRefusal(p, []inventory.Plan{p, beside}, ownPathOf(nil)); err == nil || - !strings.Contains(err.Error(), "one walk at a time") { - t.Errorf("1886: a walk lost to silence was resumed beside a related open plan: %v", err) - } -} diff --git a/cmd/mesh-controller/walk_said.go b/cmd/mesh-controller/walk_said.go deleted file mode 100644 index d1a8d3b5..00000000 --- a/cmd/mesh-controller/walk_said.go +++ /dev/null @@ -1,343 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "slices" - "sort" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" -) - -// What a walk touches, its bound and its outcome (novox/hq ADR 0308 decision 1, decision 5): said in -// `plan-moved` on the save that starts the walk and the save that ends it, beside the walk whole, for the holder -// of the operator's channel to hold what the walk touches while it runs and tell its result once. -// -// **What a walk touches** is every module it moves on every machine it sends it to, and every module a moved -// tool runner (node-tools) or node-engine (mesh-host) carries on that machine: what restarts there. At the start -// that is what the walk will send — each module of its tiers whose policy rolls it out, on every machine running -// it; at the end it is what it reached — the machines each module was sent to, first and rest, and what a gated -// send carried with it. -// -// **Its bound** is twice the largest delivery budget among the walk's deliveries (deliveryBudgets: a core -// module's ten minutes, a leaf module's budget in force), and fifteen minutes, not doubled, for a walk none of -// whose deliveries has a budget: a release plan (builds waiting for a person's gate), a walk whose every module's -// policy records rather than rolls out (held for a person, ADR 0282 decision 1), or a walk kept without a class. - -// walkNoBudgetBound is the bound of a walk with no budget (ADR 0308 decision 5): not doubled. -const walkNoBudgetBound = 15 * time.Minute - -// budgetNone is the budget class of a walk none of whose deliveries has a budget. -const budgetNone = "none" - -// walkSaid is what `plan-moved` adds on a walk's start and end, beside the walk whole. -type walkSaid struct { - Walk string `json:"walk"` - Phase string `json:"phase"` - // Touches is, per machine, the modules the walk moves or reaches there and what a moved tool runner or - // node-engine carries there; never null. - Touches []walkTouch `json:"touches"` - // Outcome is on the end only: delivered, failed or stopped. - Outcome string `json:"outcome,omitempty"` - // BoundSeconds and BudgetClass are said on both, read from the walk's class at the start. - BoundSeconds int `json:"bound_seconds"` - BudgetClass string `json:"budget_class"` - // StartedAt is when the walk was said started, on both; EndedAt when it was said ended, on the end: the - // bound runs from StartedAt, whenever the event is heard. - StartedAt *time.Time `json:"started_at,omitempty"` - EndedAt *time.Time `json:"ended_at,omitempty"` -} - -// walkTouch is what a walk touches on one machine. -type walkTouch struct { - Machine string `json:"machine"` - Modules []string `json:"modules"` - // Carried names those of Modules that are there only because a moved tool runner or node-engine carries - // them: what restarts with it, not what the walk builds. - Carried []string `json:"carried,omitempty"` - // Declared marks a machine the walk sends nothing to whose declaration its merges change (novox/hq ADR 0505 §2): - // said again by the serving controller once it has compared every machine (walk_declared.go). - Declared bool `json:"declared,omitempty"` -} - -// planMovedBody is `plan-moved`'s body: the walk whole, as before, and on its start and end what it touches. -type planMovedBody struct { - inventory.Plan - *walkSaid -} - -// walkFacts is what the touches are read from: the controller's inventory. -type walkFacts interface { - Running(ctx context.Context, module string) ([]string, error) - UpgradeOf(ctx context.Context, module string) (inventory.Upgrade, error) - Assigned(ctx context.Context, node string) ([]string, error) - SentBuilds(ctx context.Context, node string) (map[string]string, bool, error) - Catalogue(ctx context.Context) (map[string]catalogue.Manifest, error) -} - -// sayWalk is what `plan-moved` adds to the save that started or ended the walk; nil on every other save. A fact -// that cannot be read is said on the controller's log and left out, never in the way of the event: a walk said -// with fewer touches is held less, never longer. -func sayWalk(ctx context.Context, facts walkFacts, p inventory.Plan) *walkSaid { - if p.WalkSaid == "" { - return nil - } - w := &walkSaid{Walk: p.ID, Phase: p.WalkSaid} - if p.Times != nil { - w.StartedAt = p.Times.Started - if p.WalkSaid == inventory.WalkEnded { - w.EndedAt, w.Outcome = p.Times.Ended, p.Outcome() - } - } - read := readWalkFacts(ctx, facts, p) - w.BudgetClass = walkBudgetClass(p, read.rollsOut) - w.BoundSeconds = int(walkBound(w.BudgetClass) / time.Second) - var moved map[string][]string - if p.WalkSaid == inventory.WalkStarted { - moved = walkSends(p, read.running, read.rollsOut) - } else { - moved = walkReached(p, read.running) - } - var shelf map[string]catalogue.Manifest - if len(moved[broker.RuntimeModule]) > 0 { - // Which modules the tool runner serves is the catalogue's: read only when the walk moves it. - shelf = read.shelf() - } - w.Touches = touchesOf(moved, read.onMachine(moved), shelf) - return w -} - -// walkBudgetClass is the largest budget class among the walk's deliveries: its class read at the cut when one of -// its modules rolls out, and none when it has no budget. -func walkBudgetClass(p inventory.Plan, rollsOut func(string) bool) string { - if p.Release != nil || p.Times == nil { - return budgetNone - } - if _, budgeted := deliveryBudgets[p.Times.Class]; !budgeted { - return budgetNone - } - for _, tier := range p.Tiers { - for _, m := range tier { - if rollsOut(m) { - return p.Times.Class - } - } - } - return budgetNone -} - -// walkBound is a walk's bound by its budget class: twice its budget, or fifteen minutes with none. -func walkBound(class string) time.Duration { - if b, ok := deliveryBudgets[class]; ok { - return 2 * b - } - return walkNoBudgetBound -} - -// walkSends is what a walk will send at its start, per module the machines it goes to: each module of its tiers -// whose policy rolls it out, on every machine running it (for a release plan, those of its order). -func walkSends(p inventory.Plan, running func(string) []string, rollsOut func(string) bool) map[string][]string { - out := map[string][]string{} - for _, tier := range p.Tiers { - for _, m := range tier { - if s := p.Modules[m]; s != nil && s.State == planDeleted { - continue - } - if p.Release == nil && !rollsOut(m) { - continue - } - for _, n := range running(m) { - if p.Release != nil && !slices.Contains(p.Release.Order, n) { - continue - } - out[m] = appendNew(out[m], n) - } - } - } - return out -} - -// walkReached is what a walk sent by its end, per module the machines it reached: its first machines, its rest, -// and every module a gated send carried with it. A release plan reached the machines it walked: those done and -// the one being judged. -func walkReached(p inventory.Plan, running func(string) []string) map[string][]string { - out := map[string][]string{} - carried := func(g *inventory.PlanGate) { - if g == nil { - return - } - for _, c := range g.Carried { - out[c.Module] = appendNew(out[c.Module], c.Node) - } - } - if r := p.Release; r != nil { - reached := append([]string{}, r.Done...) - if r.Gate != nil { - reached = appendNew(reached, r.Gate.Machines...) - carried(r.Gate) - } - for _, g := range r.Gates { - if g != nil && !g.Pending { - reached = appendNew(reached, g.Machines...) - carried(g) - } - } - for _, tier := range p.Tiers { - for _, m := range tier { - for _, n := range running(m) { - if slices.Contains(reached, n) { - out[m] = appendNew(out[m], n) - } - } - } - } - return out - } - for m, s := range p.Modules { - if s == nil { - continue - } - out[m] = appendNew(out[m], s.First...) - for n := range s.Rest { - out[m] = appendNew(out[m], n) - } - carried(s.Gate) - if len(out[m]) == 0 { - delete(out, m) - } - } - return out -} - -// touchesOf turns what a walk moves, per module, into what it touches, per machine: the modules it moves there, -// and, where it moves the tool runner, every module there whose tools it serves, and where it moves the -// node-engine, every module there. Sorted by machine and module. -func touchesOf(moved map[string][]string, on map[string][]string, shelf map[string]catalogue.Manifest) []walkTouch { - byMachine := map[string][]string{} - for m, machines := range moved { - for _, n := range machines { - byMachine[n] = appendNew(byMachine[n], m) - } - } - out := make([]walkTouch, 0, len(byMachine)) - for n, modules := range byMachine { - t := walkTouch{Machine: n, Modules: modules} - engine, runner := slices.Contains(modules, hostModule), slices.Contains(modules, broker.RuntimeModule) - if engine || runner { - for _, m := range on[n] { - if slices.Contains(t.Modules, m) || (!engine && !servesTools(shelf[m])) { - continue - } - t.Modules = append(t.Modules, m) - t.Carried = append(t.Carried, m) - } - } - sort.Strings(t.Modules) - sort.Strings(t.Carried) - out = append(out, t) - } - sort.Slice(out, func(i, j int) bool { return out[i].Machine < out[j].Machine }) - return out -} - -// servesTools says the tool runner serves something of this module: a bundle it loads entrypoints from, or the -// module's own tools. -func servesTools(m catalogue.Manifest) bool { - if len(m.Tools) > 0 { - return true - } - for _, b := range m.Bundles { - if len(b.Loads) > 0 { - return true - } - } - return false -} - -// walkRead is the facts a walk's touches are worked out from, read once per event. -type walkRead struct { - running func(string) []string - rollsOut func(string) bool - onMachine func(moved map[string][]string) map[string][]string - shelf func() map[string]catalogue.Manifest -} - -// readWalkFacts reads what the walk's touches need. Without facts (a test, a command with no store), nothing runs -// anywhere and every module rolls out: the touches are then what the walk itself recorded. -func readWalkFacts(ctx context.Context, facts walkFacts, p inventory.Plan) walkRead { - cantRead := func(what string, err error) { - fmt.Fprintf(os.Stderr, "%s: %s cannot be read for what the walk touches: %v\n", p.ID, what, err) - } - r := walkRead{ - running: func(string) []string { return nil }, - rollsOut: func(string) bool { return true }, - onMachine: func(map[string][]string) map[string][]string { return nil }, - shelf: func() map[string]catalogue.Manifest { return nil }, - } - if facts == nil { - return r - } - running := map[string][]string{} - r.running = func(m string) []string { - if on, read := running[m]; read { - return on - } - on, err := facts.Running(ctx, m) - if err != nil { - cantRead("where "+m+" runs", err) - } - running[m] = on - return on - } - rolls := map[string]bool{} - r.rollsOut = func(m string) bool { - if yes, read := rolls[m]; read { - return yes - } - u, err := facts.UpgradeOf(ctx, m) - if err != nil { - cantRead(m+"'s policy", err) - } - // A policy that cannot be read is taken to roll out: the walk is then held more, never less. - rolls[m] = err != nil || u.RollOut - return rolls[m] - } - r.onMachine = func(moved map[string][]string) map[string][]string { - out := map[string][]string{} - for _, core := range []string{hostModule, broker.RuntimeModule} { - for _, n := range moved[core] { - if _, read := out[n]; read { - continue - } - var there []string - if sent, known, err := facts.SentBuilds(ctx, n); err != nil { - cantRead("what "+n+" was sent", err) - } else if known { - for m := range sent { - there = append(there, m) - } - } - if assigned, err := facts.Assigned(ctx, n); err != nil { - cantRead("what "+n+" is assigned", err) - } else { - there = appendNew(there, assigned...) - } - sort.Strings(there) - out[n] = there - } - } - return out - } - r.shelf = func() map[string]catalogue.Manifest { - shelf, err := facts.Catalogue(ctx) - if err != nil { - cantRead("the catalogue", err) - } - return shelf - } - return r -} diff --git a/cmd/mesh-controller/walk_said_integration_test.go b/cmd/mesh-controller/walk_said_integration_test.go deleted file mode 100644 index 52f09231..00000000 --- a/cmd/mesh-controller/walk_said_integration_test.go +++ /dev/null @@ -1,285 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "os" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/builderkeys" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -func TestAWalkLetGoAndStoppedByVerbsIsSaidStartedAndEnded(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - next := planMovedHeard(t) - register(t, open, catalogue.Manifest{Module: "app", Version: "1"}) - if _, err := assign(ctx, open, "anchor", "app"); err != nil { - t.Fatal(err) - } - cut := time.Now().UTC() - waiting := inventory.Plan{ID: "plan-308-live", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c8c8c8c8", - Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {}}, - Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}, - Times: &inventory.PlanTimes{Cut: &cut, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(ctx, &waiting); err != nil { - t.Fatal(err) - } - if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil { - t.Fatal(err) - } - started := next(inventory.WalkStarted) - touches, _ := json.Marshal(started["touches"]) - if started["walk"] != waiting.ID || string(touches) != `[{"machine":"anchor","modules":["app"]}]` || - started["budget_class"] != inventory.ClassLeaf || - started["bound_seconds"] != float64(2*deliveryBudgets[inventory.ClassLeaf]/time.Second) { - t.Fatalf("the start: %v", started) - } - if err := deliveryCommand(ctx, []string{"stop", waiting.ID, "--why", "a test", "--by", "a person"}); err != nil { - t.Fatal(err) - } - ended := next(inventory.WalkEnded) - touches, _ = json.Marshal(ended["touches"]) - if ended["walk"] != waiting.ID || ended["outcome"] != inventory.WalkStopped || string(touches) != `[]` { - t.Fatalf("the end, which sent nothing: %v", ended) - } -} - -func planMovedHeard(t *testing.T) func(phase string) map[string]any { - t.Helper() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - before := handActConn - handActConn = js.Conn() - t.Cleanup(func() { handActConn = before }) - heard := make(chan *nats.Msg, 64) - sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - return func(phase string) map[string]any { - t.Helper() - deadline := time.After(5 * time.Second) - for { - select { - case m := <-heard: - var said map[string]any - if err := json.Unmarshal(m.Data, &said); err != nil { - t.Fatal(err) - } - if said["phase"] == phase { - return said - } - case <-deadline: - t.Fatalf("no walk was said %s", phase) - } - } - } -} - -func TestAWalkStoppedOrClosedByAPersonIsSaidEndedStopped(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - next := planMovedHeard(t) - register(t, open, catalogue.Manifest{Module: "app", Version: "1"}) - if _, err := assign(ctx, open, "anchor", "app"); err != nil { - t.Fatal(err) - } - for _, verb := range []string{"stop", "close"} { - cut := time.Now().UTC() - walk := inventory.Plan{ID: "plan-308-" + verb, Repository: "novox/mesh-catalog", Branch: "main", - Commit: "c8c8c8c8", Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {}}, - Times: &inventory.PlanTimes{Cut: &cut, Started: &cut, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(ctx, &walk); err != nil { - t.Fatal(err) - } - if err := plansCommand(ctx, []string{verb, walk.ID, "--why", "a test"}); err != nil { - t.Fatal(err) - } - ended := next(inventory.WalkEnded) - if ended["walk"] != walk.ID || ended["outcome"] != inventory.WalkStopped { - t.Fatalf("plans %s: the end is %v", verb, ended) - } - p, err := open.inventory.PlanByID(ctx, walk.ID) - if err != nil { - t.Fatal(err) - } - if p.Outcome() != inventory.WalkStopped || p.Delivery == nil || p.Delivery.StoppedWhy != "a test" { - t.Fatalf("plans %s kept the walk %s, %+v", verb, p.Outcome(), p.Delivery) - } - } -} - -func TestAWalkRetriedByAPersonIsSaidStartedAgain(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asksRecorded(t) - next := planMovedHeard(t) - twoTiers(t, open) - before := time.Now().UTC().Add(-time.Hour) - ended := before.Add(time.Minute) - failed := inventory.Plan{ID: "plan-308-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee", - Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}}, - Note: "a failed to build in tier 0", - Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", - Why: "killed by hand"}}, - Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(ctx, &failed); err != nil { - t.Fatal(err) - } - if err := plansCommand(ctx, []string{"retry", failed.ID}); err != nil { - t.Fatal(err) - } - started := next(inventory.WalkStarted) - if started["walk"] != failed.ID || started["budget_class"] != inventory.ClassLeaf { - t.Fatalf("the retried start: %v", started) - } -} - -func TestABacklogReleasedByAPersonIsSaidStarted(t *testing.T) { - b := aBacklog(t) - next := planMovedHeard(t) - if err := backlogCommand(t.Context(), "release-backlog", []string{"--why", "a test"}); err != nil { - t.Fatal(err) - } - started := next(inventory.WalkStarted) - if p := b.release(t); started["walk"] != p.ID { - t.Fatalf("the release's start: %v", started) - } -} - -func asTyped(t *testing.T, args ...string) error { - t.Helper() - was := os.Args - os.Args = append([]string{"mesh-controller"}, args...) - t.Cleanup(func() { os.Args = was }) - err := run() - // run gives back the lease as the process would end; the test's process goes on, and takes it again. - theLease.mu.Lock() - theLease.held, theLease.kv, theLease.noBus = nil, nil, false - theLease.mu.Unlock() - return err -} - -func TestAWalkStoppedByHandAndRetriedEndsFailedWhenItFailsAgain(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asksRecorded(t) - twoTiers(t, open) - for _, again := range [][]string{{"plans", "retry"}} { - before := time.Now().UTC().Add(-time.Hour) - walk := aStartedWalkAsking(t, open, "plan-308-again-"+again[0], "build-1", before) - if err := asTyped(t, "plans", "stop", walk.ID, "--why", "a test"); err != nil { - t.Fatal(err) - } - // The build asked before the stop fails after it. - p, _ := open.inventory.PlanByID(ctx, walk.ID) - p.Modules["a"].State, p.Modules["a"].Why = "failed", "it broke" - if err := open.inventory.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - args := again - if again[0] == "plans" { - args = append(append([]string(nil), again...), walk.ID) - } - if err := asTyped(t, args...); err != nil { - t.Fatal(err) - } - p, _ = open.inventory.PlanByID(ctx, walk.ID) - if p.State != inventory.PlanBuilding || (p.Delivery != nil && p.Delivery.Stopped != "") { - t.Fatalf("%s: the walk is %s and keeps its stop: %+v", again[0], p.State, p.Delivery) - } - // The retry takes a from its key (it asks no build), at the commit the walk carries. - if a := p.Modules["a"]; a == nil || !a.Key || a.KeyAt != "c0ffee" { - t.Fatalf("%s: the retried walk does not take a from its key: %+v", again[0], a) - } - // The key fails again: the builder's record says so, and the look settles the walk from it. - wholeKeys := openPlanKeys - openPlanKeys = func() (planKeyReads, func(), error) { - all, done, err := wholeKeys() - return keysFailing{all, "a", "it broke again"}, done, err - } - t.Cleanup(func() { openPlanKeys = wholeKeys }) - advancePlans(ctx, open) - if p, _ = open.inventory.PlanByID(ctx, walk.ID); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed { - t.Fatalf("%s: failing again, the walk is %s and ends %s", again[0], p.State, p.Outcome()) - } - } -} - -// novox/hq ADR 0505 §2: a walk's start is said again, over the real bus and against the real store, with a machine it -// sends nothing to whose declaration differs from what it was last sent (here: assigned a module and never pushed), -// marked declared. -func TestAWalksStartIsSaidAgainWithAMachineWhoseDeclarationChanged(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - next := planMovedHeard(t) - register(t, open, catalogue.Manifest{Module: "app", Version: "1"}) - if _, err := assign(ctx, open, "anchor", "app"); err != nil { - t.Fatal(err) - } - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - core, err := js.Conn().JetStream() - if err != nil { - t.Fatal(err) - } - body, err := json.Marshal(planMovedBody{Plan: inventory.Plan{ID: "plan-505", State: inventory.PlanBuilding}, - walkSaid: &walkSaid{Walk: "plan-505", Phase: inventory.WalkStarted, BudgetClass: inventory.ClassLeaf, - BoundSeconds: 360, Touches: []walkTouch{}}}) - if err != nil { - t.Fatal(err) - } - sayDeclared(ctx, link.OverNATS{Conn: js.Conn(), JS: core}, open, body) - again := next(inventory.WalkStarted) - raw, _ := json.Marshal(again["touches"]) - var touches []walkTouch - _ = json.Unmarshal(raw, &touches) - anchor := false - for _, tc := range touches { - anchor = anchor || (tc.Machine == "anchor" && tc.Declared && len(tc.Modules) == 0) - if !tc.Declared { - t.Fatalf("a machine the walk sends nothing to was said sent: %s", raw) - } - } - if again["walk"] != "plan-505" || again[walkDeclaredRead] != true || !anchor { - t.Fatalf("said again: %v", again) - } -} - -// keysFailing is the stand-in builder's keys but for one module, whose key is failed in these words. -type keysFailing struct { - planKeyReads - module string - words string -} - -func (k keysFailing) Build(ctx context.Context, module, fp string) (builderkeys.Key, error) { - if module == k.module { - return builderkeys.Key{Module: module, Fingerprint: fp, Found: true, Revision: 1, - Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildFailed, Error: k.words}}, nil - } - return k.planKeyReads.Build(ctx, module, fp) -} diff --git a/cmd/mesh-controller/walk_said_test.go b/cmd/mesh-controller/walk_said_test.go deleted file mode 100644 index f279146a..00000000 --- a/cmd/mesh-controller/walk_said_test.go +++ /dev/null @@ -1,269 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "reflect" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" -) - -// fakeWalkFacts is the inventory as far as a walk's touches read it. -type fakeWalkFacts struct { - running map[string][]string - records map[string]bool // modules whose policy records rather than rolls out - sent map[string]map[string]string - assigned map[string][]string - shelf map[string]catalogue.Manifest -} - -func (f fakeWalkFacts) Running(_ context.Context, m string) ([]string, error) { - return f.running[m], nil -} -func (f fakeWalkFacts) UpgradeOf(_ context.Context, m string) (inventory.Upgrade, error) { - return inventory.Upgrade{RollOut: !f.records[m]}, nil -} -func (f fakeWalkFacts) Assigned(_ context.Context, n string) ([]string, error) { - return f.assigned[n], nil -} -func (f fakeWalkFacts) SentBuilds(_ context.Context, n string) (map[string]string, bool, error) { - s, ok := f.sent[n] - return s, ok, nil -} -func (f fakeWalkFacts) Catalogue(context.Context) (map[string]catalogue.Manifest, error) { - return f.shelf, nil -} - -// theMesh is four machines: the tool runner on each, messenger and mesh-issues on novox, gitea on ace. -func theMesh() fakeWalkFacts { - return fakeWalkFacts{ - running: map[string][]string{ - broker.RuntimeModule: {"ace", "g14", "novox", "shanks"}, - "messenger": {"novox"}, - "gitea": {"ace"}, - "plex": {"ace"}, - }, - sent: map[string]map[string]string{ - "novox": {broker.RuntimeModule: "c0", "messenger": "c1", "mesh-issues": "c2", "postgres": "c3"}, - "ace": {broker.RuntimeModule: "c0", "gitea": "c4"}, - }, - assigned: map[string][]string{"g14": {broker.RuntimeModule, "desktop"}, "shanks": {broker.RuntimeModule}}, - shelf: map[string]catalogue.Manifest{ - "messenger": {Module: "messenger", Tools: []string{"notify"}}, - "mesh-issues": {Module: "mesh-issues", Bundles: []catalogue.Bundle{{Name: "tools", Loads: []string{"main"}}}}, - "postgres": {Module: "postgres"}, - "gitea": {Module: "gitea", Tools: []string{"gitea_api"}}, - "desktop": {Module: "desktop", Tools: []string{"notify_send"}}, - }, - } -} - -func walkOfTiers(class string, tiers ...[]string) inventory.Plan { - cut := time.Date(2026, 10, 12, 11, 0, 0, 0, time.UTC) - started := cut.Add(time.Second) - p := inventory.Plan{ID: "plan-308", State: inventory.PlanBuilding, Tiers: tiers, - Modules: map[string]*inventory.PlanModule{}, Times: &inventory.PlanTimes{Cut: &cut, Class: class, Started: &started}} - for _, tier := range tiers { - for _, m := range tier { - p.Modules[m] = &inventory.PlanModule{} - } - } - p.WalkSaid = inventory.WalkStarted - return p -} - -// novox/hq ADR 0308 How it is checked §1: the start names the modules the walk moves and the machines it sends -// them to, and every module a moved tool runner carries there. -func TestAWalksStartNamesWhatItTouchesWithWhatTheToolRunnerCarries(t *testing.T) { - p := walkOfTiers(inventory.ClassCore, []string{broker.RuntimeModule}, []string{"messenger"}) - w := sayWalk(t.Context(), theMesh(), p) - if w == nil || w.Walk != "plan-308" || w.Phase != inventory.WalkStarted || w.Outcome != "" { - t.Fatalf("the start is said: %+v", w) - } - want := []walkTouch{ - {Machine: "ace", Modules: []string{"gitea", broker.RuntimeModule}, Carried: []string{"gitea"}}, - {Machine: "g14", Modules: []string{"desktop", broker.RuntimeModule}, Carried: []string{"desktop"}}, - // postgres serves no tools: the tool runner does not carry it. - {Machine: "novox", Modules: []string{"mesh-issues", "messenger", broker.RuntimeModule}, Carried: []string{"mesh-issues"}}, - {Machine: "shanks", Modules: []string{broker.RuntimeModule}}, - } - if !reflect.DeepEqual(w.Touches, want) { - t.Fatalf("touches:\n got %+v\nwant %+v", w.Touches, want) - } - if w.BudgetClass != inventory.ClassCore || w.BoundSeconds != 1200 || w.StartedAt == nil { - t.Fatalf("a core walk is bound at 20 minutes: %+v", w) - } -} - -// A moved node-engine carries every module on its machine; a module whose policy records is not sent, and not -// touched. -func TestAMovedNodeEngineCarriesEverythingOnItsMachine(t *testing.T) { - f := theMesh() - f.running[hostModule] = []string{"novox"} - f.records = map[string]bool{"plex": true} - p := walkOfTiers(inventory.ClassCore, []string{hostModule, "plex"}) - w := sayWalk(t.Context(), f, p) - want := []walkTouch{{Machine: "novox", - Modules: []string{"mesh-host", "mesh-issues", "messenger", "node-tools", "postgres"}, - Carried: []string{"mesh-issues", "messenger", "node-tools", "postgres"}}} - if !reflect.DeepEqual(w.Touches, want) { - t.Fatalf("touches:\n got %+v\nwant %+v", w.Touches, want) - } -} - -// The end names what the walk reached — its first machines, its rest, what a gated send carried — and its -// outcome; the modules a reached tool runner carries are touched there too. -func TestAWalksEndNamesWhatItReachedAndItsOutcome(t *testing.T) { - p := walkOfTiers(inventory.ClassCore, []string{broker.RuntimeModule}, []string{"messenger"}) - ended := p.Times.Started.Add(4 * time.Minute) - p.Times.Ended = &ended - p.State, p.WalkSaid = inventory.PlanDone, inventory.WalkEnded - p.Modules[broker.RuntimeModule] = &inventory.PlanModule{First: []string{"shanks"}, - Rest: map[string]inventory.SentDeclaration{"novox": {}}, - Gate: &inventory.PlanGate{Carried: []inventory.CarriedMove{{Module: "gitea", Node: "shanks"}}}} - // messenger was built and found unchanged: no move, not reached. - p.Modules["messenger"] = &inventory.PlanModule{Why: "no move"} - w := sayWalk(t.Context(), theMesh(), p) - if w.Phase != inventory.WalkEnded || w.Outcome != inventory.WalkDelivered || w.EndedAt == nil { - t.Fatalf("the end is said with its outcome: %+v", w) - } - want := []walkTouch{ - {Machine: "novox", Modules: []string{"mesh-issues", "messenger", broker.RuntimeModule}, - Carried: []string{"mesh-issues", "messenger"}}, - {Machine: "shanks", Modules: []string{"gitea", broker.RuntimeModule}}, - } - if !reflect.DeepEqual(w.Touches, want) { - t.Fatalf("touches:\n got %+v\nwant %+v", w.Touches, want) - } - for outcome, end := range map[string]func(*inventory.Plan){ - inventory.WalkFailed: func(p *inventory.Plan) { p.State = inventory.PlanFailed }, - inventory.WalkStopped: func(p *inventory.Plan) { - p.State, p.Delivery = inventory.PlanFailed, &inventory.PlanDelivery{Stopped: "a person"} - }, - } { - q := p - end(&q) - if got := sayWalk(t.Context(), theMesh(), q).Outcome; got != outcome { - t.Errorf("outcome %q, want %q", got, outcome) - } - } -} - -// novox/hq ADR 0308 How it is checked §1 and decision 5: the bound is twice the largest budget in force among -// the walk's deliveries — 20 minutes for a core walk, twice the leaf budget for a leaf walk — and 15 minutes, not -// doubled, for a walk none of whose deliveries has a budget. -func TestAWalksBoundIsTwiceItsBudgetOrFifteenMinutesWithNone(t *testing.T) { - f := theMesh() - f.records = map[string]bool{"plex": true} - release := walkOfTiers("", []string{"gitea"}) - release.Times, release.Release = &inventory.PlanTimes{}, &inventory.PlanRelease{Order: []string{"ace"}} - for name, c := range map[string]struct { - p inventory.Plan - class string - bound int - }{ - "core": {walkOfTiers(inventory.ClassCore, []string{"messenger"}), inventory.ClassCore, 1200}, - "leaf": {walkOfTiers(inventory.ClassLeaf, []string{"messenger"}), inventory.ClassLeaf, int(2 * deliveryBudgets[inventory.ClassLeaf] / time.Second)}, - "held for a person": {walkOfTiers(inventory.ClassLeaf, []string{"plex"}), budgetNone, 900}, - "a release": {release, budgetNone, 900}, - "kept without a class": {walkOfTiers("", []string{"messenger"}), budgetNone, 900}, - } { - w := sayWalk(t.Context(), f, c.p) - if w.BudgetClass != c.class || w.BoundSeconds != c.bound { - t.Errorf("%s: %s bound %ds, want %s %ds", name, w.BudgetClass, w.BoundSeconds, c.class, c.bound) - } - } - // The leaf budget in force is three minutes (ADR 0304 decision 4): a leaf walk is bound at six. - if deliveryBudgets[inventory.ClassLeaf] == 3*time.Minute { - if w := sayWalk(t.Context(), f, walkOfTiers(inventory.ClassLeaf, []string{"messenger"})); w.BoundSeconds != 360 { - t.Errorf("a leaf walk at a three-minute budget is bound at 360 s: %d", w.BoundSeconds) - } - } -} - -// plan-moved's body: the walk whole as before, and on its start the walk's fields beside it; on any other save -// none of them. -func TestPlanMovedCarriesTheWalkWholeAndOnItsStartWhatItTouches(t *testing.T) { - p := walkOfTiers(inventory.ClassLeaf, []string{"messenger"}) - body, err := json.Marshal(planMovedBody{Plan: p, walkSaid: sayWalk(t.Context(), theMesh(), p)}) - if err != nil { - t.Fatal(err) - } - var said map[string]any - if err := json.Unmarshal(body, &said); err != nil { - t.Fatal(err) - } - for _, k := range []string{"id", "state", "tiers", "modules", "times", "walk", "phase", "touches", "bound_seconds", "budget_class", "started_at"} { - if _, ok := said[k]; !ok { - t.Errorf("plan-moved lacks %q: %s", k, body) - } - } - if _, ok := said["outcome"]; ok { - t.Errorf("a start says no outcome: %s", body) - } - if said["walk"] != "plan-308" || said["phase"] != "started" || said["budget_class"] != "leaf" { - t.Errorf("the walk's fields: %s", body) - } - p.WalkSaid = "" - body, _ = json.Marshal(planMovedBody{Plan: p, walkSaid: sayWalk(t.Context(), theMesh(), p)}) - said = map[string]any{} - _ = json.Unmarshal(body, &said) - for _, k := range []string{"walk", "phase", "touches", "bound_seconds"} { - if _, ok := said[k]; ok { - t.Errorf("a save neither starting nor ending the walk says %q: %s", k, body) - } - } - // A walk that touches nothing says an empty list, never null. - q := walkOfTiers(inventory.ClassLeaf, []string{"nobody-runs-this"}) - body, _ = json.Marshal(planMovedBody{Plan: q, walkSaid: sayWalk(t.Context(), theMesh(), q)}) - said = map[string]any{} - _ = json.Unmarshal(body, &said) - if touches, ok := said["touches"].([]any); !ok || len(touches) != 0 { - t.Errorf("touches nothing: %s", body) - } -} - -// Live, with the store and a bus: a walk let go by the delivery's verb is said started with what it touches and -// its bound, and stopped by the verb is said ended, stopped (novox/hq ADR 0308 How it is checked §1). - -// planMovedHeard raises a bus of the test's own, points the verbs' hand-act connection at it, and returns what -// waits for the next plan-moved said with the given phase, failing the test when none comes within 5 s. - -// Live, with the store and a bus: a walk a person stops with `plans stop` is said ended, outcome stopped, by the -// verb itself, and is kept stopped by that person; `plans close` ends one the same way (novox/hq ADR 0308 -// decision 1, review 15 on mesh-controller#247). - -// Live, with the store and a bus: a failed walk retried with `plans retry` starts again, and the verb says its -// start (novox/hq ADR 0308 decision 1, review 15 on mesh-controller#247). - -// Live, with the store and a bus: the release plan `upgrade release-backlog` opens is said started by the verb -// (novox/hq ADR 0308 decision 1, review 15 on mesh-controller#247). - -// asTyped runs a verb as the controller's command line runs it, through run, as a person's terminal would. - -// aStartedWalkAsking keeps a live walk, said started, whose first tier asked a as build. -func aStartedWalkAsking(t *testing.T, open *stores, id, build string, asked time.Time) inventory.Plan { - t.Helper() - walk := inventory.Plan{ID: id, Repository: "novox/a", Branch: "main", Commit: "c0ffee", Created: asked, - State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, - Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: build}}, - Times: &inventory.PlanTimes{Cut: &asked, Started: &asked, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(t.Context(), &walk); err != nil { - t.Fatal(err) - } - return walk -} - -// Live, with the store and a bus: `cancel ` and `clear` fail the walk that asked the build, and the verb -// says its end, failed (novox/hq ADR 0308 decision 1, review 21 on mesh-controller#247). - -// Live, with the store and a bus: `rebuild ` joining a failed walk sets it building again, and the verb -// says its start (novox/hq ADR 0308 decision 1, review 21 on mesh-controller#247). - -// Live, with the store: a walk a person stopped and then retried, by `plans retry` or by `rebuild`, walks again -// without the stop, so when it fails again it ends failed, not stopped, and the search for the merge that -// brought the failure is not kept from it (novox/hq ADR 0308 decision 1, review 21 on mesh-controller#247). diff --git a/cmd/mesh-controller/walk_wait_rebuild_754_test.go b/cmd/mesh-controller/walk_wait_rebuild_754_test.go deleted file mode 100644 index 97b134cb..00000000 --- a/cmd/mesh-controller/walk_wait_rebuild_754_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package main - -import ( - "reflect" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq issue 754: a tier partly asked asks the rest; a module held built that the plan never asked — the live -// shape, a hand rebuild taken in during the wait, built after the plan was made — is asked again; one asked is kept. -func TestATierPartlyAskedAsksTheRest(t *testing.T) { - created := time.Date(2026, 10, 14, 23, 30, 0, 0, time.UTC) - earlier, later := created.Add(-time.Hour), created.Add(7*time.Minute) - p := inventory.Plan{Created: created, Tiers: [][]string{{"build-agent", "gitea", "route-proxy", "stale", "kept", "own"}}, - Modules: map[string]*inventory.PlanModule{ - "gitea": {}, - "route-proxy": {State: "built", BuiltAt: &later, Commit: "c51357f8"}, - "stale": {State: "built", BuiltAt: &earlier}, - "kept": {State: "asked", AskedAt: &created, Build: "build-kept"}, - "own": {State: "built", AskedAt: &created, BuiltAt: &later, Build: "build-own"}, - }} - if got, want := unaskedOf(p), []string{"build-agent", "gitea", "route-proxy", "stale"}; !reflect.DeepEqual(got, want) { - t.Fatalf("unasked %v, want %v", got, want) - } - if !askedInTier(p) { - t.Fatal("a tier with a module asked under an id reads as never asked") - } - p.Modules["kept"], p.Modules["own"] = nil, nil - if askedInTier(p) { - t.Fatal("a tier the plan never asked reads as asked") - } -} - -// novox/hq issue 754, as seen on 2026-10-14: a rebuild lands while the walk waits for its word; the walk is let go; -// it asks every module of its tier, and completes it once those builds come in. diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index 208c98da..ec1325a7 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -6,7 +6,6 @@ import ( "fmt" "os" "slices" - "sort" "sync" "time" @@ -49,25 +48,9 @@ type signalFacts struct { // since then at the most. toolsHeardFrom time.Time - plans []planFacts - plansErr error - // waits are the walks waiting for their delivery's word (S16, novox/hq ADR 0239). - waits []waitFacts - // batches are the batches not yet cut (S18, S19, novox/hq ADR 0276). - batches []batchFacts - // failed are the delivery plans that failed and are asked of the operator (S23, novox/hq ADR 0449). - failed []failedPlanFacts - failedErr error - // held are the machines with moves of recorded modules waiting for a person's push (S24, novox/hq issue 1732). - held []heldFacts - heldErr error // images are the modules left out of a send because the artifact store lacks an image they name (S25, issue 1742). images []imageFacts imagesErr error - // refusedSends are the sends a machine refused within the hour for the generation they were composed - // from, each naming its sender (S20, novox/hq issue 234). - refusedSends []inventory.Send - refusedSendsErr error // doing is every machine whose last report was not a clean apply, with since when it has said the same // failure (S21, novox/hq issue 233). doing []inventory.Doing @@ -120,9 +103,6 @@ type signalFacts struct { // facts is the snapshot this controller keeps for merge checks (S14). facts factsFacts - - bus busFacts - busErr error } // factsFacts is when the newest facts snapshot was taken, when this controller began keeping it, and @@ -347,13 +327,6 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts { } } f.machines, f.machinesErr = w.gatherMachines(ctx, inv, now) - f.bus, f.busErr = gatherBus(ctx, inv) - f.plans, f.waits, f.plansErr = gatherPlans(ctx, inv, now, f.bus.heldByTheBus()) - if f.plansErr == nil { - f.batches, f.plansErr = gatherBatches(ctx, inv, now) - } - f.failed, f.failedErr = gatherFailedPlans(ctx, inv, now) - f.held, f.heldErr = gatherHeldMoves(ctx, inv) f.images, f.imagesErr = gatherMissingImagesHere(ctx, inv) f.loop, f.loopErr = w.gatherLoop() if open, err := w.keeper.Open(ctx); err != nil { @@ -390,7 +363,6 @@ func (w *watchdogs) gather(ctx context.Context) *signalFacts { f.deadLettersAging, f.advisoriesErr = link.AgingDeadLetters(w.js.Context(), f.deadLetters) } f.handActs, f.handActsErr = w.gatherHandActs(ctx, now) - f.refusedSends, f.refusedSendsErr = inv.RefusedSendsSince(ctx, now.Add(-generationRefusalsSaid)) f.doing, f.doingErr = inv.NotDoingWhatTheyWereTold(ctx) if f.doingErr == nil && w.js != nil { f.provFailed, f.doingErr = provisioningFailed(w.js.Context()) @@ -504,118 +476,6 @@ func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory return out, nil } -// gatherBus is a new bus build waiting for its step and the walks refused for it (S17, novox/hq issue 336). -func gatherBus(ctx context.Context, inv *inventory.Inventory) (busFacts, error) { - b, err := pendingBus(ctx, inv) - if err != nil { - return busFacts{}, fmt.Errorf("what a bus upgrade would do cannot be read: %w", err) - } - plans, err := inv.OpenPlans(ctx) - if err != nil { - return busFacts{}, fmt.Errorf("the open plans cannot be read: %w", err) - } - f := busWaitsOf(plans, b, busRefusedFirst.of) - busRefusedFirst.keepOnly(f.heldByTheBus()) - return f, nil -} - -// gatherPlans is every open plan, its tier's bound from what was measured, and what it waits on; byTheBus are -// the walks held only by the bus's planned step, which S3 leaves to S17. -func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time, byTheBus map[string]bool) ([]planFacts, []waitFacts, error) { - plans, err := inv.OpenPlans(ctx) - if err != nil { - return nil, nil, fmt.Errorf("the open plans cannot be read: %w", err) - } - if len(plans) == 0 { - return nil, nil, nil - } - bounds, err := measuredTierBounds(ctx, inv, now) - if err != nil { - return nil, nil, err - } - var out []planFacts - var waits []waitFacts - for _, p := range plans { - // A walk waiting for its delivery's word is no tier late (novox/hq ADR 0239); how long it has waited is - // S16's, whatever mesh-delivery says or does not say. - if p.Waiting() { - waits = append(waits, waitFacts{id: p.ID, repository: p.Repository, commit: p.Commit, - awaits: p.Delivery.Awaits, since: p.Created, modules: planModules(p), merges: p.Delivery.Merges}) - continue - } - // A walk let go and waiting for the walk before it to end (ADR 0276) is no tier late either: the walk - // before it is the one S3 watches. - if deferred(p) { - continue - } - // A walk waiting on, or held by, its backups is no tier late either: the backup condition is the one voice - // (novox/hq ADR 0404 decision 4, issue 1066). - if waitingOnBackups(p) { - continue - } - bound := bounds.of(p.Repository) - out = append(out, planFacts{id: p.ID, repository: p.Repository, commit: p.Commit, tier: p.Tier, - tiers: len(p.Tiers), entered: inTierSince(p), bound: bound, - waiting: planLineWith(p, now, bound), bus: byTheBus[p.ID]}) - } - return out, waits, nil -} - -// tierBounds is how long a plan's tier may take before it is stalled (S3) and `plans` and `status` call -// it LATE, by repository: three times the ninetieth percentile of the tiers measured for it, and never -// less than tierAtLeast. One bound for the watchdog and the lines a person reads, so the two cannot -// disagree about one plan (novox/hq issue 296). A repository nothing was measured for, or a nil -// tierBounds, is given tierAtLeast. -type tierBounds map[string]time.Duration - -func (b tierBounds) of(repository string) time.Duration { - if d, ok := b[repository]; ok { - return d - } - return tierAtLeast -} - -// measuredTierBounds is the tier bounds from the last fourteen days of measured plan tiers. -func measuredTierBounds(ctx context.Context, inv *inventory.Inventory, now time.Time) (tierBounds, error) { - tiers, err := inv.Durations(ctx, inventory.DurationPlanTier, now.Add(-14*24*time.Hour)) - if err != nil { - return nil, fmt.Errorf("the measured plan tiers cannot be read: %w", err) - } - return boundsOf(tiers), nil -} - -// readTierBounds is measuredTierBounds for a line a person reads: what cannot be read is said, and -// every plan is then given the least bound, as the watchdog gives a repository nothing was measured for. -func readTierBounds(ctx context.Context, inv *inventory.Inventory, now time.Time) tierBounds { - bounds, err := measuredTierBounds(ctx, inv, now) - if err != nil { - fmt.Fprintf(os.Stderr, "%v; a plan is called late past %s\n", err, tierAtLeast) - } - return bounds -} - -func boundsOf(tiers []inventory.Duration) tierBounds { - measured := map[string][]time.Duration{} - for _, d := range tiers { - measured[d.Subject] = append(measured[d.Subject], d.Took) - } - out := tierBounds{} - for repository, took := range measured { - out[repository] = max(tierAtLeast, 3*p90(took)) - } - return out -} - -// p90 is the ninetieth percentile of measurements; zero for none. -func p90(took []time.Duration) time.Duration { - if len(took) == 0 { - return 0 - } - sorted := append([]time.Duration(nil), took...) - sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] }) - return sorted[int(0.9*float64(len(sorted)-1))] -} - // loopConsumers are the controller's own consumers (issue 1063). func loopConsumers() []broker.Consumer { return broker.ControllerConsumers() } diff --git a/cmd/mesh-controller/worker_order_test.go b/cmd/mesh-controller/worker_order_test.go deleted file mode 100644 index 86fb9073..00000000 --- a/cmd/mesh-controller/worker_order_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package main - -import ( - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// The holder of the build seat follows the controller that defines its worker (novox/hq issue 206). -// On 2026-10-03 a plan put the build machine in tier 0 and the controller in tier 1; the new build -// machine could not bind the worker the old controller had defined, and nothing could build the -// controller that would have redefined it. The built-by edge from the controller to its build -// machine yields to that order: the controller is built by whichever build machine is running. -func TestTheBuildSeatsHolderFollowsTheControllerThatDefinesItsWorker(t *testing.T) { - edges := []inventory.Edge{ - {From: "build-agent", To: "mesh-controller", Kind: inventory.EdgePackages}, - {From: "build-agent", To: "mesh-controller", Kind: inventory.EdgeWorkerOf}, - {From: "mesh-controller", To: "build-agent", Kind: inventory.EdgeBuiltBy}, - {From: "route-proxy", To: "mesh-controller", Kind: inventory.EdgePackages}, - {From: "route-proxy", To: "build-agent", Kind: inventory.EdgeBuiltBy}, - } - // A packages edge recorded before novox/hq ADR 0267 widens nothing: the controller moved alone moves - // alone, and a change to a package all three build from moves all three, each by its own build source. - if alone := reachableFrom([]string{"mesh-controller"}, edges); len(alone) != 1 { - t.Fatalf("the controller alone, whatever packages its repository: %v", alone) - } - set := reachableFrom([]string{"mesh-controller", "build-agent", "route-proxy"}, edges) - tiers := tiersOf(set, edges) - pos := map[string]int{} - for i, tier := range tiers { - for _, m := range tier { - pos[m] = i - } - } - if pos["mesh-controller"] != 0 { - t.Fatalf("the controller first, built by the build machine that is running: %v", tiers) - } - if pos["build-agent"] <= pos["mesh-controller"] { - t.Fatalf("the build machine after the controller that defines its worker: %v", tiers) - } - // Built-by orders nothing (novox/hq ADR 0360 §3): whichever builder runs builds the proxy, beside the controller. - if pos["route-proxy"] != 0 { - t.Fatalf("what the builder builds waits for a new one: %v", tiers) - } - if hasCycle(tiers, edges) { - t.Fatalf("no cycle here: %v", tiers) - } -}