From 788fb681ec38878fa5cbf77fdfc737ac99cdff61 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 24 Oct 2026 16:42:02 +0200 Subject: [PATCH] Drop the plan state from the store and the assignments: migration 0108 and placement-only assignments (release 6, issue 1945 B9, B4) --- cmd/mesh-controller/adopting_test.go | 76 -- cmd/mesh-controller/ask_bus_wait.go | 52 - cmd/mesh-controller/away.go | 100 -- cmd/mesh-controller/away_test.go | 67 -- cmd/mesh-controller/build.go | 14 - cmd/mesh-controller/build_source.go | 1 - .../build_source_integration_test.go | 44 +- cmd/mesh-controller/busobjects_test.go | 45 - cmd/mesh-controller/chain.go | 34 - cmd/mesh-controller/chain_links.go | 121 --- cmd/mesh-controller/chain_links_test.go | 69 -- cmd/mesh-controller/compose_input.go | 18 +- cmd/mesh-controller/declaration_files_test.go | 78 -- cmd/mesh-controller/durations.go | 148 --- cmd/mesh-controller/generation.go | 266 ----- .../grants_integration_test.go | 91 -- cmd/mesh-controller/handacts_test.go | 17 - cmd/mesh-controller/hold.go | 37 - cmd/mesh-controller/issue348_test.go | 32 - .../issue_204_integration_test.go | 33 - cmd/mesh-controller/issue_204_test.go | 69 -- .../last_send_integration_test.go | 51 - cmd/mesh-controller/last_send_test.go | 121 --- cmd/mesh-controller/lease_integration_test.go | 5 - cmd/mesh-controller/main.go | 6 +- cmd/mesh-controller/moved.go | 70 +- cmd/mesh-controller/moved_bus_test.go | 7 +- .../new_module_approved_test.go | 14 +- cmd/mesh-controller/new_module_bus_test.go | 7 +- .../new_module_on_merge_test.go | 3 - cmd/mesh-controller/nodes.go | 19 - cmd/mesh-controller/plain_words.go | 23 - cmd/mesh-controller/plan.go | 179 ---- .../plan_moved_every_writer_test.go | 123 --- .../plans_gone_store_integration_test.go | 35 + cmd/mesh-controller/probes.go | 26 +- cmd/mesh-controller/provisioned.go | 100 -- cmd/mesh-controller/provisioning_failed.go | 31 + cmd/mesh-controller/push.go | 715 ------------- cmd/mesh-controller/push_test.go | 82 -- cmd/mesh-controller/readable.go | 45 - cmd/mesh-controller/reissue_hearing_test.go | 39 - cmd/mesh-controller/seatverbs.go | 18 +- cmd/mesh-controller/send_stub.go | 23 + .../sendable_integration_test.go | 10 - .../sent_required_by_integration_test.go | 101 -- cmd/mesh-controller/sent_required_by_test.go | 31 - cmd/mesh-controller/status.go | 10 +- cmd/mesh-controller/status_summary.go | 5 - cmd/mesh-controller/unbound_test.go | 3 - cmd/mesh-controller/upgrades.go | 136 ++- cmd/mesh-controller/watchdogs.go | 26 - internal/chain/chain_test.go | 49 +- internal/chain/pending_test.go | 24 - internal/chain/records.go | 22 +- internal/inventory/batches.go | 138 --- internal/inventory/build_lookups.go | 34 + internal/inventory/busy_integration_test.go | 66 -- internal/inventory/busy_test.go | 140 --- internal/inventory/chain.go | 8 - internal/inventory/chain_assignments.go | 22 +- internal/inventory/chain_builds.go | 46 +- internal/inventory/chain_builds_test.go | 1 - .../chain_first_read_lays_out_test.go | 10 + internal/inventory/chain_links.go | 328 ------ internal/inventory/chain_pending_test.go | 3 - internal/inventory/chain_rollouts.go | 986 ------------------ internal/inventory/chain_rollouts_test.go | 374 ------- internal/inventory/chain_sent_copy_test.go | 111 -- internal/inventory/chain_sent_test.go | 116 --- internal/inventory/chain_store_test.go | 70 +- internal/inventory/collection.go | 24 +- internal/inventory/durations.go | 134 --- internal/inventory/durations_test.go | 80 -- internal/inventory/gate.go | 335 ------ internal/inventory/heartbeat_gap_gone_test.go | 12 - internal/inventory/heldhere_test.go | 32 - internal/inventory/hold.go | 149 --- internal/inventory/hold_plans_test.go | 40 - internal/inventory/hold_test.go | 93 -- .../0108-the-plans-leave-the-store.sql | 16 + internal/inventory/nodes.go | 40 +- internal/inventory/older_build_test.go | 126 --- internal/inventory/order_test.go | 128 --- internal/inventory/plan_backups_test.go | 35 - internal/inventory/plan_id_test.go | 47 - internal/inventory/plans.go | 765 -------------- .../inventory/plans_leave_the_store_test.go | 47 + internal/inventory/plans_test.go | 225 ---- internal/inventory/provisioned.go | 83 -- internal/inventory/provisioned_test.go | 124 --- .../registered_held_integration_test.go | 59 -- internal/inventory/sends.go | 364 ------- internal/inventory/sends_test.go | 342 ------ internal/inventory/sent_builds_test.go | 86 -- internal/inventory/sent_bus_users_test.go | 27 - internal/inventory/sent_plans_test.go | 95 -- internal/inventory/source.go | 6 +- internal/inventory/stored_silent_file_test.go | 6 +- .../tier_entered_resume_1886_test.go | 32 - internal/inventory/waiting_merges_test.go | 33 - internal/inventory/walkphases.go | 676 ------------ .../walkphases_582_integration_test.go | 49 - internal/inventory/walkphases_582_test.go | 170 --- .../inventory/walkphases_integration_test.go | 77 -- internal/inventory/walkphases_test.go | 263 ----- internal/inventory/walksaid.go | 81 -- .../inventory/walksaid_integration_test.go | 50 - internal/inventory/walksaid_test.go | 89 -- internal/link/enrolment.go | 32 - 110 files changed, 512 insertions(+), 10584 deletions(-) delete mode 100644 cmd/mesh-controller/ask_bus_wait.go delete mode 100644 cmd/mesh-controller/away.go delete mode 100644 cmd/mesh-controller/away_test.go delete mode 100644 cmd/mesh-controller/chain_links.go delete mode 100644 cmd/mesh-controller/chain_links_test.go delete mode 100644 cmd/mesh-controller/durations.go delete mode 100644 cmd/mesh-controller/generation.go delete mode 100644 cmd/mesh-controller/grants_integration_test.go delete mode 100644 cmd/mesh-controller/hold.go delete mode 100644 cmd/mesh-controller/issue348_test.go delete mode 100644 cmd/mesh-controller/issue_204_integration_test.go delete mode 100644 cmd/mesh-controller/issue_204_test.go delete mode 100644 cmd/mesh-controller/last_send_integration_test.go delete mode 100644 cmd/mesh-controller/last_send_test.go delete mode 100644 cmd/mesh-controller/plan_moved_every_writer_test.go create mode 100644 cmd/mesh-controller/plans_gone_store_integration_test.go delete mode 100644 cmd/mesh-controller/provisioned.go create mode 100644 cmd/mesh-controller/provisioning_failed.go delete mode 100644 cmd/mesh-controller/push_test.go delete mode 100644 cmd/mesh-controller/reissue_hearing_test.go create mode 100644 cmd/mesh-controller/send_stub.go delete mode 100644 cmd/mesh-controller/sent_required_by_integration_test.go delete mode 100644 cmd/mesh-controller/sent_required_by_test.go delete mode 100644 internal/inventory/batches.go create mode 100644 internal/inventory/build_lookups.go delete mode 100644 internal/inventory/busy_integration_test.go delete mode 100644 internal/inventory/busy_test.go delete mode 100644 internal/inventory/chain_rollouts.go delete mode 100644 internal/inventory/chain_rollouts_test.go delete mode 100644 internal/inventory/chain_sent_copy_test.go delete mode 100644 internal/inventory/chain_sent_test.go delete mode 100644 internal/inventory/durations.go delete mode 100644 internal/inventory/durations_test.go delete mode 100644 internal/inventory/gate.go delete mode 100644 internal/inventory/heartbeat_gap_gone_test.go delete mode 100644 internal/inventory/heldhere_test.go delete mode 100644 internal/inventory/hold.go delete mode 100644 internal/inventory/hold_plans_test.go delete mode 100644 internal/inventory/hold_test.go create mode 100644 internal/inventory/migrations/0108-the-plans-leave-the-store.sql delete mode 100644 internal/inventory/older_build_test.go delete mode 100644 internal/inventory/order_test.go delete mode 100644 internal/inventory/plan_backups_test.go delete mode 100644 internal/inventory/plan_id_test.go delete mode 100644 internal/inventory/plans.go create mode 100644 internal/inventory/plans_leave_the_store_test.go delete mode 100644 internal/inventory/plans_test.go delete mode 100644 internal/inventory/provisioned_test.go delete mode 100644 internal/inventory/registered_held_integration_test.go delete mode 100644 internal/inventory/sends.go delete mode 100644 internal/inventory/sends_test.go delete mode 100644 internal/inventory/sent_builds_test.go delete mode 100644 internal/inventory/sent_bus_users_test.go delete mode 100644 internal/inventory/sent_plans_test.go delete mode 100644 internal/inventory/tier_entered_resume_1886_test.go delete mode 100644 internal/inventory/waiting_merges_test.go delete mode 100644 internal/inventory/walkphases.go delete mode 100644 internal/inventory/walkphases_582_integration_test.go delete mode 100644 internal/inventory/walkphases_582_test.go delete mode 100644 internal/inventory/walkphases_integration_test.go delete mode 100644 internal/inventory/walkphases_test.go delete mode 100644 internal/inventory/walksaid.go delete mode 100644 internal/inventory/walksaid_integration_test.go delete mode 100644 internal/inventory/walksaid_test.go diff --git a/cmd/mesh-controller/adopting_test.go b/cmd/mesh-controller/adopting_test.go index 5a5191c1..8232286c 100644 --- a/cmd/mesh-controller/adopting_test.go +++ b/cmd/mesh-controller/adopting_test.go @@ -495,52 +495,6 @@ func TestTheFlipActsOnlyOnThePreviewTheOperatorSaw(t *testing.T) { } } -// The flip holds the node from its checks to its send, so a push composed meanwhile waits and is -// composed after it — never sent after it with the node still adopted. And the send inside the -// flip is not made to wait on the flip's own hold. -func TestTheFlipHoldsTheNodeWhileItSends(t *testing.T) { - open, _ := anAdoptedAnchor(t) - ctx := t.Context() - if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil { - t.Fatal(err) - } - reportsHolding(t, open, heldFile) - preview, err := converge(ctx, open, "anchor", false, "", "") - if err != nil { - t.Fatal(err) - } - var heldElsewhere, heldHere error - sendNodes = func(inner context.Context, open *stores, names []string) error { - // Another caller cannot hold the node while the flip sends it. - waiting, cancel := context.WithTimeout(ctx, 300*time.Millisecond) - defer cancel() - if release, err := open.inventory.HoldNodes(waiting, names); err == nil { - release() - heldElsewhere = errors.New("another caller held the node while the flip sent it") - } - // The flip's own send holds it without waiting on itself. - _, release, err := holdNodes(inner, open, names) - if err != nil { - heldHere = err - return err - } - release() - return nil - } - if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil { - t.Fatal(err) - } - if heldElsewhere != nil || heldHere != nil { - t.Fatalf("%v %v", heldElsewhere, heldHere) - } - // And given back once it is done. - release, err := open.inventory.HoldNodes(ctx, []string{"anchor"}) - if err != nil { - t.Fatal(err) - } - release() -} - // novox/hq ADR 0103: the preview names every kind of thing a module the flip takes holds as found, // not only its files, and the digest changes when any of them does. func TestThePreviewNamesEveryHeldKind(t *testing.T) { @@ -611,33 +565,3 @@ func TestTheFlipIsRefusedOnAnAccountNamingNothingReachable(t *testing.T) { t.Fatal("a refused flip changed something") } } - -// An assignment cannot land between a preview and the flip that takes every module: assigning -// holds the node, so it waits for whatever is converging it. -func TestAssigningWaitsForWhateverIsConvergingTheNode(t *testing.T) { - open, _ := anAdoptedAnchor(t) - ctx := t.Context() - if _, err := take(ctx, open, "anchor", "hello-web", takeOptions{Yes: true}); err != nil { - t.Fatal(err) - } - reportsHolding(t, open, heldFile) - preview, err := converge(ctx, open, "anchor", false, "", "") - if err != nil { - t.Fatal(err) - } - saved, savedPoll := inventory.HoldWaitFor, inventory.HoldPoll - inventory.HoldWaitFor, inventory.HoldPoll = time.Second, 50*time.Millisecond - defer func() { inventory.HoldWaitFor, inventory.HoldPoll = saved, savedPoll }() - - var whileFlipping error - sendNodes = func(context.Context, *stores, []string) error { - _, whileFlipping = assign(ctx, open, "anchor", "notes") - return nil - } - if _, err := converge(ctx, open, "anchor", true, digestIn(t, preview), ""); err != nil { - t.Fatal(err) - } - if !errors.Is(whileFlipping, inventory.ErrNodeBusy) { - t.Fatalf("an assignment landed while the node was being converged: %v", whileFlipping) - } -} diff --git a/cmd/mesh-controller/ask_bus_wait.go b/cmd/mesh-controller/ask_bus_wait.go deleted file mode 100644 index 4488e903..00000000 --- a/cmd/mesh-controller/ask_bus_wait.go +++ /dev/null @@ -1,52 +0,0 @@ -package main - -import ( - "fmt" - "slices" - "sort" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// busWaitBound is how long a module whose ask the bus did not answer is asked again at every look before the plan fails -// (issue 1334). -const busWaitBound = 10 * time.Minute - -// busWaiting says an open plan has a module of its current tier that the bus did not answer an ask of: the look must come back -// for it, as nothing else will when no build of the tier is in flight (issue 1334). -func busWaiting(p inventory.Plan) bool { - return len(busWaitingIn(p)) > 0 -} - -// busWaitingIn are the modules of the plan's current tier left unasked because the bus did not answer, sorted. -func busWaitingIn(p inventory.Plan) []string { - if !p.Open() || p.Tier >= len(p.Tiers) { - return nil - } - var out []string - for _, m := range p.Tiers[p.Tier] { - if s := p.Modules[m]; s != nil && s.State == "" && s.BusWaitFrom != nil { - out = append(out, m) - } - } - sort.Strings(out) - return out -} - -// busWaitWords is how a verb says modules are waiting for the bus: not asked yet, and asked again by the plan at its next look. -func busWaitWords(p inventory.Plan, modules []string) string { - return fmt.Sprintf("%s not asked yet: the bus did not answer; %s asks again at its next look", strings.Join(modules, ", "), p.ID) -} - -// slicesWithout is names without those in drop, in order. -func slicesWithout(names, drop []string) []string { - var out []string - for _, n := range names { - if !slices.Contains(drop, n) { - out = append(out, n) - } - } - return out -} diff --git a/cmd/mesh-controller/away.go b/cmd/mesh-controller/away.go deleted file mode 100644 index fa04575a..00000000 --- a/cmd/mesh-controller/away.go +++ /dev/null @@ -1,100 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "slices" - "sort" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" -) - -// A machine out of touch when a tier is sent is not sent to and not waited for (novox/hq ADR 0435 §1, issue 1161): the -// delivery goes on with the machines heard, says the others were away, and the away ones get the builds by the send -// they are given when heard again (ADR 0353, ADR 0360 §5). Away is the silent check's one verdict, machineFacts.silent -// over the watchman's presence (ADR 0433 §3), never a second rule; a sleeping machine is away once presence says so. - -// awayOf is the machines out of touch at a moment: those the silent check calls silent (a sleeping machine -// is away once presence says so). A machine -// never heard from has not joined and is not away. -func awayOf(machines []machineFacts, now time.Time) map[string]bool { - out := map[string]bool{} - for _, m := range machines { - if m.silent(now) { - out[m.name] = true - } - } - return out -} - -// judgeAway is the walk's reading of which machines are away as a tier is sent: every node with its last-heard time and -// the watchman's verdict on its presence key. A bus that cannot be asked leaves none away, which is what a walk did -// before: a machine is not passed on a reading that was not made. -var judgeAway = func(ctx context.Context, open *stores, now time.Time) (map[string]bool, error) { - nodes, err := open.inventory.Nodes(ctx) - if err != nil { - return nil, err - } - address, err := broker.BusAddress() - if err != nil { - return nil, err - } - js, err := broker.Dial(address) - if err != nil { - return nil, err - } - defer js.Close() - machines := make([]machineFacts, len(nodes)) - for i, n := range nodes { - machines[i] = machineFacts{name: n.Name, lastHeard: n.LastSeen} - } - if err := withPresence(ctx, js.Conn(), machines, now); err != nil { - return nil, err - } - return awayOf(machines, now), nil -} - -// rememberAway keeps on the plan the machines of running that are away: sorted, each once. -func rememberAway(p *inventory.Plan, running []string, away map[string]bool) { - for _, n := range running { - if away[n] && !slices.Contains(p.Away, n) { - p.Away = append(p.Away, n) - } - } - sort.Strings(p.Away) -} - -// runningHeard is the machines of running that the walk waits on for a module: those a send of the walk reached, and -// those it did not pass as away. -func runningHeard(p inventory.Plan, running []string, s *inventory.PlanModule) []string { - return slices.DeleteFunc(slices.Clone(running), func(n string) bool { - if !slices.Contains(p.Away, n) { - return false - } - _, sent := s.Rest[n] - return !sent - }) -} - -// awaySaying is what a plan says of the machines it passed as away, empty when it passed none. -func awaySaying(p inventory.Plan) string { - var said []string - for _, n := range p.Away { - said = append(said, n+" was away: sent when it is heard again") - } - return strings.Join(said, "; ") -} - -// judgedAway reads which machines are away for a tier about to be sent, and says when it cannot. -func judgedAway(ctx context.Context, open *stores, now time.Time) map[string]bool { - away, err := judgeAway(ctx, open, now) - if err != nil { - fmt.Fprintf(os.Stderr, "which machines are away cannot be read, so none is passed over: %v\n", err) - return nil // empty-on-error: none is passed over on a reading that was not made; the error is said just above - } - return away -} diff --git a/cmd/mesh-controller/away_test.go b/cmd/mesh-controller/away_test.go deleted file mode 100644 index f31e9a48..00000000 --- a/cmd/mesh-controller/away_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package main - -import ( - "slices" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// A machine out of touch when a tier is sent is not sent to and not waited for (novox/hq ADR 0435 §1, issue 1161): the -// walk judges it with the silent check's one verdict, machineFacts.silent, and a sleeping machine is away once presence says so. - -// 1161/AC1: away is the silent check's verdict; AC4: a machine heard is not away. -func TestAMachineIsAwayAsTheSilentCheckJudgesIt(t *testing.T) { - now := time.Now().UTC() - machines := []machineFacts{ - {name: "heard", lastHeard: now.Add(-time.Minute)}, - {name: "quiet", lastHeard: now.Add(-silentBound - time.Minute)}, - {name: "stuck", lastHeard: now, presence: presenceSilent}, - {name: "back", lastHeard: now.Add(-time.Hour), presence: presenceHeard}, - {name: "new"}, - } - got := awayOf(machines, now) - for node, want := range map[string]bool{"heard": false, "quiet": true, "stuck": true, "back": false, - "new": false} { - if got[node] != want { - t.Errorf("1161/AC1: %s away is %v; want %v (away: %v)", node, got[node], want, got) - } - } -} - -// 1161/AC1, AC2: the plan keeps the machines of the tier it passed as away, sorted and once. -func TestAPlanKeepsTheMachinesItPassedAsAway(t *testing.T) { - p := &inventory.Plan{} - away := map[string]bool{"g14": true, "ace": true} - rememberAway(p, []string{"anchor", "g14"}, away) - rememberAway(p, []string{"g14", "ace"}, away) - if !slices.Equal(p.Away, []string{"ace", "g14"}) { - t.Fatalf("1161/AC2: the plan keeps %v away; want ace, g14", p.Away) - } -} - -// 1161/AC2: the plan says it, where its other passed machines are said. -func TestAPlanSaysTheMachinesItPassedAsAway(t *testing.T) { - say := awaySaying(inventory.Plan{Away: []string{"g14"}}) - if !strings.Contains(say, "g14 was away: sent when it is heard again") { - t.Fatalf("1161/AC2: the plan says %q", say) - } - if got := awaySaying(inventory.Plan{}); got != "" { - t.Fatalf("1161/AC2: a plan that passed no machine says %q", got) - } -} - -// 1161/AC1: the gate does not wait for a machine passed as away, unless a send reached it; AC4: a machine heard is waited for. -func TestAGateDoesNotWaitForAMachinePassedAsAway(t *testing.T) { - p := inventory.Plan{Away: []string{"g14"}} - none := &inventory.PlanModule{} - if got := runningHeard(p, []string{"anchor", "g14"}, none); !slices.Equal(got, []string{"anchor"}) { - t.Fatalf("1161/AC1: the gate waits on %v; want anchor alone", got) - } - sent := &inventory.PlanModule{Rest: map[string]inventory.SentDeclaration{"g14": {Digest: "d"}}} - if got := runningHeard(p, []string{"anchor", "g14"}, sent); !slices.Equal(got, []string{"anchor", "g14"}) { - t.Fatalf("1161/AC4: a machine a send reached is waited for: %v", got) - } -} diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index 77dc7156..f4c78b0d 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -150,16 +150,6 @@ func registerJudged(ctx context.Context, inv *inventory.Inventory, j takenBuild) return notRegistered(err) } } - // **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard - // twice, or replayed, would otherwise make the build a rollback put back what the module is again, - // and the next push would send it. - if failed, err := inv.GateFailed(ctx, kept.ID); err != nil { - return manifest, err - } else if failed { - return manifest, fmt.Errorf("%s built %s (%s), which failed its gate on its first machine and was put "+ - "back: it is recorded and not registered again — a newer build is", j.by, manifest.Module, - short(j.commit)) - } // Then the module is judged against the list the mesh publishes, as every manifest read again is: one naming a // key that list does not hold, or of another type, is refused here and never enters the catalogue. if j.carried != nil { @@ -283,8 +273,4 @@ type answers struct { // applied, is not well: an assignment somebody made is not made yet, or will not be. pending []inventory.PendingAssignment pendingUnread string - // lastSent is every machine's last recorded send, by name, with what the machine answered of it (novox/hq - // issue 485); lastSentUnread why it could not be read. - lastSent map[string]inventory.Send - lastSentUnread string } diff --git a/cmd/mesh-controller/build_source.go b/cmd/mesh-controller/build_source.go index 619aef0c..85f105c6 100644 --- a/cmd/mesh-controller/build_source.go +++ b/cmd/mesh-controller/build_source.go @@ -524,7 +524,6 @@ type registrations interface { SourceIdentity(ctx context.Context, module string) (int64, error) Catalogued(ctx context.Context) ([]inventory.Entry, error) BuildRequestByID(ctx context.Context, id string) (inventory.BuildRequest, bool, error) - MergeOf(ctx context.Context, repository, commit string) (inventory.BatchedMerge, bool, error) } // mayRegisterIn is mayRegisterFrom on what it reads of the mesh handed in. diff --git a/cmd/mesh-controller/build_source_integration_test.go b/cmd/mesh-controller/build_source_integration_test.go index 66645552..5ae12b9b 100644 --- a/cmd/mesh-controller/build_source_integration_test.go +++ b/cmd/mesh-controller/build_source_integration_test.go @@ -129,41 +129,6 @@ func TestADeliveryFromTheRegisteredRepositoryIsRegistered(t *testing.T) { } } -func TestARollbackNeverPutsBackABuildFromAnotherRepository(t *testing.T) { - open := theCatalogue(t) - ctx := t.Context() - inv := open.inventory - fork := onTrunk("build-1791500000000000000", "agent/mesh-catalog", "git", "modules/sudo", - map[string]any{"module": "sudo", "version": "evil"}) - fork.Commit = "c0ffee0123456789" // the commit sudo was registered at - keptAsked(t, inv, fork.ID, "agent/mesh-catalog", "modules/sudo", false) - if _, _, err := takeIn(ctx, inv, fork); !errors.Is(err, errNotItsSource) { - t.Fatalf("the fork's build was taken in: %v", err) - } - // Asked after the registered build, whenever the test runs: an id naming a fixed moment read as older - // than the registered build once the clock passed it (2026-10-10 02:40 UTC), and the test failed on main. - failed := onTrunk(fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()), "novox/mesh-catalog", "git", "modules/sudo", - map[string]any{"module": "sudo", "version": "2"}) - failed.Commit = "badbadbad0123456" - if _, _, err := takeIn(ctx, inv, failed); err != nil { - t.Fatal(err) - } - record, _, err := inv.BuildByID(ctx, failed.ID) - if err != nil { - t.Fatal(err) - } - previous, found, err := inv.PreviousBuild(ctx, "sudo", "c0ffee0123456789", record) - if err != nil { - t.Fatal(err) - } - if found && previous.ID == fork.ID { - t.Fatalf("a rollback would put back the fork's build %s", previous.ID) - } - if !found || previous.ID != "build-sudo" { - t.Fatalf("a rollback puts back %q (found %v), want the registered build-sudo", previous.ID, found) - } -} - func TestATrunkTheForgeDoesNotProtectRegistersNothing(t *testing.T) { open := theCatalogue(t) ctx := t.Context() @@ -305,10 +270,11 @@ func TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt(t *testing.T) { t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk }) event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7, "head_sha": a.merge.HeadSHA, "merge_commit_sha": commit}) - if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-host", Branch: "main", Commit: commit, - Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil { - t.Fatal(err) - } + var forgeMerge keptMerge + _ = json.Unmarshal(event, &forgeMerge) + wasForge := keptForgeMerge + t.Cleanup(func() { keptForgeMerge = wasForge }) + keptForgeMerge = func(context.Context, string, string, string) (keptMerge, bool, error) { return forgeMerge, true, nil } id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()) if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-host", Seat: "git", Path: "modules/sudo", Ref: "main", Commit: commit, For: c.request}); err != nil { diff --git a/cmd/mesh-controller/busobjects_test.go b/cmd/mesh-controller/busobjects_test.go index 53a9145e..cd3b0b69 100644 --- a/cmd/mesh-controller/busobjects_test.go +++ b/cmd/mesh-controller/busobjects_test.go @@ -7,14 +7,9 @@ import ( "strings" "testing" - "github.com/nats-io/nats.go" - "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - - "github.com/novox/mesh-controller/internal/testbus" ) // novox/hq issue 208: the bus's objects a declaration implies are asserted whenever one is sent, not @@ -165,43 +160,3 @@ func TestASendThatCannotAssertTheBusSaysSoAndRaisesACondition(t *testing.T) { t.Fatalf("a send that asserted everything left the condition open: %v", err) } } - -// Against a real bus, through the send's own grant: a module assigned after start has its consumer -// once a declaration is sent, and a holder assigned after start its seat's worker. -func TestNatsAModuleAssignedAfterStartGetsItsConsumerAtItsFirstSend(t *testing.T) { - url := testbus.URL(t) - open := aMesh(t) - js, err := broker.Dial(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} { - _ = js.Context().DeleteStream(s) - } - // The controller starting, before either was assigned. - if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil { - t.Fatal(err) - } - consumer := aCarriedConsumer(t, open) - worker := aLateHolder(t, open) - _ = js.Context().DeleteConsumer(worker.Stream, worker.Name) - for _, c := range []broker.Consumer{consumer, worker} { - if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); !errors.Is(err, nats.ErrConsumerNotFound) { - t.Fatalf("%s on %s exists before any send; the test proves nothing: %v", c.Name, c.Stream, err) - } - } - - server := link.ConnectNats(js, nil, nil) - if err := (overTheBus{open: open, server: server}).grant(t.Context(), nil); err != nil { - t.Fatal(err) - } - for _, c := range []broker.Consumer{consumer, worker} { - if _, err := js.Context().ConsumerInfo(c.Stream, c.Name); err != nil { - t.Fatalf("%s on %s is not on the bus after a send: %v", c.Name, c.Stream, err) - } - } - if _, raised, _ := conditionsFrom.Get(t.Context(), "bus.objects.unasserted"); raised { - t.Fatal("a send that asserted everything raised a condition") - } -} diff --git a/cmd/mesh-controller/chain.go b/cmd/mesh-controller/chain.go index fa28d8ae..5c58ff7a 100644 --- a/cmd/mesh-controller/chain.go +++ b/cmd/mesh-controller/chain.go @@ -54,7 +54,6 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link. // The requests the move of delivery 1 found never handed over are not asked again: the controller asks no build // (novox/hq ADR 0501; issue 1851, A10), and the builder builds every fingerprint the indexer writes. // And what delivery 1 recorded of what each machine was sent, copied onto the keys of the assignments (CopySentBuilds). - go copySentBuildsSoon(ctx, inv) return nil } @@ -62,39 +61,6 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link. // succeeded: until it has, a module an assignment pulled in may be missing from what a machine is read to run. const kindSentBuildsNotCopied = "sent-builds-not-copied" -// copySentBuildsSoon copies what the store recorded of what each machine was sent onto the keys of the assignments, as soon -// as this controller holds the lease (the chain refuses a write before), and tries again every half minute while it fails, -// each time it fails saying so as an urgent condition until it has worked. It is idempotent: CopySentBuilds overwrites no key -// that names a commit and empties each column it has copied. -func copySentBuildsSoon(ctx context.Context, inv *inventory.Inventory) { - failed := false - for { - _, err := inv.CopySentBuilds(ctx) - o := conditions.Observation{Scope: conditions.ScopeBuild, ID: kindSentBuildsNotCopied, Token: kindSentBuildsNotCopied, - Kind: kindSentBuildsNotCopied, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, - Source: "chain", Summary: "what each machine was sent is not copied onto its assignments: "} - if err != nil { - fmt.Printf("chain: what the machines were sent is not copied onto the assignments: %v\n", err) - if keeper := conditionsFrom; keeper != nil && !failed { - o.Summary += err.Error() - if _, oerr := keeper.Observe(ctx, o); oerr == nil { - failed = true - } - } - } else { - if keeper := conditionsFrom; keeper != nil && failed { - _, _ = keeper.Clear(ctx, o.Key(), "copied") - } - return - } - select { - case <-ctx.Done(): - return - case <-time.After(chainMoveRetry): - } - } -} - // chainMoveRetry is how long the copy waits before it tries again. const chainMoveRetry = 30 * time.Second diff --git a/cmd/mesh-controller/chain_links.go b/cmd/mesh-controller/chain_links.go deleted file mode 100644 index b6d01adb..00000000 --- a/cmd/mesh-controller/chain_links.go +++ /dev/null @@ -1,121 +0,0 @@ -package main - -import ( - "context" - "fmt" - "slices" - "time" - - "github.com/novox/mesh-controller/internal/chain" - "github.com/novox/mesh-controller/internal/inventory" -) - -// The links a rollout makes (novox/hq ADR 0408 §1, design 47 acceptance 10, issue 1095): the send of a plan's tier puts a -// build on a machine; the assignment of the module there names the build, its commit and the rollout (written by the record of -// the send, inventory.RecordSent, from the context a send is made in), and the rollout lists the assignments it changed with the -// build before and after. - -// rolloutModules is what a plan sends of each module: the commit and the build of it. -func rolloutModules(p *inventory.Plan) map[string]inventory.RolloutModule { - out := make(map[string]inventory.RolloutModule, len(p.Modules)) - for m, s := range p.Modules { - if s != nil && s.Commit != "" { - out[m] = inventory.RolloutModule{Commit: s.Commit, Build: s.Build} - } - } - return out -} - -// sendingAs is the context a plan's send is made in: what is recorded of the send names this rollout. -func sendingAs(ctx context.Context, p *inventory.Plan) context.Context { - return inventory.WithRollout(ctx, p.ID, rolloutModules(p)) -} - -// ranBefore is what each module on a machine ran before a send, as the chain names it: its assignment's build and commit, -// else the commit the machine was last sent for it. Read before the send, which writes the new ones. -func ranBefore(ctx context.Context, inv *inventory.Inventory, node string) map[string]chain.Ref { - was := map[string]chain.Ref{} - if sent, known, err := inv.SentBuilds(ctx, node); err == nil && known { - for m, commit := range sent { - was[m] = chain.Ref{Commit: commit} - } - } - if ran, err := inv.AssignmentRuns(ctx, node); err == nil { - for m, r := range ran { - was[m] = r - } - } - return was -} - -// noteChanges writes into a plan that a send to a machine changed the assignments of the modules it carried: the build and -// commit before and after, the builds the rollout carried, and the step taken. A change noted already for the machine and the -// module is replaced, so a send made again says it once. -func noteChanges(p *inventory.Plan, node string, modules []string, was map[string]chain.Ref, at time.Time) { - for _, m := range modules { - s := p.Modules[m] - if s == nil { - continue - } - change := chain.Change{Node: node, Module: m, Before: was[m], After: chain.Ref{Build: s.Build, Commit: s.Commit}} - if at := slices.IndexFunc(p.Changed, func(c chain.Change) bool { return c.Node == node && c.Module == m }); at >= 0 { - change.Before = p.Changed[at].Before - p.Changed[at] = change - } else { - p.Changed = append(p.Changed, change) - } - if s.Build != "" && !slices.Contains(p.Builds, s.Build) { - p.Builds = append(p.Builds, s.Build) - } - } - if !slices.ContainsFunc(p.Steps, func(s chain.Step) bool { return s.Tier == p.Tier && s.Node == node }) { - sent := at.UTC() - p.Steps = append(p.Steps, chain.Step{Tier: p.Tier, Node: node, Sent: &sent}) - } -} - -// refSaid is a build and the commit it was made from as a person reads them; "nothing" for a module that ran none. -func refSaid(r chain.Ref) string { - switch { - case r.Build == "" && r.Commit == "": - return "nothing" - case r.Build == "": - return "commit " + short(r.Commit) - } - return r.Build + " (" + short(r.Commit) + ")" -} - -// changedLines say the assignments a rollout changed: the machine, the module, and the build before and after. -func changedLines(p inventory.Plan) []string { - if len(p.Changed) == 0 { - return nil - } - out := []string{" changed:"} - for _, c := range p.Changed { - out = append(out, fmt.Sprintf(" %s/%s %s → %s", c.Node, c.Module, refSaid(c.Before), refSaid(c.After))) - } - return out -} - -// runLines say what each module assigned to a machine runs there, as its assignment names it: the build, the commit, the -// rollout that put it there and since when. -func runLines(assigned []chain.Assignment) []string { - var out []string - for _, a := range assigned { - if a.Build == "" && a.Commit == "" { - continue - } - line := fmt.Sprintf(" %-22s %s", a.Module, refSaid(chain.Ref{Build: a.Build, Commit: a.Commit})) - if a.Rollout != "" { - line += ", sent by " + a.Rollout - } - if !a.Since.IsZero() { - line += " at " + a.Since.Local().Format("2006-01-02 15:04") - } - out = append(out, line) - } - if len(out) == 0 { - return nil - } - return append([]string{" what they run, as the chain names it:"}, out...) -} diff --git a/cmd/mesh-controller/chain_links_test.go b/cmd/mesh-controller/chain_links_test.go deleted file mode 100644 index 8652bd60..00000000 --- a/cmd/mesh-controller/chain_links_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package main - -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/chain" - "github.com/novox/mesh-controller/internal/inventory" -) - -// **A rollout lists the assignments it changed, each with the build before and after** (ADR 0408 §1, issue 1095): one -// change for a machine and a module however often the send is noted, the build before kept from the first, the builds the -// rollout carried once each, and one step for a machine and a tier. -func TestARolloutListsTheAssignmentsItChangedWithTheBuildBeforeAndAfter(t *testing.T) { - at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC) - p := inventory.Plan{ID: "plan-1", Tier: 1, Modules: map[string]*inventory.PlanModule{ - "notes": {Commit: "c2", Build: "b2"}, "wiki": {Commit: "c2w", Build: "b2w"}, "nothing": nil}} - was := map[string]chain.Ref{"notes": {Build: "b1", Commit: "c1"}, "wiki": {Commit: "c1w"}} - - noteChanges(&p, "anchor", []string{"notes", "wiki", "nothing"}, was, at) - noteChanges(&p, "laptop", []string{"notes"}, nil, at) - // The send noted again, after what was sent had become what the machine runs: the first before stands. - noteChanges(&p, "anchor", []string{"notes"}, map[string]chain.Ref{"notes": {Build: "b2", Commit: "c2"}}, at.Add(time.Minute)) - - want := []chain.Change{ - {Node: "anchor", Module: "notes", Before: chain.Ref{Build: "b1", Commit: "c1"}, After: chain.Ref{Build: "b2", Commit: "c2"}}, - {Node: "anchor", Module: "wiki", Before: chain.Ref{Commit: "c1w"}, After: chain.Ref{Build: "b2w", Commit: "c2w"}}, - {Node: "laptop", Module: "notes", After: chain.Ref{Build: "b2", Commit: "c2"}}, - } - if len(p.Changed) != len(want) { - t.Fatalf("changed %+v", p.Changed) - } - for k := range want { - if p.Changed[k] != want[k] { - t.Errorf("change %d is %+v, want %+v", k, p.Changed[k], want[k]) - } - } - if len(p.Builds) != 2 || p.Builds[0] != "b2" || p.Builds[1] != "b2w" { - t.Errorf("builds %v", p.Builds) - } - if len(p.Steps) != 2 || p.Steps[0].Node != "anchor" || p.Steps[0].Tier != 1 || p.Steps[0].Sent == nil || p.Steps[1].Node != "laptop" { - t.Errorf("steps %+v", p.Steps) - } - lines := changedLines(p) - if len(lines) != 4 || !strings.Contains(lines[1], "anchor/notes b1 (c1) → b2 (c2)") || - !strings.Contains(lines[3], "laptop/notes nothing → b2 (c2)") { - t.Errorf("changed lines %q", lines) - } - if got := changedLines(inventory.Plan{}); got != nil { - t.Errorf("a plan that changed nothing says %q", got) - } -} - -func TestWhatAMachineRunsIsSaidAsItsAssignmentNamesIt(t *testing.T) { - at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC) - lines := runLines([]chain.Assignment{ - {Node: "anchor", Module: "notes", Build: "b2", Commit: "c2c2c2c2c2c2", Rollout: "plan-1", Since: at}, - {Node: "anchor", Module: "web", Commit: "c3"}, - {Node: "anchor", Module: "never-sent"}, - }) - if len(lines) != 3 || !strings.Contains(lines[1], "notes") || !strings.Contains(lines[1], "b2 (c2c2c2c2)") || - !strings.Contains(lines[1], "sent by plan-1") || !strings.Contains(lines[2], "web") || !strings.Contains(lines[2], "commit c3") { - t.Fatalf("lines %q", lines) - } - if runLines([]chain.Assignment{{Node: "a", Module: "m"}}) != nil { - t.Fatal("a module never sent is said to run something") - } -} diff --git a/cmd/mesh-controller/compose_input.go b/cmd/mesh-controller/compose_input.go index 217ad7ae..dd5b22ae 100644 --- a/cmd/mesh-controller/compose_input.go +++ b/cmd/mesh-controller/compose_input.go @@ -248,25 +248,11 @@ func readNode(ctx context.Context, open *stores, n inventory.Node, place invento if out.Epoch, err = inv.SentEpoch(ctx, n.ID); err != nil { return out, err } - if out.Generation, err = inv.SentGeneration(ctx, n.ID); err != nil { - return out, err - } - if out.ReadsModules, err = inv.ReadsModules(ctx, n.ID); err != nil { - return out, err - } + // Every node-engine of release 6 reads a module per resource and a build per module. + out.ReadsModules = true if out.SentBuilds, out.SentKnown, err = inv.SentBuilds(ctx, n.Name); err != nil { return out, err } - runs, err := inv.AssignmentRuns(ctx, n.Name) - if err != nil { - return out, err - } - if len(runs) > 0 { - out.Runs = make(map[string]compose.Run, len(runs)) - for module, ref := range runs { - out.Runs[module] = compose.Run{Commit: ref.Commit, Build: ref.Build} - } - } stated, has, err := inv.HealthOf(ctx, n.Name) if err != nil { return out, err diff --git a/cmd/mesh-controller/declaration_files_test.go b/cmd/mesh-controller/declaration_files_test.go index c428d418..3577c6db 100644 --- a/cmd/mesh-controller/declaration_files_test.go +++ b/cmd/mesh-controller/declaration_files_test.go @@ -243,54 +243,6 @@ func TestObjectsAreInTheBucketBeforeTheDeclarationIsSent(t *testing.T) { // 1696 } } -// fallbackDelivery refuses the first declaration it is sent with the bucket's failure, as a bus that has not yet been given the -// controller's grant does, and takes the second. -type fallbackDelivery struct { - bodies [][]byte - first error -} - -func (f *fallbackDelivery) grant(context.Context, []readyNode) error { return nil } -func (f *fallbackDelivery) declare(_ context.Context, _ readyNode, body []byte) (string, error) { - f.bodies = append(f.bodies, body) - if len(f.bodies) == 1 && f.first != nil { - return "", fmt.Errorf("x: %w: %w", errFilesNotPut, f.first) - } - return "digest", nil -} - -// 1696/AC5: when the bucket cannot be written yet (the grant travels in the declaration of the machine holding the bus), that send goes -// with every content inline and is made; a send that fails for another reason is not retried inline. -func TestASendWhoseFilesCannotBePutGoesInlineOnce(t *testing.T) { // 1696/AC5 - users := contentOf("users", 100<<10) - s := readyNode{node: "novox", declared: sendable{fileReferences: true, Resources: []map[string]any{aFileResource("nats.bus-users", users)}}} - f := &fallbackDelivery{first: errors.New("timeout")} - digests, err := deliver(t.Context(), f, "novox", []readyNode{s}) - if err != nil || digests["novox"] != "digest" || len(f.bodies) != 2 { - t.Fatalf("%v %v %d", digests, err, len(f.bodies)) - } - if strings.Contains(string(f.bodies[0]), users[:100]) || !strings.Contains(string(f.bodies[1]), users[:100]) { - t.Error("the first body should name the file by digest and the second carry it inline") - } - other := &fallbackDelivery{} - other.first = nil - failing := &failingDelivery{err: errors.New("the bus is down")} - if _, err := deliver(t.Context(), failing, "novox", []readyNode{s}); err == nil || failing.calls != 1 { - t.Fatalf("a failure of another kind: %v after %d send(s)", err, failing.calls) - } -} - -type failingDelivery struct { - err error - calls int -} - -func (f *failingDelivery) grant(context.Context, []readyNode) error { return nil } -func (f *failingDelivery) declare(context.Context, readyNode, []byte) (string, error) { - f.calls++ - return "", f.err -} - // A bucket for the sweep. type sweepBucketFake struct { objects map[string]time.Time // zero: named at the last look @@ -502,33 +454,3 @@ func whileHeld(node string) bool { return true } } - -// 1696/AC9: a send holds the node's lock from the put to the publication, so the sweep never removes an object between them. The -// test goes through overTheBus.declare: while another holds the lock the declaration is not published, and at the publication the -// lock is held. -func TestASendHoldsTheNodesLockFromThePutToThePublication(t *testing.T) { // 1696/AC9 - const node = "lock-send-node" - bus := &aBusThatSays{published: make(chan string, 1)} - bus.heldAtPublish = func() bool { return whileHeld(node) } - send := overTheBus{server: link.SendingOn(bus), signer: aSignerOfZeros{}} - release := lockFilesOf(node) // a sweep is judging this node's bucket - done := make(chan error, 1) - go func() { - _, err := send.declare(t.Context(), readyNode{node: node}, []byte(`{}`)) - done <- err - }() - select { - case n := <-bus.published: - t.Fatalf("%s was sent its declaration while the sweep held the node's bucket", n) - case <-time.After(150 * time.Millisecond): - } - release() - select { - case <-bus.published: - case <-time.After(5 * time.Second): - t.Fatal("the send did not go on once the sweep let go") - } - if err := <-done; err == nil || !strings.Contains(err.Error(), "nothing is recorded") { - t.Fatalf("the send did not reach the publication (a lock held at it): %v", err) - } -} diff --git a/cmd/mesh-controller/durations.go b/cmd/mesh-controller/durations.go deleted file mode 100644 index b03db5ce..00000000 --- a/cmd/mesh-controller/durations.go +++ /dev/null @@ -1,148 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "os" - "slices" - "sort" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// What the core's bounds are set from (novox/hq to-be 45 Phase 0). -// -// **A bound is set from what was measured, not from what seemed reasonable.** Phase 1 puts a watchdog -// on each row of the signals table, and each has a bound: S1 three heartbeat intervals, S2 three times -// a machine's last apply, S3 a tier's build and apply time, S6 a build's timeout. Marked provisional -// in the design until a fortnight of these says what the mesh actually takes. Recorded by the serving -// controller as it hears each — a send's first report, a machine's next word, a plan leaving a tier, a -// build's outcome — and summarised here per machine, repository or module. - -// durationSummary is one subject's measurements of one kind. -type durationSummary struct { - Kind string `json:"kind"` - Subject string `json:"subject"` - Count int `json:"count"` - Median string `json:"median"` - P90 string `json:"p90"` - Max string `json:"max"` - // Bound is what to-be 45's rule would make of these, where the rule is a multiple of a measured - // time: three times the slowest apply (S2), three times the median word interval (S1). - Suggests string `json:"suggests,omitempty"` -} - -func summarise(ds []inventory.Duration) []durationSummary { - type key struct{ kind, subject string } - by := map[key][]time.Duration{} - for _, d := range ds { - k := key{d.Kind, d.Subject} - by[k] = append(by[k], d.Took) - } - var out []durationSummary - for k, took := range by { - slices.Sort(took) - at := func(q float64) time.Duration { return took[int(q*float64(len(took)-1))] } - s := durationSummary{Kind: k.kind, Subject: k.subject, Count: len(took), - Median: round(at(0.5)), P90: round(at(0.9)), Max: round(took[len(took)-1])} - switch k.kind { - case inventory.DurationApply: - s.Suggests = "S2 bound max(2m, 3×last apply) ≈ " + round(max(2*time.Minute, 3*at(0.9))) + " at the p90" - } - out = append(out, s) - } - sort.Slice(out, func(i, j int) bool { - ki, kj := slices.Index(inventory.DurationKinds, out[i].Kind), slices.Index(inventory.DurationKinds, out[j].Kind) - if ki != kj { - return ki < kj - } - return out[i].Subject < out[j].Subject - }) - return out -} - -func round(d time.Duration) string { - switch { - case d < time.Second: - return d.Round(time.Millisecond).String() - case d < time.Minute: - return d.Round(100 * time.Millisecond).String() - default: - return d.Round(time.Second).String() - } -} - -// durationsCommand is `durations`: the summary per kind and subject, or every measurement as data. -func durationsCommand(ctx context.Context, args []string) error { - set := flag.NewFlagSet("durations", flag.ContinueOnError) - kind := set.String("kind", "", "one kind: "+strings.Join(inventory.DurationKinds, ", ")) - days := set.Int("days", 14, "how many days back") - asJSON := set.Bool("json", false, "the summary as data") - all := set.Bool("all", false, "every measurement rather than the summary") - if _, err := parseAround(set, args); err != nil { - return err - } - if *kind != "" && !slices.Contains(inventory.DurationKinds, *kind) { - return fmt.Errorf("%q is not a kind of duration: %s", *kind, strings.Join(inventory.DurationKinds, ", ")) - } - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - ds, err := open.inventory.Durations(ctx, *kind, time.Now().Add(-time.Duration(*days)*24*time.Hour)) - if err != nil { - return err - } - if *all { - body, err := json.MarshalIndent(ds, "", " ") - if err != nil { - return err - } - fmt.Println(string(body)) - return nil - } - summary := summarise(ds) - if *asJSON { - body, err := json.MarshalIndent(map[string]any{"days": *days, "durations": summary}, "", " ") - if err != nil { - return err - } - fmt.Println(string(body)) - return nil - } - if len(summary) == 0 { - fmt.Printf("nothing measured in the last %d day(s): the serving controller records apply, "+ - "plan-tier and build durations as it hears them\n", *days) - return nil - } - fmt.Printf("durations over the last %d day(s) — what the core's bounds are set from (to-be 45 Phase 0)\n\n", *days) - fmt.Printf(" %-14s %-28s %6s %10s %10s %10s\n", "kind", "of", "count", "median", "p90", "max") - for _, s := range summary { - fmt.Printf(" %-14s %-28s %6d %10s %10s %10s\n", s.Kind, s.Subject, s.Count, s.Median, s.P90, s.Max) - if s.Suggests != "" { - fmt.Printf(" %-14s %-28s %s\n", "", "", s.Suggests) - } - } - return nil -} - -// forgettingOldDurations removes what is older than a month, at start and daily after. -func forgettingOldDurations(ctx context.Context, inv *inventory.Inventory) { - for { - if n, err := inv.ForgetOldDurations(ctx); err != nil { - fmt.Fprintf(os.Stderr, "durations older than %s could not be removed: %v\n", inventory.DurationsKeptFor, err) - } else if n > 0 { - fmt.Printf("removed %d duration(s) older than %s\n", n, inventory.DurationsKeptFor) - } - select { - case <-ctx.Done(): - return - case <-time.After(24 * time.Hour): - } - } -} diff --git a/cmd/mesh-controller/generation.go b/cmd/mesh-controller/generation.go deleted file mode 100644 index 66bca026..00000000 --- a/cmd/mesh-controller/generation.go +++ /dev/null @@ -1,266 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "io" - "os" - "strings" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// The assignment generation a declaration was composed from, and the record of every send (novox/hq -// issue 234). -// -// On 2026-10-04 a declaration newer in sequence than every other named four fewer modules than the -// assignments held, and a machine applied it and undeclared all four. The sequence orders arrival and -// cannot tell a later send that carries an older view of the assignments. So a declaration now carries -// the generation of the assignments it was composed from — a counter the store raises in the same -// transaction as every assignment change — and a machine refuses one older than it applied, unless it is -// a gate's put-back. And every send is written down with who sent it, from which generation and naming -// which modules: the controller logged no send then, and which process sent the stale declaration could -// not be read back from anything the mesh kept. - -// kindOlderGeneration is S20's kind: a machine refused a send for the generation it was composed from. -const kindOlderGeneration = "older-generation" - -// generationRefusalsSaid is how long a refused send is said after its refusal: an hour, as an advisory is. -const generationRefusalsSaid = advisoryQuiet - -// stamp gives a composed declaration the order allotted to it before it was composed: its sequence, and -// the epoch and generation when the machine reads them — and keeps the generation and acting epoch for the -// record of the send whether or not the machine is sent them. -// -// **No put-back mark is sent.** A gate puts a machine back by composing it again (sendRollout), so its -// declaration is allotted here like any other and carries the generation as it stands — never older than -// what the machine applied. The node-engine reads a `put_back` key (mesh-host#82); this controller never -// needs to send one. -func (o order) stamp(d *sendable) { - d.Sequence, d.Epoch = o.sequence, o.epoch - d.composedFrom, d.actingEpoch = o.generation, o.acting - d.Generation = 0 - if o.readsGeneration && o.generation > 0 { - d.Generation = o.generation - } -} - -// sentRecord is what the record of a send keeps beside its digest. -type sentRecord struct { - sequence int64 - epoch uint64 - generation int64 - // toldGeneration is the generation on the wire, which the would-send is stamped with: zero for a machine - // whose node-engine has not said it reads one. - toldGeneration int64 - sender string - modules []string - // plans is each module the send handed to the provisioner, its plan, as JSON (design 57 P6); nil for none. - plans []byte -} - -// sentRecordOf is a composed send's record: its sender is the caller this process acts for. -func sentRecordOf(ctx context.Context, d sendable) sentRecord { - r := sentRecord{sequence: d.Sequence, epoch: d.actingEpoch, generation: d.composedFrom, - toldGeneration: d.Generation, sender: senderOf(callerOf(ctx)), modules: d.modules} - if len(d.plans) > 0 { - kept := map[string]keptPlan{} - for module, p := range d.plans { - kept[module] = keptPlan{Plan: p, RequiredBy: d.requiredBy[module]} - } - // A map of plain strings and slices: it marshals. - r.plans, _ = json.Marshal(kept) - } - return r -} - -// keptPlan is a handed module's plan as a send keeps it (migration 0100): the plan, and for a module only pulled in -// by another, the module that requires it (issue 765, inventory.SentProvisioningPlans). -type keptPlan struct { - catalogue.Plan - RequiredBy string `json:"required-by,omitempty"` -} - -// callerOf is who asked for what this process does: the seat call's caller when ctx belongs to one, the -// caller the controller ran this command for, or the account at the shell. -func callerOf(ctx context.Context) string { - if c := link.CallerIn(ctx); c != "" { - return c - } - return link.Caller() -} - -// senderOf is a send's sender in words: who asked, and the process and build that composed it — the -// answer issue 234 could not find, because two controllers and a one-shot push were all sending then. -func senderOf(caller string) string { - host, _ := os.Hostname() - return fmt.Sprintf("%s (pid %d on %s, build %s)", caller, os.Getpid(), host, version) -} - -// heardGenerationRefusal keeps a machine's refusal of a send for its generation, so S20 names its sender -// and says it (novox/hq issue 234): on the send it refused or — when the mesh has no record of sending that -// sequence — as a refusal of its own, naming the sender as unknown. Either way S20 is raised: a refusal the -// mesh cannot attribute is the louder fact, not a quieter one. -// -// **And raises the mesh's counter past what the machine applied, when the refused send was composed from -// the counter as it stands** (counterBehind): then the sender's view was not stale — the counter is behind -// the machine, which is a store put back from a backup — and every send after would be refused for ever. A -// stale sender's generation is below the counter, and the counter is left alone for it. -// -// Nothing is sent from here. This runs in the controller's receive loop, and a push from it would hold that -// loop, and the machine's hold, for as long as the push takes — the deaf controller of issues 184 and 185. -// S20 names `push ` for that case instead. -func heardGenerationRefusal(ctx context.Context, inv *inventory.Inventory, report link.Report) { - r := report.OlderGeneration - if r == nil || inv == nil || report.Node == "" { - return - } - node, err := inv.NodeByName(ctx, report.Node) - if err != nil { - fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the machine cannot be "+ - "read, so it is not raised: %v\n", report.Node, err) - return - } - var raised int64 - if now, err := inv.AssignmentGeneration(ctx); err != nil { - fmt.Fprintf(os.Stderr, "mesh-controller: %s refused a send for its generation, and the mesh's own cannot be "+ - "read: %v\n", report.Node, err) - } else if counterBehind(*r, now) { - if raised, err = inv.RaiseAssignmentGeneration(ctx, r.Applied); err != nil { - fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation (%d) is behind what %s applied (%d), "+ - "and could not be raised: %v\n", now, report.Node, r.Applied, err) - raised = 0 - } else { - fmt.Fprintf(os.Stderr, "mesh-controller: the assignment generation was %d, behind what %s applied (%d) — "+ - "a store put back from a backup — and is raised to %d; `push %s` sends it what the mesh holds now\n", - now, report.Node, r.Applied, raised, report.Node) - } - } - send, found, err := inv.RefusedSend(ctx, node.ID, report.Sequence, r.Applied, raised) - if err == nil && !found { - send, err = inv.RecordUnrecordedRefusal(ctx, inventory.Send{Node: node.ID, Sequence: report.Sequence, - Epoch: report.Epoch, Generation: r.Generation, Digest: report.Declared, RefusedApplied: r.Applied, - CounterRaisedTo: raised}) - } - if err != nil { - fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d for its generation, and the refusal could not be "+ - "kept, so it is not raised: %v\n", report.Node, report.Sequence, err) - return - } - fmt.Fprintf(os.Stderr, "mesh-controller: %s refused send %d from %s: composed from assignment generation %d, "+ - "and it applied %d\n", report.Node, report.Sequence, send.Sender, r.Generation, r.Applied) -} - -// counterBehind says a refusal shows the mesh's counter behind the machine rather than a stale sender: the -// refused send carried the counter as it stands now, and the machine applied more than that. -func counterBehind(r link.GenerationRefusal, now int64) bool { - return r.Generation >= now && r.Applied > r.Generation -} - -// modulesWords is a send's modules as a sentence says them. -func modulesWords(modules []string) string { - if len(modules) == 0 { - return "no module the mesh recorded" - } - return strings.Join(modules, ", ") -} - -// writeLastSend says what a machine was last told, by whom and from which generation (novox/hq issue 234), and -// what it answered (issue 485): nothing when the mesh has not recorded a send to it. -func writeLastSend(ctx context.Context, w io.Writer, inv *inventory.Inventory, node string) error { - s, found, err := inv.LastSend(ctx, node) - if err != nil || !found { - return err - } - fmt.Fprintf(w, "%s:\n", node) - for _, line := range lastSendLines(s) { - fmt.Fprintln(w, line) - } - return nil -} - -// lastSendLines is a machine's last send in full, as `node show` and `plan` say it (novox/hq issue 485): when, -// by whom, under which lease epoch, from which assignment generation, naming which modules, and what the -// machine answered of it — the refusal in its own words. -func lastSendLines(s inventory.Send) []string { - const stamp = "2006-01-02 15:04:05" - epoch := "under no lease epoch" - if s.Epoch > 0 { - epoch = fmt.Sprintf("under lease epoch %d", s.Epoch) - } - generation := "no assignment generation recorded" - if s.Generation > 0 { - generation = fmt.Sprintf("assignment generation %d", s.Generation) - } - lines := []string{ - fmt.Sprintf(" last sent sequence %d at %s", s.Sequence, s.SentAt.Local().Format(stamp)), - fmt.Sprintf(" by %s", s.Sender), - fmt.Sprintf(" %s, composed from %s", epoch, generation), - fmt.Sprintf(" naming %s", modulesWords(s.Modules)), - } - answer := func(words string) { lines = append(lines, " its answer "+words) } - switch a := s.Answer; { - case s.RefusedAt != nil: - // Before the report: a refusal for the generation is kept on the send itself and is the machine's - // answer to this send in particular. - answer(fmt.Sprintf("refused it at %s: it had applied assignment generation %d, newer than this one", - s.RefusedAt.Local().Format(stamp), s.RefusedApplied)) - case a.Outcome == "" || a.At == nil: - answer("the machine has not reported on it yet") - case a.Outcome == inventory.OutcomeApplied: - answer("applied it at " + a.At.Local().Format(stamp)) - case a.Outcome == inventory.OutcomeRefused: - answer("refused it at " + a.At.Local().Format(stamp) + ", in its words:") - for _, line := range strings.Split(strings.TrimRight(a.Refused, "\n"), "\n") { - lines = append(lines, " "+strings.TrimSpace(line)) - } - default: - answer(fmt.Sprintf("%s at %s: %d resource(s) failed — `status` says which", a.Outcome, - a.At.Local().Format(stamp), a.Failed)) - } - return lines -} - -// sendAnswerWord is what a machine answered of a send, in a word or few, as status says it in one line. -func sendAnswerWord(s inventory.Send) string { - switch a := s.Answer; { - case s.RefusedAt != nil: - return fmt.Sprintf("refused: it had applied generation %d", s.RefusedApplied) - case a.Outcome == "" || a.At == nil: - return "not reported on yet" - case a.Outcome == inventory.OutcomeFailed: - return fmt.Sprintf("failed (%d resource(s))", a.Failed) - default: - return a.Outcome - } -} - -// printLastSends says, one line per machine, when it was last sent a declaration, by whom, from which assignment -// generation and what it answered (novox/hq issue 485). `node show ` says the send in full. -func printLastSends(nodes []inventory.Node, sends map[string]inventory.Send, unread string) { - if unread != "" { - fmt.Printf("what each machine was last sent could NOT be read: %s\n\n", unread) - return - } - if len(nodes) == 0 { - return - } - fmt.Println("last sent, per machine:") - for _, n := range nodes { - s, found := sends[n.Name] - if !found { - fmt.Printf(" %-12s no send recorded\n", n.Name) - continue - } - generation := "no generation" - if s.Generation > 0 { - generation = fmt.Sprintf("generation %d", s.Generation) - } - fmt.Printf(" %-12s %s by %s, %s — %s\n", n.Name, s.SentAt.Local().Format("2006-01-02 15:04"), s.Sender, - generation, sendAnswerWord(s)) - } - fmt.Printf("\n `node show ` says a machine's last send in full\n\n") -} diff --git a/cmd/mesh-controller/grants_integration_test.go b/cmd/mesh-controller/grants_integration_test.go deleted file mode 100644 index 3180a764..00000000 --- a/cmd/mesh-controller/grants_integration_test.go +++ /dev/null @@ -1,91 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" -) - -// 1394/AC1 of C9 (ADR 0528 clause 2): when the controller has issued mesh-x's credential for fingerprint F (mesh-x -// registered at a build of F, its account minted, the bus's machine sent its declaration with the user list, through -// overTheBus.declare as a push sends it), then grants.mesh-x.F exists on the bus, -// and a second pass writes it again as nothing new. -func TestTheGrantOfAnIssuedCredentialIsOnTheBus(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - fp := grantFP('d') - m := catalogue.Manifest{Module: "mesh-x", Version: "1", OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/var/lib/mesh-x/broker"}}} - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-x1", Repository: "novox/mesh-x", Module: "mesh-x", Commit: "c0ffee1", - SourceFingerprint: fp}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-x", BuiltFrom: "c0ffee1", Head: "c0ffee1"}); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "anchor", "mesh-x"); err != nil { - t.Fatal(err) - } - user := broker.Principal{Kind: broker.KindModule, Node: "anchor", Module: "mesh-x"}.Username() - if _, err := inv.MintBusPassword(ctx, inventory.BusUser{Username: user, Kind: inventory.BusModule, Node: "anchor", Module: "mesh-x"}); err != nil { - t.Fatal(err) - } - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - js, err := jetstream.New(conn) - if err != nil { - t.Fatal(err) - } - if err := broker.OnConn(conn).EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - // The live path: a send of the bus's machine's declaration carrying the user list (overTheBus.declare), which records - // the list sent and then writes the grants it issued. - legacy, err := conn.JetStream() - if err != nil { - t.Fatal(err) - } - _ = js.DeleteStream(ctx, "GRANTS_DECLARED") - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "GRANTS_DECLARED", Subjects: []string{link.DeclareSubject("anchor")}}); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = js.DeleteStream(context.Background(), "GRANTS_DECLARED") }) - send := overTheBus{open: open, server: link.SendingOn(link.OverNATS{Conn: conn, JS: legacy}), signer: aSignerOfZeros{}} - ready := readyNode{node: "anchor", declared: sendable{BusUsers: "the user list the bus's machine is sent"}} - if _, err := send.declare(ctx, ready, []byte(`{}`)); err != nil { - t.Fatalf("the send of the bus's machine: %v", err) - } - kv, err := js.KeyValue(ctx, broker.GrantsBucket) - if err != nil { - t.Fatal(err) - } - key, _ := sdkdelivery.ControllerGrantKey("mesh-x", fp) - e, err := kv.Get(ctx, key) - if err != nil { - t.Fatalf("no grant %s after the list was issued: %v", key, err) - } - g, err := sdkdelivery.DecodeControllerGrant(e.Value()) - if err != nil || len(g.Accounts) != 1 || g.Accounts[0] != user { - t.Fatalf("the grant reads %+v, %v; want the account %s", g, err, user) - } - if _, err := send.declare(ctx, ready, []byte(`{}`)); err != nil { - t.Fatalf("the second send: %v", err) - } - if again, _ := kv.Get(ctx, key); again.Revision() != e.Revision() { - t.Errorf("a second pass wrote the grant again (revision %d, was %d)", again.Revision(), e.Revision()) - } -} diff --git a/cmd/mesh-controller/handacts_test.go b/cmd/mesh-controller/handacts_test.go index 27e53b3d..a4055ffe 100644 --- a/cmd/mesh-controller/handacts_test.go +++ b/cmd/mesh-controller/handacts_test.go @@ -4,9 +4,6 @@ import ( "context" "strings" "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" ) // With a reason, the seat passes it to the command, and a verb that only reads is not held to one. @@ -27,20 +24,6 @@ func TestARepairByHandCarriesItsReason(t *testing.T) { } } -// The summary of durations says, per kind and subject, what a bound would be set from. -func TestDurationsAreSummarisedPerSubject(t *testing.T) { - var ds []inventory.Duration - for i := 1; i <= 10; i++ { - ds = append(ds, inventory.Duration{Kind: inventory.DurationApply, Subject: "anchor", - Took: time.Duration(i) * time.Second}) - } - got := summarise(ds) - if len(got) != 1 || got[0].Kind != inventory.DurationApply || got[0].Count != 10 || - got[0].Max != "10s" || got[0].Median != "5s" || got[0].P90 != "9s" { - t.Fatalf("%+v", got) - } -} - // **A drill has its own verb** — `hand-act drill`, the seat's `drill` — and `hand-act record` refuses // the cause, so a repair cannot pass for a drill by the word it gives. func TestADrillIsRecordedThroughItsOwnVerb(t *testing.T) { diff --git a/cmd/mesh-controller/hold.go b/cmd/mesh-controller/hold.go deleted file mode 100644 index 6c7d4d8e..00000000 --- a/cmd/mesh-controller/hold.go +++ /dev/null @@ -1,37 +0,0 @@ -package main - -import ( - "context" -) - -// heldKey carries the nodes this call already holds, so an act that holds a node and then sends -// through sendTo does not wait on itself. -type heldKey struct{} - -// holdNodes holds the named nodes for composing and sending their declarations (novox/hq ADR -// 0100), skipping any the context already holds, and returns a context that says it holds them. -// Release gives back only what this call took. -func holdNodes(ctx context.Context, open *stores, names []string) (context.Context, func(), error) { - already, _ := ctx.Value(heldKey{}).(map[string]bool) - var take []string - for _, n := range names { - if !already[n] { - take = append(take, n) - } - } - if len(take) == 0 { - return ctx, func() {}, nil - } - release, err := open.inventory.HoldNodes(ctx, take) - if err != nil { - return ctx, nil, err - } - held := map[string]bool{} - for n := range already { - held[n] = true - } - for _, n := range take { - held[n] = true - } - return context.WithValue(ctx, heldKey{}, held), release, nil -} diff --git a/cmd/mesh-controller/issue348_test.go b/cmd/mesh-controller/issue348_test.go deleted file mode 100644 index 1f8544c4..00000000 --- a/cmd/mesh-controller/issue348_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package main - -import ( - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq issue 348: on 2026-10-09 the control node's resolver stopped answering on its private address -// at 10:57:57 UTC; the machine's network condition was raised at 10:58:45. The node-engine and the -// controller were sent at 10:59:34. The new node-engine's first statement judged the names once — unknown, -// "one look failed; a second decides" — and that statement cleared the condition, though its last evidence -// still said "connection refused". The next look raised it again at 11:00:23, after the send, and both -// builds failed their gate at 11:10 with "raised since it was sent" and were put back, for a fault that -// began before they were sent. - -// TestReplay348 replays the statements of the outage: the condition raised before the send is not -// cleared by the restarted engine's first, undecided statement, and the gate does not count it against -// the builds sent after it began. - -// A release walks its modules without a record per module: D10 counts what its tier names as rolling, -// so the node-engine a release walks is not "behind, and no plan is rolling it out" on its first machine. -func TestAReleaseRollsOutWhatItsTierNames(t *testing.T) { - plans := []inventory.Plan{ - {ID: "release-1", State: inventory.PlanRolling, Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{}}, - {ID: "plan-2", State: inventory.PlanRolling, Modules: map[string]*inventory.PlanModule{"letta": {}}}, - } - got := rollingModules(plans) - if !got["mesh-host"] || !got["letta"] || len(got) != 2 { - t.Fatalf("rolling: %v", got) - } -} diff --git a/cmd/mesh-controller/issue_204_integration_test.go b/cmd/mesh-controller/issue_204_integration_test.go deleted file mode 100644 index 666519c8..00000000 --- a/cmd/mesh-controller/issue_204_integration_test.go +++ /dev/null @@ -1,33 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) { - inv := inventory.ForTest(t) - ctx, cancel := context.WithCancel(t.Context()) - if _, err := inv.AddNode(ctx, "anchor"); err != nil { - t.Fatal(err) - } - cancel() // the sender is going away: its context is cancelled between the send and the record - body := []byte(`{"declaration":1,"resources":[]}`) - digest, err := recordSent(ctx, inv, "anchor", body, nil, 0, sentRecord{sender: "a test"}) - if err != nil { - // NodeByName on the cancelled context may itself refuse; the record must still be possible - // through the detached context, so look the node up again on a live one. - t.Fatalf("recording a send after cancellation failed: %v", err) - } - outstanding, err := inv.Outstanding(t.Context(), "anchor") - if err != nil { - t.Fatal(err) - } - if outstanding != digest || digest != digestOf(body) { - t.Fatalf("the send was not recorded: outstanding %q, sent %q", outstanding, digest) - } -} diff --git a/cmd/mesh-controller/issue_204_test.go b/cmd/mesh-controller/issue_204_test.go deleted file mode 100644 index 950fd9cb..00000000 --- a/cmd/mesh-controller/issue_204_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package main - -import ( - "context" - "testing" -) - -// A declaration composed earlier is numbered lower than one composed later, whatever order the two -// are sent in (novox/hq issue 204). The number used to be taken at send time, after composing, so a -// declaration composed before an assignment changed and sent after a newer one carried the higher -// number — and the machine, which refuses a lower number, took the older content as the mesh's -// newest word. Taken before the composition reads anything, the order of numbers is the order of -// compositions, and the host's refusal does what it is for. -func TestADeclarationComposedEarlierIsNumberedLowerWhateverOrderItIsSent(t *testing.T) { - allot := numbered() - var composed []string - compose := func(stamp string) func(string) (sendable, error) { - return func(node string) (sendable, error) { - composed = append(composed, stamp) - return sendable{Resources: []map[string]any{{"id": node + "." + stamp}}}, nil - } - } - // Composed first — before an assignment changed — and sent last. - stale, _ := composeEach([]string{"anchor"}, allot, compose("before")) - // Composed after the change, sent first. - fresh, _ := composeEach([]string{"anchor"}, allot, compose("after")) - - if stale[0].declared.Sequence != 1 || fresh[0].declared.Sequence != 2 { - t.Fatalf("the numbers do not follow the compositions: before=%d after=%d", - stale[0].declared.Sequence, fresh[0].declared.Sequence) - } - // Sent in the other order, the numbers do not change — so the machine that has applied the - // fresh one (2) refuses the stale one (1) when it arrives late. - if !(stale[0].declared.Sequence < fresh[0].declared.Sequence) { - t.Fatal("a declaration composed earlier must carry the lower number, however late it is sent") - } - if len(composed) != 2 || composed[0] != "before" { - t.Fatalf("compositions happened in an unexpected order: %v", composed) - } -} - -// The number is taken before the first read of the composition, not after it: an allotter that -// fails leaves nothing composed for that machine, and the others are still composed. -func TestTheNumberIsTakenBeforeComposingAndItsFailureIsARefusal(t *testing.T) { - calls := 0 - allot := func(node string) (order, error) { - if node == "anchor" { - return order{}, context.DeadlineExceeded - } - return order{sequence: 7}, nil - } - sending, refusals := composeEach([]string{"anchor", "laptop"}, allot, func(node string) (sendable, error) { - calls++ - if node == "anchor" { - t.Fatal("anchor was composed although its number could not be taken") - } - return sendable{}, nil - }) - if calls != 1 || len(sending) != 1 || sending[0].node != "laptop" || sending[0].declared.Sequence != 7 { - t.Fatalf("laptop should be composed with its number and anchor refused: %v / %v", sending, refusals) - } - if len(refusals) != 1 { - t.Fatalf("anchor's failed number should be a refusal naming it: %v", refusals) - } -} - -// What was sent is written down even when the sender's context is already cancelled (issue 204): a -// controller replaced mid-send had told the machine and never recorded it, so status read "applied, -// current" over a machine that had just been sent something else. diff --git a/cmd/mesh-controller/last_send_integration_test.go b/cmd/mesh-controller/last_send_integration_test.go deleted file mode 100644 index 4cb33fb7..00000000 --- a/cmd/mesh-controller/last_send_integration_test.go +++ /dev/null @@ -1,51 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -func TestNodeShowAndStatusReadTheLastSendFromTheStore(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - record, err := open.inventory.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordSentWith(ctx, record.ID, "d12", nil, 57, 40, inventory.Send{Sequence: 12, - Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12", - Modules: []string{"docker", "sudo"}}); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeRefused, - Declared: "d12", Refused: "unknown field \"generation\""}); err != nil { - t.Fatal(err) - } - shown := printed(t, func() error { return showNode(ctx, open, "laptop") }) - for _, want := range []string{"a one-shot push by jochen at a shell on anchor", "lease epoch 57", "generation 40", - "docker, sudo", "unknown field \"generation\""} { - if !strings.Contains(shown, want) { - t.Errorf("node show laptop does not say %q:\n%s", want, shown) - } - } - body := printed(t, func() error { return statusFor(ctx, open, true) }) - var got struct { - LastSent []struct { - Node string `json:"node"` - Sender string `json:"sender"` - Answer string `json:"answer"` - } `json:"lastSent"` - } - if err := json.Unmarshal([]byte(body), &got); err != nil { - t.Fatalf("%v:\n%s", err, body) - } - if len(got.LastSent) != 1 || got.LastSent[0].Node != "laptop" || got.LastSent[0].Answer != "refused" || - got.LastSent[0].Sender != "a one-shot push by jochen at a shell on anchor" { - t.Fatalf("status --json's last sends read %+v", got.LastSent) - } -} diff --git a/cmd/mesh-controller/last_send_test.go b/cmd/mesh-controller/last_send_test.go deleted file mode 100644 index 33c329f7..00000000 --- a/cmd/mesh-controller/last_send_test.go +++ /dev/null @@ -1,121 +0,0 @@ -package main - -import ( - "encoding/json" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// novox/hq issue 485: the mesh records every send — sender, lease epoch, generation, modules — and only `plan` -// said it. `status` says per machine when it was last sent a declaration and by whom, in one line; `node show` -// says the last send in full and what the machine answered; and `status --json` carries it as fields, since -// the store's reader may not read the controller's tables and these verbs are the only window onto them. - -func aLastSend(answer inventory.SendAnswer) inventory.Send { - sent := time.Date(2026, 10, 11, 18, 40, 5, 0, time.Local) - return inventory.Send{NodeName: "laptop", Sequence: 12, Epoch: 57, - Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12", - Modules: []string{"docker", "sudo"}, SentAt: sent, Recorded: true, Answer: answer} -} - -func TestStatusSaysEachMachinesLastSendInOneLine(t *testing.T) { - at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local) - asked := answers{ - nodes: []inventory.Node{{Name: "anchor", LastSeen: time.Now()}, {Name: "laptop", LastSeen: time.Now()}}, - lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{ - Outcome: inventory.OutcomeApplied, At: &at})}, - } - shown := printed(t, func() error { return printStatus(asked) }) - var laptop, anchor []string - for _, line := range strings.Split(shown, "\n") { - fields := strings.Fields(line) - if len(fields) > 0 && fields[0] == "laptop" { - laptop = append(laptop, line) - } - if len(fields) > 0 && fields[0] == "anchor" { - anchor = append(anchor, line) - } - } - if len(laptop) != 1 || !strings.Contains(laptop[0], "2026-10-11 18:40") || - !strings.Contains(laptop[0], "by a one-shot push by jochen at a shell on anchor") || - !strings.Contains(laptop[0], "generation 40") || !strings.Contains(laptop[0], "applied") { - t.Fatalf("status does not say laptop's last send in one line:\n%s", shown) - } - if len(anchor) != 1 || !strings.Contains(anchor[0], "no send recorded") { - t.Fatalf("status does not say anchor has no send recorded, in one line:\n%s", shown) - } -} - -func TestStatusSaysTheLastSendsCouldNotBeRead(t *testing.T) { - asked := answers{nodes: []inventory.Node{{Name: "laptop", LastSeen: time.Now()}}, - lastSentUnread: "the store went away"} - if shown := printed(t, func() error { return printStatus(asked) }); !strings.Contains(shown, "the store went away") { - t.Fatalf("a record of sends that could not be read is not said:\n%s", shown) - } - if asked.well() { - t.Error("a mesh whose sends could not be read is called well") - } -} - -func TestNodeShowSaysTheLastSendInFull(t *testing.T) { - at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.Local) - refused := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused, - Refused: "unknown field \"generation\"\nsecond line", At: &at})), "\n") - for _, want := range []string{"sequence 12", "2026-10-11 18:40:05", "a one-shot push by jochen at a shell on anchor", - "lease epoch 57", "generation 40", "docker, sudo", "refused", "unknown field \"generation\""} { - if !strings.Contains(refused, want) { - t.Errorf("node show's last send does not say %q:\n%s", want, refused) - } - } - if applied := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeApplied, - At: &at})), "\n"); !strings.Contains(applied, "applied it at 2026-10-11 18:41:00") { - t.Errorf("an applied send is not said applied:\n%s", applied) - } - if waiting := strings.Join(lastSendLines(aLastSend(inventory.SendAnswer{})), "\n"); !strings.Contains(waiting, - "not reported on it yet") { - t.Errorf("a send not reported on is not said so:\n%s", waiting) - } - stale := aLastSend(inventory.SendAnswer{}) - when := at - stale.RefusedAt, stale.RefusedApplied = &when, 44 - if s := strings.Join(lastSendLines(stale), "\n"); !strings.Contains(s, "refused it at 2026-10-11 18:41:00") || - !strings.Contains(s, "generation 44") { - t.Errorf("a send refused for its generation is not said so:\n%s", s) - } -} - -func TestStatusJSONCarriesEachMachinesLastSend(t *testing.T) { - at := time.Date(2026, 10, 11, 18, 41, 0, 0, time.UTC) - asked := answers{nodes: []inventory.Node{{Name: "anchor"}, {Name: "laptop"}}, - lastSent: map[string]inventory.Send{"laptop": aLastSend(inventory.SendAnswer{Outcome: inventory.OutcomeRefused, - Refused: "unknown field", At: &at})}} - body, err := statusAsJSON(asked) - if err != nil { - t.Fatal(err) - } - var got struct { - LastSent []map[string]any `json:"lastSent"` - } - if err := json.Unmarshal(body, &got); err != nil { - t.Fatal(err) - } - if len(got.LastSent) != 1 { - t.Fatalf("status --json carries %d last sends, want laptop's only:\n%s", len(got.LastSent), body) - } - s := got.LastSent[0] - if s["node"] != "laptop" || s["sequence"] != float64(12) || s["epoch"] != float64(57) || - s["generation"] != float64(40) || s["sender"] != "a one-shot push by jochen at a shell on anchor" || - s["digest"] != "d12" || s["sentAt"] == nil || s["answer"] != "refused" || s["refused"] != "unknown field" || - s["answeredAt"] == nil { - t.Fatalf("laptop's last send reads %v", s) - } - if modules, _ := s["modules"].([]any); len(modules) != 2 || modules[0] != "docker" { - t.Fatalf("the modules it named read %v", s["modules"]) - } -} - -// And through the store: a send recorded and refused by the machine is what `node show` and `status --json` -// read back. diff --git a/cmd/mesh-controller/lease_integration_test.go b/cmd/mesh-controller/lease_integration_test.go index 4c7e980f..4ec880ec 100644 --- a/cmd/mesh-controller/lease_integration_test.go +++ b/cmd/mesh-controller/lease_integration_test.go @@ -135,11 +135,6 @@ func TestTwoControllersOneActs(t *testing.T) { if _, err := inv.AddNode(ctx, "anchor"); err != nil { t.Fatal(err) } - saved := inventory.Plan{ID: "plan-after-loss", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(), - State: inventory.PlanBuilding} - if err := inv.SavePlan(ctx, &saved); err == nil { - t.Fatal("B wrote a plan after losing the lease") - } store := conditions.NewInMemory() keeper := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, Epoch: func() (uint64, error) { return b.epoch(ctx) }}) diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index c7a70520..49af526b 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -86,7 +86,7 @@ func run() error { case "builds": return buildsCommand(ctx, args[1:]) case "forget-checks", "queue", "cancel", "clear", "kill", "replay", "pause", "resume", "plans", "plan", "push", "deliver", - "delivery-stop", "delivery-backups-again", "delivery-walks", "delivery-order", "delivery", "upgrade": + "delivery-stop", "delivery-backups-again", "delivery-walks", "delivery-order", "delivery": return removedByName(args[0]) case "try": return tryCommand(ctx, args[1:]) @@ -128,6 +128,8 @@ func run() error { case "facts": return factsCommand(ctx, args[1:]) // The bus as a planned step (novox/hq to-be 45 §8, ADR 0236). + case "upgrade": + return upgradeCommand(ctx, args[1:]) case "bus": return busCommand(ctx, args[1:]) case "overlay": @@ -163,7 +165,7 @@ func run() error { return handActCommand(ctx, append([]string{"list"}, args[1:]...)) // What the mesh's bounds will be set from (novox/hq to-be 45 Phase 0). case "durations": - return durationsCommand(ctx, args[1:]) + return removedByName(args[0]) // What is wrong, and the self-check (novox/hq to-be 45 §2, §4). case "conditions": return conditionsCommand(ctx, args[1:]) diff --git a/cmd/mesh-controller/moved.go b/cmd/mesh-controller/moved.go index e1282fc6..d9c57391 100644 --- a/cmd/mesh-controller/moved.go +++ b/cmd/mesh-controller/moved.go @@ -284,14 +284,14 @@ func approvedMove(ctx context.Context, inv registrations, forms *sourceForms, as if !onForge { return "", errors.New("the repository is not on the mesh's forge") } - merged, known, err := inv.MergeOf(ctx, owner+"/"+name, commit) + merged, known, err := keptForgeMerge(ctx, owner, name, commit) if err != nil { return "", err } if !known { return "", fmt.Errorf("the controller kept no merge of %s/%s at %.8s", owner, name, commit) } - return approvedAnnounced(ctx, inv, owner, name, announcedOf(merged)) + return approvedAnnounced(ctx, inv, owner, name, merged) } // errWordNotAnApprove says the merge cannot be shown to be the operator's Approve of its head: its announcement names no @@ -343,15 +343,67 @@ func claimantsOf(entries []inventory.Entry, seat string) []string { return out } -// announcedOf is a kept merge as the forge announced it, and what it moved; empty where the record says none. -func announcedOf(m inventory.BatchedMerge) keptMerge { - var k keptMerge - _ = json.Unmarshal(m.Event, &k) - return k +// forgeMergeKey is where the forge's announcement of a merge is kept, by the merge commit: the one thing a build of a merge +// names, from which the pull request and its head are found (novox/hq ADR 0420 §7). +func forgeMergeKey(owner, repo, commit string) string { + return fmt.Sprintf("merge/%s/%s/%s", owner, repo, commit) } -// keptMerge is a merge as the record keeps it: the forge's announcement, and the modules it moved when it was -// heard. The announcement reads back as a SourceMoved alone, which ignores the rest. +// keepForgeMerge keeps the forge's announcement of a merge in the bucket of the merges, a week at most as that bucket +// keeps them. The controller plans nothing from a merge any more (ADR 0539 §17); this is all it keeps of one. +var keepForgeMerge = func(ctx context.Context, m link.SourceMoved) error { + serving := servingBus.Load() + if serving == nil { + return fmt.Errorf("%w: the bus is not held", link.ErrTryAgain) + } + api, err := jetstream.New(serving.Conn()) + if err != nil { + return fmt.Errorf("%w: %w", link.ErrTryAgain, err) + } + kv, err := api.KeyValue(ctx, broker.DeliveryMergesBucket) + if err != nil { + return fmt.Errorf("%w: the bucket of the merges: %w", link.ErrTryAgain, err) + } + body, err := json.Marshal(keptMerge{SourceMoved: m}) + if err != nil { + return err + } + if _, err := kv.Put(ctx, forgeMergeKey(m.Owner, m.Repo, m.Commit), body); err != nil { + return fmt.Errorf("%w: %w", link.ErrTryAgain, err) + } + return nil +} + +// keptForgeMerge is the forge's announcement of a merge, by its commit; false when none was kept. A variable so a test can +// hand it one. +var keptForgeMerge = func(ctx context.Context, owner, repo, commit string) (keptMerge, bool, error) { + serving := servingBus.Load() + if serving == nil { + return keptMerge{}, false, errors.New("only the serving controller keeps what the forge announced") + } + api, err := jetstream.New(serving.Conn()) + if err != nil { + return keptMerge{}, false, err + } + kv, err := api.KeyValue(ctx, broker.DeliveryMergesBucket) + if err != nil { + return keptMerge{}, false, err + } + e, err := kv.Get(ctx, forgeMergeKey(owner, repo, commit)) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return keptMerge{}, false, nil + } + if err != nil { + return keptMerge{}, false, err + } + var k keptMerge + if err := json.Unmarshal(e.Value(), &k); err != nil { + return keptMerge{}, false, err + } + return k, true, nil +} + +// keptMerge is a merge as the record keeps it: the forge's announcement. type keptMerge struct { link.SourceMoved Moves []string `json:"moves,omitempty"` diff --git a/cmd/mesh-controller/moved_bus_test.go b/cmd/mesh-controller/moved_bus_test.go index 277295bd..26e99fb8 100644 --- a/cmd/mesh-controller/moved_bus_test.go +++ b/cmd/mesh-controller/moved_bus_test.go @@ -103,8 +103,11 @@ func TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTh const commit = "c0ffee0123456789" event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7, "head_sha": a.merge.HeadSHA, "merge_commit_sha": commit}) - if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-host", Branch: "main", Commit: commit, - Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil { + var forgeMerge link.SourceMoved + if err := json.Unmarshal(event, &forgeMerge); err != nil { + t.Fatal(err) + } + if err := keepForgeMerge(ctx, forgeMerge); err != nil { t.Fatal(err) } id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()) diff --git a/cmd/mesh-controller/new_module_approved_test.go b/cmd/mesh-controller/new_module_approved_test.go index 67725f52..1780a81b 100644 --- a/cmd/mesh-controller/new_module_approved_test.go +++ b/cmd/mesh-controller/new_module_approved_test.go @@ -6,7 +6,6 @@ import ( "errors" "strings" "testing" - "time" "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -19,7 +18,7 @@ import ( type anEmptyMesh struct { request inventory.BuildRequest asked bool - merge inventory.BatchedMerge + merge keptMerge } func (m *anEmptyMesh) Catalogue(context.Context) (map[string]catalogue.Manifest, error) { @@ -33,9 +32,6 @@ func (m *anEmptyMesh) Catalogued(context.Context) ([]inventory.Entry, error) { r func (m *anEmptyMesh) BuildRequestByID(_ context.Context, id string) (inventory.BuildRequest, bool, error) { return m.request, m.asked && id == m.request.ID, nil } -func (m *anEmptyMesh) MergeOf(context.Context, string, string) (inventory.BatchedMerge, bool, error) { - return m.merge, true, nil -} // aDeliveryMergeOf is the Approved merge of a pull request of novox/mesh-delivery, its heads keyed by that repository, // and the ask and warrant bound to exactly those heads (the digest the Approve left covers them). @@ -102,8 +98,12 @@ func TestANewModuleIsRegisteredOnTheOperatorsApproveOfItsMergeAndRefusedOtherwis } event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "base": "main", "number": 7, "head_sha": head, "merge_commit_sha": commit}) - mesh := &anEmptyMesh{asked: true, merge: inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", - Commit: commit, Merged: time.Now(), Event: event}, + var forgeMerge keptMerge + _ = json.Unmarshal(event, &forgeMerge) + wasForge := keptForgeMerge + t.Cleanup(func() { keptForgeMerge = wasForge }) + keptForgeMerge = func(context.Context, string, string, string) (keptMerge, bool, error) { return forgeMerge, true, nil } + mesh := &anEmptyMesh{asked: true, request: inventory.BuildRequest{ID: "build-1", Repository: "novox/mesh-delivery", Seat: "git", Path: "", Ref: "main", Commit: commit, For: c.request}} forms := &sourceForms{bases: map[string]string{}, base: func(seat string) string { diff --git a/cmd/mesh-controller/new_module_bus_test.go b/cmd/mesh-controller/new_module_bus_test.go index 03ff6e9a..2300c0a3 100644 --- a/cmd/mesh-controller/new_module_bus_test.go +++ b/cmd/mesh-controller/new_module_bus_test.go @@ -104,8 +104,11 @@ func TestANewModuleFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTheB const commit = "c0ffee0123456789" event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-delivery", "base": "main", "number": 7, "head_sha": a.merge.HeadSHA, "merge_commit_sha": commit}) - if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: commit, - Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil { + var forgeMerge link.SourceMoved + if err := json.Unmarshal(event, &forgeMerge); err != nil { + t.Fatal(err) + } + if err := keepForgeMerge(ctx, forgeMerge); err != nil { t.Fatal(err) } id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()) diff --git a/cmd/mesh-controller/new_module_on_merge_test.go b/cmd/mesh-controller/new_module_on_merge_test.go index 4a763321..73db2ae5 100644 --- a/cmd/mesh-controller/new_module_on_merge_test.go +++ b/cmd/mesh-controller/new_module_on_merge_test.go @@ -39,9 +39,6 @@ func TestAMergeAddingAModuleEndsRegisteredFromItsKeyAndAsksNoBuild(t *testing.T) if err := (following{open: open}).SourceMoved(ctx, m); err != nil { t.Fatal(err) } - if plans, err := inv.OpenPlans(ctx); err != nil || len(plans) != 0 { - t.Fatalf("a merge moving nothing the mesh holds opened a plan: %+v %v", plans, err) - } // The indexer writes the module's trunk value; the builder builds it, then its key is done. fp := fpOf('7') manifest := b.indexed(t, "systemd-resolved", "1", "novox/mesh-catalog", "modules/systemd-resolved", commit, fp) diff --git a/cmd/mesh-controller/nodes.go b/cmd/mesh-controller/nodes.go index 232bd89e..70f275f4 100644 --- a/cmd/mesh-controller/nodes.go +++ b/cmd/mesh-controller/nodes.go @@ -796,19 +796,6 @@ func showNode(ctx context.Context, stored *stores, name string) error { if err := showMode(ctx, inv, node); err != nil { return err } - // What it was last sent, by whom, under which epoch, from which generation, naming which modules, and what - // it answered (novox/hq issue 485): the record of sends is read through this verb, never the store's reader. - // Unreadable is said, not taken for none. - switch last, found, err := inv.LastSend(ctx, name); { - case err != nil: - fmt.Printf(" what it was last sent could NOT be read: %v\n", err) - case !found: - fmt.Printf(" last sent no send recorded since the mesh began keeping them\n") - default: - for _, line := range lastSendLines(last) { - fmt.Println(line) - } - } // Whom agents run as here, and whether that account can become root without a person (ADR 0266). for _, line := range agentAccountLines(ctx, inv, node) { fmt.Println(line) @@ -915,12 +902,6 @@ func showNode(ctx context.Context, stored *stores, name string) error { } if len(assigned) > 0 { fmt.Printf("\n assigned: %s\n", strings.Join(assigned, ", ")) - // And what each runs, as its assignment names it (novox/hq ADR 0408, issue 1095). - if on, err := inv.AssignmentsOn(ctx, name); err == nil { - for _, line := range runLines(on) { - fmt.Println(line) - } - } } // And which of them a push leaves out, and why (novox/hq issue 380): judged as the push judges it, the same // reason its condition says. Not computable is said, never read as "all applied". diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 3c7cf997..06cfc19a 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -10,7 +10,6 @@ package main import ( "fmt" - "sort" "strings" "time" @@ -697,16 +696,6 @@ func namesWords(names []string, most int) string { return strings.Join(names[:most], ", ") + fmt.Sprintf(" and %d more", len(names)-most) } -// planModules are the modules a walk moves, sorted. -func planModules(p inventory.Plan) []string { - out := make([]string, 0, len(p.Modules)) - for m := range p.Modules { - out = append(out, m) - } - sort.Strings(out) - return out -} - // deliveryWhat is what a delivery changes, as a person names it: its modules when they are few, else // their count and the repository. func deliveryWhat(modules []string, repository string) string { @@ -727,18 +716,6 @@ func deliveryName(modules []string, repository string) string { return repoName(repository) + " delivery" } -// walkWaitingWords explains a walk waiting for its delivery's word. -func walkWaitingWords(w waitFacts, in time.Duration, severity conditions.Severity) string { - what := deliveryWhat(w.modules, w.repository) - if severity == conditions.Urgent { - return fmt.Sprintf("The change to %s is merged and built, and mesh-delivery (the module that decides when a "+ - "delivery goes out) has not let it start for %s, so mesh-delivery may be stuck.", what, humanDuration(in)) - } - return fmt.Sprintf("The change to %s is merged and built, and has waited %s for mesh-delivery (the module that "+ - "decides when a delivery goes out) to let it start. It becomes a question for you if it still waits after %s.", - what, humanDuration(in), humanDuration(waitUrgentAfter)) -} - // waitingNeeds is what the operator does about a walk waiting past its urgent bound: nothing before it. // Start and Stop are also asked of the operator (novox/hq ADR 0259); the condition's own words keep saying // where they are given without a channel, and the ask's text drops that (askText). diff --git a/cmd/mesh-controller/plan.go b/cmd/mesh-controller/plan.go index bd0a70fe..24cba62e 100644 --- a/cmd/mesh-controller/plan.go +++ b/cmd/mesh-controller/plan.go @@ -1,11 +1,8 @@ package main import ( - "bytes" "context" - "encoding/json" "errors" - "flag" "fmt" "io" "os" @@ -278,182 +275,6 @@ func listensLines(m catalogue.Manifest) []string { return out } -func planCommand(ctx context.Context, args []string) error { - // One reading of the mesh for every machine this looks at. - ctx = withInputCache(ctx) - set := flag.NewFlagSet("plan", flag.ContinueOnError) - // Because "one resource" does not tell you whether the settings landed. Being able to read - // the file before it is sent is the difference between believing a merge worked and knowing. - show := set.Bool("files", false, "print the files this node would be given") - // The declaration exactly as the node would receive it. For handing to something else -- - // checking it against the host's own parser, most usefully, which is the only way to know - // that what the control plane emits is what the host accepts. - asJSON := set.Bool("json", false, "print the declaration this node would be sent") - // What a push would change, against what the machine was last sent (novox/hq ADR 0217): read - // before sending, so a change that removes, moves or recreates something is seen first. - asDiff := set.Bool("diff", false, "what a push would change on this node, against what it was last sent") - positionals, err := parseAround(set, args) - if err != nil { - return err - } - if len(positionals) != 1 { - return errors.New("plan [--files] [--json] [--diff]") - } - args = positionals - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - - plan, settings, err := resolveNode(ctx, open, args[0]) - if err != nil { - return err - } - if len(plan.Modules) == 0 { - fmt.Printf("%s is assigned nothing\n", args[0]) - return nil - } - // Which optional requirements are unbound (novox/hq ADR 0330 §7): beside the document on the - // error stream where the answer is the declaration itself, which carries no such field — the - // node-engine refuses an envelope key it does not know. - if *asDiff || *asJSON { - for _, line := range unboundLines(plan) { - fmt.Fprintf(os.Stderr, "%s:%s\n", args[0], line) - } - } - if *asDiff { - declared, err := declarationFor(ctx, open, args[0], plan, settings) - if err != nil { - return err - } - body, err := declared.Body() - if err != nil { - return err - } - // A module left out is not in the body, so the diff alone would say "nothing would change" for - // a module just assigned whose settings cannot compose — success-shaped silence. Said first, with - // why, as push and the plain plan say it (novox/hq ADR 0163, rule 6). - reportLeftOut(args[0], declared) - return writePlanDiff(ctx, open.inventory, args[0], body) - } - if *asJSON { - declared, err := declarationFor(ctx, open, args[0], plan, settings) - if err != nil { - return err - } - // The bytes a push would send, indented: one marshaller, so `plan --json` cannot show an - // envelope other than the one sent. - body, err := declared.Body() - if err != nil { - return err - } - var indented bytes.Buffer - if err := json.Indent(&indented, body, "", " "); err != nil { - return err - } - fmt.Println(indented.String()) - return nil - } - - // What the machine was last told, by whom and from which assignment generation (novox/hq issue 234), - // beside what it would be told now. A record that cannot be read is said, not taken for none. - if err := writeLastSend(ctx, os.Stdout, open.inventory, args[0]); err != nil { - fmt.Printf("what %s was last sent could not be read: %v\n", args[0], err) - } - // Which modules a push would leave out, and why — said before the plan, since the plan is of - // what the machine would be told (novox/hq ADR 0163, rule 6). Judged, never composed: `plan` - // without --json allocates nothing. - if record, err := open.inventory.NodeByName(ctx, args[0]); err == nil { - left := plan.LeftOut(settings, record.Adopted) - reportLeftOut(args[0], sendable{LeftOut: sortedKeysOf(left), leftOutWhy: left}) - } - // And which of its modules no provider will grant, because the identity overflows the bound of - // what it requires (novox/hq ADR 0225) — said on the machine the remedy is for. - for _, o := range plan.Overflowing() { - fmt.Printf("%s: %s\n", args[0], o) - } - // And a setting that reaches nothing — refused where it is stored, and said here for one - // stored before its definition moved from under it. - for _, m := range plan.Modules { - for _, stray := range catalogue.UnusedSettings(m, settings[m.Module]) { - fmt.Printf("%s: a setting reaches nothing — %s\n", args[0], stray) - } - } - fmt.Printf("%s would run:\n", args[0]) - for _, m := range plan.Modules { - fmt.Printf(" %-20s %s\n", m.Module, plan.Because[m.Module]) - for _, line := range listensLines(m) { - fmt.Println(line) - } - } - // What was assigned here and cannot run here. Said with the rest rather than as a refusal: it is - // one module on the wrong machine, the others still run, and the remedy is to move this one. - for _, u := range plan.Unhostable { - for _, c := range u.Missing { - fmt.Printf(" %-20s not applied — %s\n", u.Module, catalogue.WrongMachine(u.Module, c, args[0])) - } - } - for _, c := range plan.Claims { - fmt.Printf(" holds %s, one per %s\n", c.Claim, c.Scope) - } - // What this machine depends on that is not on it. Worth saying out loud: it is the only part - // of a node's set that stops working when a *different* machine goes away, and nothing else - // in this output would have told anybody that. - for _, n := range plan.Needs { - fmt.Printf(" needs %s from %s, for %s\n", n.Name, n.From, n.For) - // And a served key the provider left out, because its machine composes no name it asks for - // (novox/hq ADR 0346 §3). - for _, line := range n.LeftOut { - fmt.Printf(" %s\n", line) - } - } - // What the providers on this machine leave out of what they serve, and why: said where the remedy - // is (novox/hq ADR 0346 §3). - for _, line := range plan.ServedLeftOut(settings) { - fmt.Printf("%s: %s\n", args[0], line) - } - for _, line := range unboundLines(plan) { - fmt.Println(line) - } - // And a container's busy that this machine's node-engine is too old to be sent (novox/hq ADR 0335): its - // container is replaced at once, as before. A statement that cannot be read is said as such. - if reads, err := engineReadsBusy(ctx, open.inventory, args[0]); err != nil { - fmt.Printf("%s: whether its node-engine reads a container's busy could not be read: %v\n", args[0], err) - } else { - for _, line := range busyNotSent(plan.Modules, reads) { - fmt.Printf("%s: %s\n", args[0], line) - } - } - declared, err := declarationFor(ctx, open, args[0], plan, settings) - if err != nil { - return err - } - resources := declared.Resources - for module, layers := range settings { - for _, layer := range layers { - fmt.Printf(" %-20s settings from %s\n", module, layer.From) - } - } - // What each port reached from the local network resolved to on this machine (novox/hq ADR 0340 §8), - // or that it resolved to none. - for _, p := range declared.localNetwork { - fmt.Println(" " + p.Said()) - } - fmt.Printf("\n%d resource(s)\n", len(resources)) - - if *show { - for _, r := range resources { - content, ok := r["content"].(string) - if !ok { - continue - } - fmt.Printf("\n--- %s ---\n%s", shownAs(r), content) - } - } - return nil -} - // shownAs is the heading `plan --show` puts over a resource's content. // // **A file written into says so.** Its content is the mesh's part of a file that is otherwise the diff --git a/cmd/mesh-controller/plan_moved_every_writer_test.go b/cmd/mesh-controller/plan_moved_every_writer_test.go deleted file mode 100644 index 1b68d3da..00000000 --- a/cmd/mesh-controller/plan_moved_every_writer_test.go +++ /dev/null @@ -1,123 +0,0 @@ -package main - -import ( - "os" - "path/filepath" - "regexp" - "strings" - "sync" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// **Every plan is written through SavePlansMoving, which says it** (issue 974, novox/hq ADR 0406 §2: every move of a plan -// is said as plan-moved, so no move is found only by comparing). The one hook, inventory.PlanSaved, is called there after the -// write; so no code but chainSavePlans (which SavePlansMoving calls, and no other function) writes a plan to the rollouts -// bucket by anything but a change of the merges it holds. Checked by reading the sources, no store needed. -func TestNoPlanIsWrittenButThroughSavePlansMoving(t *testing.T) { - write := regexp.MustCompile(`(?is)(insert\s+into|update|delete\s+from|truncate)\s+(table\s+)?release_plan\b`) - var found []string - for _, root := range []string{"../../cmd", "../../internal"} { - err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { - return nil - } - raw, err := os.ReadFile(path) - if err != nil { - return err - } - text := string(raw) - // The regular expression is the costly part of reading every source: only a file that names the table can match. - if strings.Contains(strings.ToLower(text), "release_plan") && write.MatchString(text) { - found = append(found, path+" writes the plan table") - } - // A plan record is written by the bucket's raw create and update, which only the plans' own file calls. - if filepath.ToSlash(path) != "../../internal/inventory/chain_rollouts.go" && filepath.ToSlash(path) != "../../internal/chain/chain.go" && - strings.Contains(text, "chain.RolloutsOf") { - found = append(found, path+" reaches the rollouts bucket") - } - return nil - }) - if err != nil { - t.Fatal(err) - } - } - if len(found) > 0 { - t.Fatalf("a plan is written without plan-moved being said:\n%s", strings.Join(found, "\n")) - } - raw, err := os.ReadFile("../../internal/inventory/plans.go") - if err != nil { - t.Fatal(err) - } - text := string(raw) - save := strings.Index(text, "func (i *Inventory) SavePlansMoving") - next := strings.Index(text[save:], "\nfunc ") - if next < 0 { - next = len(text) - save - } - if !strings.Contains(text[save:save+next], "chainSavePlans(") { - t.Error("SavePlansMoving does not write through chainSavePlans") - } - chainRaw, err := os.ReadFile("../../internal/inventory/chain_rollouts.go") - if err != nil { - t.Fatal(err) - } - chainText := string(chainRaw) - at := strings.Index(chainText, "func (i *Inventory) chainSavePlans") - end := strings.Index(chainText[at:], "\nfunc ") - if !strings.Contains(chainText[at:at+end], "PlanSaved(i, *p)") { - t.Error("chainSavePlans does not say each plan it wrote") - } - if n := strings.Count(chainText, "createRaw(ctx, c, p.ID"); n != 1 { - t.Errorf("a plan is made from %d places of chain_rollouts.go, want the one in chainSavePlans", n) - } -} - -// movesSaid records what the hook hears: for each plan, the revisions it was said at. -type movesSaid struct { - mu sync.Mutex - revs map[string][]int64 -} - -func (s *movesSaid) hear(_ *inventory.Inventory, p inventory.Plan) { - s.mu.Lock() - defer s.mu.Unlock() - s.revs[p.ID] = append(s.revs[p.ID], p.Revision) -} - -// sayingEverySave installs a PlanSaved that records, as the serving controller's says plan-moved. -func sayingEverySave(t *testing.T) *movesSaid { - t.Helper() - s := &movesSaid{revs: map[string][]int64{}} - was := inventory.PlanSaved - inventory.PlanSaved = s.hear - t.Cleanup(func() { inventory.PlanSaved = was }) - return s -} - -// noUnsaid fails the test for a plan whose stored revision is not one the hook heard: a move nobody said. -func (s *movesSaid) noUnsaid(t *testing.T, open *stores) { - t.Helper() - recent, err := open.inventory.RecentPlans(t.Context(), 50) - if err != nil { - t.Fatal(err) - } - s.mu.Lock() - defer s.mu.Unlock() - if len(recent) == 0 { - t.Fatal("no plan was kept") - } - for _, p := range recent { - revs := s.revs[p.ID] - if len(revs) == 0 || revs[len(revs)-1] != p.Revision { - t.Errorf("%s is kept at revision %d and was said at %v: a move nobody said", p.ID, p.Revision, revs) - } - } -} - -// Live, with the store: the moves a merge makes of a plan — the batch it opens, the walk it is cut into, the tier that -// is asked — are each said, and a look that changes nothing says nothing (novox/hq ADR 0406 §2). diff --git a/cmd/mesh-controller/plans_gone_store_integration_test.go b/cmd/mesh-controller/plans_gone_store_integration_test.go new file mode 100644 index 00000000..d8bd652c --- /dev/null +++ b/cmd/mesh-controller/plans_gone_store_integration_test.go @@ -0,0 +1,35 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// **The controller opens on a store whose plan tables are gone** (novox/hq ADR 0539 §17, issue 1945 B9): migration 0108 has +// dropped the record of sends, the assignment generation, the gate's verdicts and the durations, and the controller still +// assigns a module, reads where it runs, takes it off and reads the nodes. +func TestTheControllerOpensOnAStoreWithoutThePlanTables(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) { + switch name := strings.Trim(row[0], `"`); name { + case "declaration_send", "assignment_generation", "build_gate", "duration": + t.Errorf("the table %s is in the store", name) + } + } + register(t, open, catalogue.Manifest{Module: "notes", Version: "1"}) + if _, err := assign(ctx, open, "anchor", "notes"); err != nil { + t.Fatalf("assign on the migrated store: %v", err) + } + if on, err := inv.Assigned(ctx, "anchor"); err != nil || len(on) != 1 || on[0] != "notes" { + t.Fatalf("assigned %v: %v", on, err) + } + if nodes, err := inv.Nodes(ctx); err != nil || len(nodes) == 0 { + t.Fatalf("nodes %v: %v", nodes, err) + } +} diff --git a/cmd/mesh-controller/probes.go b/cmd/mesh-controller/probes.go index b082fbb5..825a4c12 100644 --- a/cmd/mesh-controller/probes.go +++ b/cmd/mesh-controller/probes.go @@ -1087,16 +1087,12 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, if err != nil { return nil, err } - plans, err := inv.OpenPlans(ctx) - if err != nil { - return nil, err - } shelf, err := inv.Catalogue(ctx) if err != nil { return nil, err } hostVersions := deliveredVersions(shelf[hostModule]) - rolling := rollingModules(plans) + rolling := map[string]bool{} // no plan rolls a module out any more (ADR 0539 §17) nodes, err := inv.Nodes(ctx) if err != nil { return nil, err @@ -1154,26 +1150,6 @@ func probeCoreBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, return out, nil } -// rollingModules is every module an open plan is rolling out: those it keeps a record of, and those its -// tiers name. **A release keeps no record per module** — its walk is per machine, its modules only in its -// tier — so reading the records alone, D10 said "no plan is rolling them out" about the node-engine while -// a release walked it, and the gate on that release's first machine waited on what its own send causes -// (novox/hq issue 348). Pure. -func rollingModules(plans []inventory.Plan) map[string]bool { - rolling := map[string]bool{} - for _, p := range plans { - for m := range p.Modules { - rolling[m] = true - } - for _, tier := range p.Tiers { - for _, m := range tier { - rolling[m] = true - } - } - } - return rolling -} - // deliveredVersions are the versions a module's registered build is delivered as: the last element // of every resource path under a `versions/` directory, which registration filled from the artifact's // digest (catalogue `${version}`). The node-engine names itself by that directory. diff --git a/cmd/mesh-controller/provisioned.go b/cmd/mesh-controller/provisioned.go deleted file mode 100644 index 0c888a17..00000000 --- a/cmd/mesh-controller/provisioned.go +++ /dev/null @@ -1,100 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "os" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" -) - -// What a machine's provisioner client said of a module's files (novox/hq ADR 0362, ADR 0349 §15, design 57 P6, -// issue 672), read where a walk's end and a gate count a machine's send applied: a module the send handed to the -// provisioner is applied there only once the client reported `prov.applied` on that build's templates as well. - -// provAppliedOn is the client's newest report of a module on a machine, from the stream that keeps it -// (broker.ProvAppliedStream); false when it has said nothing of it. A variable so a test can say the client's words. -var provAppliedOn = func(ctx context.Context, node, module string) (inventory.ProvReport, bool, error) { - var raw []byte - var said bool - read := func(js nats.JetStreamContext) error { - var err error - raw, said, err = broker.LastProvApplied(js, node, module) - return err - } - var err error - if d := doctorFrom; d != nil && d.js != nil { - err = read(d.js.Context()) - } else { - err = onTheBus(func(conn *nats.Conn) error { - js, err := conn.JetStream(nats.Context(ctx)) - if err != nil { - return err - } - return read(js) - }) - } - if err != nil || !said { - return inventory.ProvReport{}, false, err - } - var r inventory.ProvReport - if err := json.Unmarshal(raw, &r); err != nil { - return inventory.ProvReport{}, false, fmt.Errorf("what %s's provisioner client said of %s is not its report: %w", - node, module, err) - } - return r, true, nil -} - -// provLookup answers the clients' reports for one reading; a report that cannot be read is not counted, which -// leaves the machine waiting, then silent, never applied — and is said. -func provLookup(ctx context.Context) inventory.ProvLookup { - return func(node, module string) (inventory.ProvReport, bool) { - r, said, err := provAppliedOn(ctx, node, module) - if err != nil { - fmt.Fprintf(os.Stderr, "what %s's provisioner client said of %s could not be read: %v\n", node, module, err) - return inventory.ProvReport{}, false - } - return r, said - } -} - -// provisioningFailed is every module whose files a machine's provisioner client last said it failed to place, from -// the stream that keeps their reports. -func provisioningFailed(js nats.JetStreamContext) ([]inventory.ProvReport, error) { - all, err := broker.ProvAppliedReports(js) - if err != nil { - return nil, err - } - var out []inventory.ProvReport - for subject, raw := range all { - var r inventory.ProvReport - if err := json.Unmarshal(raw, &r); err != nil { - return nil, fmt.Errorf("what was said on %s is not a provisioner client's report: %w", subject, err) - } - if r.Status == inventory.ProvFailed { - out = append(out, r) - } - } - return out, nil -} - -// provisionedOutcome is a machine's report of one send as a delivery's end counts it (novox/hq ADR 0362 §1-5, issue -// 840): the engine's own report, combined with its provisioner client's for a send that handed modules to it. The -// outcome of a client's failure begins inventory.OutcomeProvisioningFailed; a report not yet applied on both sides -// is none (false). A send handing nothing is the engine's report unchanged. onSend says the report is about the send. -func provisionedOutcome(node string, sent inventory.SentDeclaration, r inventory.Reported, onSend bool, - prov inventory.ProvLookup) (string, bool) { - if len(sent.Provisioned) == 0 { - return r.Outcome, onSend - } - engine := inventory.AppliedReport{Outcome: r.Outcome} - if r.At != nil { - engine.At = *r.At - } - got, ok := inventory.ProvisionedSend(node, sent, engine, onSend, prov) - return got.Outcome, ok -} diff --git a/cmd/mesh-controller/provisioning_failed.go b/cmd/mesh-controller/provisioning_failed.go new file mode 100644 index 00000000..a3617a04 --- /dev/null +++ b/cmd/mesh-controller/provisioning_failed.go @@ -0,0 +1,31 @@ +package main + +import ( + "encoding/json" + "fmt" + + "github.com/nats-io/nats.go" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" +) + +// provisioningFailed is every module whose files a machine's provisioner client last said it failed to place, from +// the stream that keeps their reports. +func provisioningFailed(js nats.JetStreamContext) ([]inventory.ProvReport, error) { + all, err := broker.ProvAppliedReports(js) + if err != nil { + return nil, err + } + var out []inventory.ProvReport + for subject, raw := range all { + var r inventory.ProvReport + if err := json.Unmarshal(raw, &r); err != nil { + return nil, fmt.Errorf("what was said on %s is not a provisioner client's report: %w", subject, err) + } + if r.Status == inventory.ProvFailed { + out = append(out, r) + } + } + return out, nil +} diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 578a3414..e42f3dbf 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -4,22 +4,18 @@ import ( "context" "crypto/sha256" "encoding/hex" - "encoding/json" "errors" "fmt" "github.com/novox/mesh-controller/internal/conditions" "io" "log" "os" - "slices" "sort" "strings" "time" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/compose" - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" @@ -194,7 +190,6 @@ func serve(ctx context.Context) (err error) { // Never by a read. go settlingPending(ctx, open) // The durations the core's bounds are set from are kept a month (novox/hq to-be 45 Phase 0). - go forgettingOldDurations(ctx, inv) // And what the catalogue decided a build meant. The builder's own result is already handled // above; this is the other half — the control plane is the only one of the three that knows // which machines run the thing, so it is the one that acts (novox/hq ADR 0072). @@ -374,108 +369,6 @@ func waitForApplied(ctx context.Context, inv *inventory.Inventory, node, digest } } -// readyNode is one machine and the declaration it would be sent. -type readyNode struct { - node string - declared sendable -} - -// composeEach works out what each named machine should be, and never lets one machine's answer -// decide another's. -// -// **A machine whose set cannot be worked out is that machine's problem** (novox/hq ADR 0066). A -// whole-mesh push used to refuse outright when any one node failed to resolve, so a single -// unanswerable requirement on a single machine — one module requiring a provision nobody had -// assigned a provider for — left every other machine in the mesh unconverged, including machines -// with no relation to it at all. Nothing was sent anywhere, and the machines that could not be sent -// were the ones with nothing wrong with them. -// -// It is the same rule a92c11b established one level down, where an un-hostable module stopped -// taking down the healthy modules beside it, applied one level up: **the blast radius of a fault is -// the thing that has it.** What could not be worked out is named and returned, so a push still ends -// with a non-zero outcome and nobody mistakes a partial convergence for a whole one. -// -// The all-or-nothing rule is kept where it means something — sendTo, which rotates a credential -// across two machines that must agree — and dropped here, where it never did. -func composeEach(names []string, allot func(node string) (order, error), - compose func(node string) (sendable, error)) ([]readyNode, []string) { - - var sending []readyNode - var refusals []string - for _, name := range names { - // **Numbered before it is composed, not before it is sent** (novox/hq issue 204). The - // number says where this declaration stands against every other the mesh composed for the - // machine, and the host refuses one lower than the last it applied. Taken at send time, as - // it was, a declaration composed a minute ago — before an assignment changed — went out with - // a number higher than one composed after the change and sent before it, and the machine - // took the older content as the newer word: on 2026-10-02 a runtime assigned and applied on - // two machines was undone two seconds later by exactly that. Taken here, before the first - // read, what was composed earlier is numbered lower whatever order the sends happen in. - numbered, err := allot(name) - if err != nil { - refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) - continue - } - declared, err := compose(name) - if err != nil { - refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) - continue - } - numbered.stamp(&declared) - if len(declared.Resources) == 0 { - // Sent, not skipped (novox/hq issue 127). A node whose declaration composes to - // nothing may have HELD something before — the broker opening a placement gave it, - // say — and skipping the empty declaration leaves that last resource in force - // forever, re-applied by the node's own heartbeat, with no way for the mesh to say - // it is gone. An empty declaration is the correction: the host drops what the mesh - // owned and keeps what it found (the adoption envelope still rides along). A node - // that never held anything applies it as the no-op it is. - fmt.Printf("%s owns nothing now — sent so it drops what it last held\n", name) - } - sending = append(sending, readyNode{name, declared}) - } - return sending, refusals -} - -// keeping is what a round of sends keeps of the machines it composed, before any is delivered: nil -// keeps them all. A push holds back the machines whose running modules' data would move (holdingBack). -type keeping func(held context.Context, sending []readyNode) ([]readyNode, error) - -// holdingBack leaves out each machine whose declaration would move a running module's data, says so, -// and names it among those held back (novox/hq ADR 0217); the rest go on to be delivered, grants first -// (issue 249). A declaration's body is the same bytes however often it is read. -func holdingBack(ctx context.Context, inv *inventory.Inventory, sending []readyNode, movable map[string]bool, - heldBack *[]string) ([]readyNode, error) { - var out []readyNode - for _, s := range sending { - body, err := s.declared.Body() - if err != nil { - return nil, err - } - moves, err := heldMoves(ctx, inv, s.node, body, movable) - if err != nil { - return nil, err - } - if len(moves) > 0 { - sayDataHeld(os.Stdout, s.node, moves) - *heldBack = append(*heldBack, s.node) - continue - } - out = append(out, s) - } - return out, nil -} - -// delivery is the two acts of sending machines what they should be, apart, so the order between -// them is one function's and can be read and tested there (novox/hq issue 249). -type delivery interface { - // grant issues what the machines' modules may do — each module's state raised and its - // membership issued — for every machine about to be sent. - grant(ctx context.Context, sending []readyNode) error - // declare sends one machine its declaration and records it sent, answering the digest. - declare(ctx context.Context, s readyNode, body []byte) (string, error) -} - // errGrants marks a send that stopped because what the machines' modules may do could not be issued // (novox/hq issue 249). Nothing about the machines is wrong; asked again, it is likely to work, so an // announcement that hits it is held and asked again. @@ -496,158 +389,9 @@ func (g *grantsRefused) Error() string { strings.Join(names, ", "), g.nodes[names[0]]) } -// deliver sends the machines their declarations: **the machine holding the bus, then the grants, -// then the rest** (novox/hq issue 249). -// -// A merge gave a module a new state; its bundle reached every machine within a minute, and the -// machines' permissions on the bus did not include the state until somebody pushed by hand: the code -// arrived before the right to use it. A module that read its new state on start failed its start; the -// one that was there retried for two minutes. The memberships were issued after the declarations — -// "because the runtime it is for arrives with it" — and a membership is retained last-per-subject on -// the bus (internal/link/bus.go), so issued first it waits for the runtime that arrives after it. A -// runtime still on the old code merely holds a grant it does not use yet. -// -// **The holder's declaration before the grants, though.** The bus's user list travels in it, and the -// controller's own right to publish memberships and raise buckets is in that list (the precedent of -// issue 183): grants first, and a grant the controller is not yet allowed to make would hold the very -// declaration that allows it — a lock only a hand on the broker could open. A runtime already running -// on that machine follows a membership issued after its declaration, as it always has. -// -// **A grant that cannot be issued holds back what it concerns, and says so as an error.** It used to -// be said and passed over — "the machines keep what they derive until the next push" — which reported -// a rollout done that had delivered code its machines could not run. A membership that failed holds -// back its own machine; a failure that names no machine (the buckets) holds back every machine but the -// holder, already sent. The error carries errGrants, so the caller's rollout is not marked sent and is -// tried again. -// -// The grants are issued, not waited on: a membership is a retained message the runtime reads when it -// comes, and the bus answers its publication; nothing here waits for a runtime to have read one. -func deliver(ctx context.Context, d delivery, holder string, sending []readyNode) (map[string]string, error) { - digests := map[string]string{} - if len(sending) == 0 { - return digests, nil - } - send := func(s readyNode) error { - // The number is inside the signed bytes, so a replayed older declaration cannot borrow a - // newer one's (novox/hq 04-ISSUES/107); it was taken when the composition began (issue 204). - body, err := s.declared.Body() - if err != nil { - return err - } - digest, err := d.declare(ctx, s, body) - if errors.Is(err, errFilesNotPut) && s.declared.fileReferences { - // **The grant that lets the controller write the bucket travels in the very declaration of the machine holding the - // bus** (novox/hq ADR 0486 §2): until the bus has it, a put is refused, and the declaration that would deliver the - // grant would wait for it. So that send, and only that, goes with every content inline, as to a node that reads no - // references, and says so; the next one tries the bucket again. - fmt.Printf("%s: the files that travel by digest could not be put in the bucket (%v): sending them inline\n", s.node, err) - s.declared.fileReferences = false - if body, err = s.declared.Body(); err != nil { - return err - } - digest, err = d.declare(ctx, s, body) - } - if err != nil { - return err - } - digests[s.node] = digest - return nil - } - var rest []readyNode - for _, s := range sending { - if holder != "" && s.node == holder { - if err := send(s); err != nil { - return digests, err - } - continue - } - rest = append(rest, s) - } - held := map[string]error{} - if err := d.grant(ctx, sending); err != nil { - var some *grantsRefused - if !errors.As(err, &some) { - var names []string - for _, s := range rest { - names = append(names, s.node) - } - if len(names) == 0 { - return digests, fmt.Errorf("%w: %w", errGrants, err) - } - return digests, fmt.Errorf("%w; %s not sent: %w", errGrants, strings.Join(names, ", "), err) - } - held = some.nodes - } - var notSent []string - for _, s := range rest { - if _, refused := held[s.node]; refused { - notSent = append(notSent, s.node) - continue - } - if err := send(s); err != nil { - return digests, err - } - } - if len(notSent) > 0 { - return digests, fmt.Errorf("%w; %s not sent: %w", errGrants, strings.Join(notSent, ", "), - &grantsRefused{nodes: held}) - } - if len(held) > 0 { - // Only the holder's own memberships failed, and it was sent before them. - return digests, fmt.Errorf("%w: %w", errGrants, &grantsRefused{nodes: held}) - } - return digests, nil -} - -// overTheBus is delivery as the mesh does it: memberships on the bus, declarations signed. -type overTheBus struct { - open *stores - server *link.Server - signer link.Signer - // indent is put before each "sent" line, for the callers whose output is nested. - indent string -} - -// putFiles puts the contents s names by digest in the bucket. -func (b overTheBus) putFiles(ctx context.Context, s readyNode) error { - objects, err := s.declared.objects() - if err != nil { - return fmt.Errorf("%s: the files that travel by digest cannot be read: %w", s.node, err) - } - if len(objects) == 0 { - return nil - } - bus, ok := b.server.Bus().(link.OverNATS) - if !ok { - return fmt.Errorf("%s: the declaration names files by digest and this bus has no object bucket", s.node) - } - putting, cancel := context.WithTimeout(ctx, 30*time.Second) - defer cancel() - store, err := openBucket(putting, bus.Conn, s.node) - if err == nil { - err = putObjects(putting, store, objects) - } - notePut(ctx, s.node, err) - if err != nil { - return fmt.Errorf("%s was not sent its declaration: %w: %w", s.node, errFilesNotPut, err) - } - return nil -} - // errFilesNotPut marks a send that stopped because the files it names by digest could not be put in the bucket. var errFilesNotPut = errors.New("the files it names by digest are not in the bucket") -func (b overTheBus) grant(ctx context.Context, sending []readyNode) error { - // The bus's objects first, which every declaration implies (novox/hq issue 208): said and raised - // when they cannot be, and never what holds the send back (assertOnSend). - if bus, ok := b.server.Bus().(link.OverNATS); ok { - js := broker.OnConn(bus.Conn) - js.Note = func(format string, args ...any) { fmt.Printf(b.indent+" "+format+"\n", args...) } - _ = assertOnSend(ctx, b.open.inventory, js, b.indent) - } - return issueMemberships(ctx, b.open, b.server, sending) -} - // brokerFirst is the machines to send in the order a grant needs (novox/hq issue 249): the machine // holding the bus first — its declaration carries the bus's user list (composeBusUsers), and a // module's new permissions are refused by the bus until that list says them. Among the machines it @@ -755,219 +499,6 @@ func couldNotBeResolved(refusals []string, sent int) error { len(refusals), sent, strings.Join(refusals, "\n\n")) } -// sendTo resolves and sends to exactly the machines named, or refuses without sending anything. -// -// The all-or-nothing rule push deliberately does NOT follow, and for a reason that holds here and -// not there: a rotation that reached the -// consumer and refused on the provider would leave one end holding a credential the other has -// never heard of — which is the state this whole mechanism exists to make impossible. -func sendTo(ctx context.Context, open *stores, names []string) error { - _, err := sendToEach(ctx, open, names) - return err -} - -// sendToEach is sendTo, answering the machines it sent: those named, and before them the machine -// holding the bus when its user list must go first (novox/hq issue 249) — so a caller that waits for -// the machines it sent waits for that one too. -func sendToEach(ctx context.Context, open *stores, names []string) ([]string, error) { - inv := open.inventory - // **No send replaces the bus but its planned step** (novox/hq ADR 0236). A plan's send to the bus's - // machine for some other module carried the bus's new build with it on 2026-10-06, and the bus - // restarted under every machine with nobody having asked. Refused whole — the send cannot leave the - // bus behind and carry the rest (ADR 0221 option 2) — and said with the remedy; the plan tries again. - if !busStepSending(ctx) { - held, err := busHeld(ctx, inv, names) - if err != nil { - return nil, err - } - for _, n := range names { - if why, h := held[n]; h { - return nil, fmt.Errorf("%w: %s", errBusWaits, why) - } - } - } - ident, err := openIdentity(ctx) - if err != nil { - return nil, err - } - defer ident.Close() - - // The private network every declaration of the send is composed with: not computable refuses the send up front. - if _, err := generatorsOf(ctx, open); err != nil { - return nil, err - } - holder, behind, err := brokerBehind(ctx, open, names) - if err != nil { - return nil, err - } - names = brokerFirst(names, holder, behind) - // Held from composing to sending (novox/hq ADR 0100); a caller that holds them already — - // converge, which flips the node and then sends it — is not made to wait on itself. - ctx, release, err := holdNodes(ctx, open, names) - if err != nil { - return nil, err - } - defer release() - - var sending []readyNode - var refusals []string - for _, name := range names { - // Numbered before composing, for the reason composeEach gives (novox/hq issue 204). - numbered, err := allot(ctx, inv, name) - if err != nil { - refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) - continue - } - // Read afresh for each machine: the send before it made and recorded things. - composing := freshInput(ctx) - plan, _, err := resolveNode(composing, open, name) - if err != nil { - refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) - continue - } - reportUnhostable(name, plan) - reportKept(ctx, plan) - declared, err := composeSend(composing, open, name) - if err != nil { - refusals = append(refusals, fmt.Sprintf("%s:\n%v", name, err)) - continue - } - numbered.stamp(&declared) - reportLeftOut(name, declared) - sending = append(sending, readyNode{name, declared}) - } - if len(refusals) > 0 { - return nil, fmt.Errorf("nothing was sent. %d machine(s) could not be resolved:\n\n%s", - len(refusals), strings.Join(refusals, "\n\n")) - } - - server, err := connectLink(ctx, nil, nil, nil) - if err != nil { - return nil, err - } - defer server.Close() - - // And every assignment on those machines its membership (novox/hq ADR 0160): composed from the - // same records the bus's accounts are, so what a runtime serves and what its account may are one - // composition. **Issued before the declarations** (novox/hq issue 249): the runtime the - // membership is for arrives with the declaration, and a membership waits for it on the bus; the - // code arriving first was refused its own state until somebody pushed. - if _, err := deliver(ctx, overTheBus{open: open, server: server, signer: ident, indent: " "}, holder, sending); err != nil { - return nil, err - } - sent := make([]string, 0, len(sending)) - for _, s := range sending { - sent = append(sent, s.node) - } - return sent, nil -} - -// issueMemberships publishes the membership of every module on the machines just sent. -// -// Each carries what its module receives and the private network's addresses, from the same -// composition as the declaration it was sent (novox/hq ADR 0167): a provider reads what it is -// given on the bus, and the file written beside it says the same thing. -func issueMemberships(ctx context.Context, open *stores, server *link.Server, sent []readyNode) error { - records, err := open.inventory.BusRecords(ctx) - if err != nil { - return err - } - bus, ok := server.Bus().(link.OverNATS) - if !ok { - return nil - } - // Which machines are root-free now (novox/hq ADR 0259 §8): a channel's verified sender is composed for the - // router only from one, beside a router on one. Judged once per push, by the root-free verb's judgement. - var whys map[string]string - records.RootFree, whys = rootVerdictsOf(judgeRootFree(ctx, newRootReader(ctx, open.inventory, bus.Conn), - records.Nodes, time.Now())) - where := broker.PlacementsOf(records, records.Interchangeable) - // What the judgement kept off is said, with its reason (novox/hq issue 930); the self-check gives it back. - for _, line := range droppedVerifiedSenders(records, where, whys) { - fmt.Println(line) - } - // **Every declared state's bucket, before the memberships that name it** (novox/hq ADR 0201). The - // raise at start asserts them too, but a module registered and assigned since would otherwise have - // its bucket only after the control plane next restarts — found the first time a module declared - // state: its bundle asked for a bucket that did not exist. Idempotent and cheap. - // - // **A failure here is the send's failure** (novox/hq issue 249). It was said and the push stood, - // because the declarations were already away; they are sent after this now — all but the bus's - // own machine, sent before it (deliver) — and a module whose state does not exist is a module - // that fails its start, so they are not sent and the caller tries again rather than reporting the - // rollout done. - buckets, err := open.inventory.DeclaredBuckets(ctx) - if err != nil { - return fmt.Errorf("the modules' state could not be read, so no bucket was asserted: %w", err) - } - if _, err := broker.RaiseBuckets(broker.OnConn(bus.Conn), buckets); err != nil { - return fmt.Errorf("the modules' state could not be asserted on the bus: %w", err) - } - // **And every declared log, for the same reason** (novox/hq ADR 0297 §2): a membership names its - // logs, and a module whose log does not exist fails its first append. - logs, err := open.inventory.DeclaredLogs(ctx) - if err != nil { - return fmt.Errorf("the modules' logs could not be read, so no log was asserted: %w", err) - } - if _, err := broker.RaiseLogs(broker.OnConn(bus.Conn), logs); err != nil { - return fmt.Errorf("the modules' logs could not be asserted on the bus: %w", err) - } - // Every membership is tried, and the first failure named once. - issued := 0 - refused := map[string]error{} - for _, s := range sent { - node := s.node - for _, d := range records.Assigned[node] { - membership := broker.MembershipFor(node, d, where) - membership.Mesh = s.declared.Mesh - for requirement, given := range s.declared.Received[d.Module] { - raw, err := json.Marshal(given) - if err != nil { - return err - } - if membership.Receives == nil { - membership.Receives = map[string]json.RawMessage{} - } - membership.Receives[requirement] = raw - } - body, err := json.Marshal(membership) - if err != nil { - return err - } - if err := bus.PublishMembership(ctx, node, d.Module, body); err != nil { - if refused[node] == nil { - refused[node] = fmt.Errorf("%s: %w", d.Module, err) - } - continue - } - issued++ - } - } - if issued > 0 { - fmt.Printf(" issued %d membership(s)\n", issued) - } - // **And what every other machine hears of a module's stream** (novox/hq ADR 0363 §6): a grant the provider's - // setting changed reaches a consumer on a machine this send does not reach. Its membership is issued again - // as last issued, with only what it hears changed, so nothing else of it moves before its own send. - reheard := reissueHearing(ctx, bus, records, where, sent) - if reheard > 0 { - fmt.Printf(" issued %d membership(s) again for what they hear of a module's stream\n", reheard) - } - // **And says again what it keeps** (novox/hq ADR 0495 clause 2, issue 1790): its word at start is refused until a push - // carries its grant; kept only when it changed, so saying it at every push changes nothing else. - if js := servingBus.Load(); js != nil { - if err := announceOwnStates(ctx, js); err != nil { - fmt.Printf(" states: %v\n", err) - } - } - if len(refused) > 0 { - // Returned, never passed over (novox/hq issue 249): the declarations of the machines they are - // for are not sent, and the rollout that asked is tried again rather than waiting for a push. - return &grantsRefused{nodes: refused} - } - return nil -} - // digestOf is what the mesh compares to answer "has this machine been sent what it should be". // // Over the same bytes that are sent, so the comparison is of the thing itself rather than of @@ -1027,11 +558,6 @@ func wouldSendFrom(ctx context.Context, open *stores, if declared.Epoch, err = open.inventory.SentEpoch(ctx, n.ID); err != nil { return nil, err } - // And the generation it was last sent, for the same reason (novox/hq issue 234): an assignment - // elsewhere in the mesh is not a change of this machine. - if declared.Generation, err = open.inventory.SentGeneration(ctx, n.ID); err != nil { - return nil, err - } body, err := declared.Body() if err != nil { return nil, err @@ -1165,120 +691,10 @@ func seatHolders(ctx context.Context, inv *inventory.Inventory) (map[string]brok return out, nil } -// number gives one send the next sequence for its node (novox/hq 04-ISSUES/107). -// allotting is allot over one inventory, in the shape composeEach takes. -func allotting(ctx context.Context, inv *inventory.Inventory) func(node string) (order, error) { - return func(node string) (order, error) { return allot(ctx, inv, node) } -} - // epochForActs is the lease's gate as a composition asks it; a variable so a test can act under an epoch // without a bus. var epochForActs = func(ctx context.Context) (uint64, error) { return theLease.epoch(ctx) } -// order is what a declaration carries of its writer's order (link/order.go): its sequence, and the -// epoch of the lease it is composed under — zero for a machine that has not said it reads one. -type order struct { - sequence int64 - epoch uint64 - // generation is the assignment generation read before the composition, and readsGeneration whether the - // machine's node-engine said it reads one (novox/hq issue 234); acting is the lease epoch the sender acts - // under, whether or not the machine is sent it — both kept for the record of the send. - generation int64 - readsGeneration bool - acting uint64 -} - -// allot takes the next sequence for a machine — the number its next declaration carries — under the -// lease: a process that may not act takes none, and composes nothing (novox/hq to-be 45 §6). -func allot(ctx context.Context, inv *inventory.Inventory, node string) (order, error) { - epoch, err := epochForActs(ctx) - if err != nil { - return order{}, fmt.Errorf("nothing was composed for %s: %w", node, err) - } - record, err := inv.NodeByName(ctx, node) - if err != nil { - return order{}, err - } - acting := epoch - if epoch > 0 { - reads, err := inv.ReadsEpoch(ctx, record.ID) - if err != nil { - return order{}, err - } - if !reads { - epoch = 0 - } - } - // **The generation is read here, before the composition reads a single assignment** (novox/hq issue - // 234). A generation only grows, so one read before is never newer than the view composed after it: - // the declaration may claim a generation older than its content, never newer — and a claim newer than - // the content is exactly the stale send the machine must be able to refuse. - generation, err := inv.AssignmentGeneration(ctx) - if err != nil { - return order{}, err - } - readsGeneration, err := inv.ReadsGeneration(ctx, record.ID) - if err != nil { - return order{}, err - } - seq, err := inv.NextSequence(ctx, record.ID) - if err != nil { - return order{}, err - } - return order{sequence: seq, epoch: epoch, generation: generation, readsGeneration: readsGeneration, - acting: acting}, nil -} - -// recordSent writes down what a machine was just sent, and returns the digest. -// -// **On a context that outlives the caller's** (novox/hq issue 204). The record is written after the -// declaration is away, so a send that failed is never recorded as current — and a controller being -// replaced mid-send had its context cancelled between the two, so the machine was told and the mesh -// never wrote it down: status read "applied, current" over a machine that had just been sent -// something else. What was sent was sent; the record of it must not depend on the sender living -// another second. Bounded, so a store that is away does not hold a dying process open for ever. -// -// And the build of each module it carried (novox/hq issue 259, ADR 0221), nil when that is not known: -// what tells a machine held back by a policy or a plan from one a push left behind. -func recordSent(ctx context.Context, inv *inventory.Inventory, node string, body []byte, - builds map[string]string, epoch uint64, sent sentRecord) (string, error) { - kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) - defer cancel() - record, err := inv.NodeByName(kept, node) - if err != nil { - return "", err - } - digest := digestOf(body) - // The digest and the generation it carried are written in one transaction (novox/hq issue 234), so the - // would-send is never stamped with a generation the machine was not sent; and the send itself, who sent - // it and from which generation, beside them. A record of the send that cannot be written does not make a - // send that is away look failed: it is said, loudly, and the machine's own record stands. - err = inv.RecordSentWith(kept, record.ID, digest, builds, epoch, sent.toldGeneration, inventory.Send{ - Sequence: sent.sequence, Epoch: int64(sent.epoch), Sender: sent.sender, Generation: sent.generation, - Digest: digest, Modules: sent.modules, Provisioning: sent.plans}) - var unrecorded *inventory.SendNotRecordedError - switch { - case errors.As(err, &unrecorded): - fmt.Fprintf(os.Stderr, "mesh-controller: SEND NOT RECORDED: %s was sent declaration %s (sequence %d), and who "+ - "sent it and from which generation could not be written down, so a refusal of it will name no sender "+ - "(novox/hq issue 234): %v\n", node, short(digest), sent.sequence, unrecorded.Err) - fmt.Printf("%s: sent, and the record of who sent it could NOT be written: %v\n", node, unrecorded.Err) - case err != nil: - return "", err - } - // And what it was, summarised, so the next push can be compared with it (novox/hq ADR 0217). - // Never a reason to fail a send that is already away: a summary that cannot be kept means the - // next comparison has nothing to hold against, which is how every machine starts. - if summary, err := summarize(body); err == nil { - if raw, err := json.Marshal(summary); err == nil { - if err := inv.RecordSentSummary(kept, record.ID, raw); err != nil { - fmt.Fprintf(os.Stderr, "%s: what it was sent is not kept for comparison: %v\n", node, err) - } - } - } - return digest, nil -} - // reportUnheldPushed says what a push's machines lack of the seats their modules depend on // (novox/hq ADR 0207): every line for a machine the push named, since that is the machine somebody // is looking at, and one line per machine otherwise — a list per machine across the mesh is the @@ -1314,134 +730,3 @@ func schemaReach() (int, error) { } return reach, nil } - -// reissueHearing issues again, on every machine not just sent, the membership of each assignment whose -// `consumes` names a module owning a stream, when what it hears changed: read as last issued from the bus, -// its `hears` replaced, and published. One that was never issued is left to its own send. Failures are said -// and passed over: the send that carries the grant is not the consumer's, and its own next send issues it. -func reissueHearing(ctx context.Context, bus link.OverNATS, records broker.Records, where broker.Placements, - sent []readyNode) int { - reached := map[string]bool{} - for _, s := range sent { - reached[s.node] = true - } - issued := 0 - for node, declared := range records.Assigned { - if reached[node] { - continue - } - for _, d := range declared { - if !d.HearsAStream { - continue - } - subject := broker.MembershipSubject(node, d.Module) - last, err := bus.JS.GetLastMsg(broker.AssignmentsStream, subject) - if err != nil { - if !errors.Is(err, nats.ErrMsgNotFound) { - fmt.Printf(" %s on %s: its membership could not be read to issue what it hears again: %v\n", - d.Module, node, err) - } - continue - } - body, changed, err := withHearing(last.Data, broker.HeardIssuedFor(node, d)) - if err != nil { - fmt.Printf(" %s on %s: its membership could not be read to issue what it hears again: %v\n", - d.Module, node, err) - continue - } - if !changed { - continue - } - if err := bus.PublishMembership(ctx, node, d.Module, body); err != nil { - fmt.Printf(" %s on %s: what it hears could not be issued again: %v\n", d.Module, node, err) - continue - } - issued++ - } - } - return issued -} - -// withHearing is a membership as issued with its `hears` replaced, and whether that changed it. Pure. -func withHearing(issued []byte, hears []broker.HeardIssued) ([]byte, bool, error) { - var fields map[string]json.RawMessage - if err := json.Unmarshal(issued, &fields); err != nil { - return nil, false, err - } - var had []broker.HeardIssued - if raw, ok := fields["hears"]; ok { - if err := json.Unmarshal(raw, &had); err != nil { - return nil, false, err - } - } - if slices.Equal(had, hears) { - return issued, false, nil - } - if len(hears) == 0 { - delete(fields, "hears") - } else { - raw, err := json.Marshal(hears) - if err != nil { - return nil, false, err - } - fields["hears"] = raw - } - body, err := json.Marshal(fields) - return body, true, err -} - -func (b overTheBus) declare(ctx context.Context, s readyNode, body []byte) (string, error) { - // The contents the declaration names by digest are in the bucket before it is sent (novox/hq ADR 0486 §1, issue 1696): a node - // that applies it finds every one. A node reading no references has none. - // **Under the node's lock** from the put to the publication, so the sweep of its bucket cannot remove an object between them. - unlock := lockFilesOf(s.node) - err := b.putFiles(ctx, s) - if err != nil { - unlock() - return "", err - } - declared := link.Declare(ctx, b.server.Bus(), b.signer, s.node, body, 15*time.Second) - unlock() - if declared != nil { - return "", declared - } - // After it is away, not before. A digest recorded for something that failed to send would make - // the machine look current for a declaration it never received. - digest, err := recordSent(ctx, b.open.inventory, s.node, body, s.declared.Builds, s.declared.Epoch, - sentRecordOf(ctx, s.declared)) - if err != nil { - return "", err - } - if len(s.declared.Bindings) > 0 { - // And where it bound each consumer of a provision that keeps its data (novox/hq ADR 0232): - // what the next resolution keeps it at. On the same outliving context as the send's record. - kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) - err := b.open.inventory.RecordBindings(kept, s.node, s.declared.Bindings) - cancel() - if err != nil { - return "", fmt.Errorf("%s was sent its declaration, and where its consumers are bound to their "+ - "data could not be recorded: %w", s.node, err) - } - } - if s.declared.BusUsers != "" { - // And the user list it carried, so the next send reads whether it must go first from the - // list alone (novox/hq issue 249). On the same outliving context as the send's record. - kept, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) - err := b.open.inventory.RecordSentBusUsers(kept, s.node, digestOf([]byte(s.declared.BusUsers))) - cancel() - if err != nil { - return "", err - } - // The list sent issued the credential of every module it carried, at the fingerprint each is registered at: - // the grants the assigner's sixth rule reads (novox/hq ADR 0528 clause 2, issue 1394). - if bus, ok := b.server.Bus().(link.OverNATS); ok { - if js, err := jetstream.New(bus.Conn); err == nil { - writing, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) - grantsIssued(writing, b.open.inventory, js, b.indent) - cancel() - } - } - } - fmt.Printf("%ssent %s %d resource(s)\n", b.indent, s.node, len(s.declared.Resources)) - return digest, nil -} diff --git a/cmd/mesh-controller/push_test.go b/cmd/mesh-controller/push_test.go deleted file mode 100644 index 541145cd..00000000 --- a/cmd/mesh-controller/push_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package main - -import ( - "errors" - "strings" - "testing" -) - -// One machine that cannot be worked out is not a reason to leave the mesh unconverged. -// -// A whole-mesh push refused outright the moment any single node failed to resolve, so a module on -// the anchor requiring a provision nobody had assigned a provider for stopped every OTHER machine -// from being sent anything — machines with no relation to the fault, and nothing wrong with them. -// The failure and the punishment were on different machines. -// -// It is the same rule an un-hostable module already follows one level down (a92c11b: one module on -// the wrong machine no longer refuses the whole node), applied one level up. -func TestOneUnresolvableNodeStillLetsTheRestBeSent(t *testing.T) { - sending, refusals := composeEach( - []string{"anchor", "home-server", "laptop"}, numbered(), - func(node string) (sendable, error) { - if node == "anchor" { - return sendable{}, errors.New(`nothing provides "acme-ca", wanted by route-proxy`) - } - return sendable{Resources: []map[string]any{{"id": node + ".thing"}}}, nil - }) - - var told []string - for _, s := range sending { - told = append(told, s.node) - } - if strings.Join(told, ",") != "home-server,laptop" { - t.Errorf("a machine with nothing wrong with it was not sent: %v", told) - } - if len(refusals) != 1 || !strings.Contains(refusals[0], "anchor") || - !strings.Contains(refusals[0], "acme-ca") { - t.Errorf("the machine that could not be worked out was not named with its reason: %v", - refusals) - } -} - -// A machine whose declaration composes to nothing is SENT the empty declaration, not skipped -// (novox/hq issue 127): it may have held something before, and only sending the empty -// declaration tells it to drop what the mesh owned. It is never a refusal. -func TestAnEmptyDeclarationIsSentSoTheNodeDropsWhatItHeld(t *testing.T) { - sending, refusals := composeEach([]string{"spare"}, numbered(), - func(string) (sendable, error) { return sendable{}, nil }) - if len(sending) != 1 || len(refusals) != 0 { - t.Errorf("an empty declaration must be sent, not skipped or refused: %v / %v", sending, refusals) - } -} - -// A push that skipped a machine still ends badly, and says what was sent. -// -// **Skipping is not succeeding.** The mesh is not in the state somebody asked for, so the command -// exits non-zero — but it says how many machines it did reach, because the old message ("nothing -// was sent") was the very claim that had become untrue. -func TestASkippedMachineIsStillAnError(t *testing.T) { - if err := couldNotBeResolved(nil, 3); err != nil { - t.Fatalf("a push that resolved every machine reported a problem: %v", err) - } - - err := couldNotBeResolved([]string{"anchor:\nnothing provides \"acme-ca\""}, 2) - if err == nil { - t.Fatal("a push that could not work out a machine reported success") - } - said := err.Error() - if strings.Contains(said, "nothing was sent") { - t.Errorf("the push says nothing was sent, and it sent two machines: %q", said) - } - for _, want := range []string{"anchor", "acme-ca", "2 other node(s) were"} { - if !strings.Contains(said, want) { - t.Errorf("the refusal does not say %q: %q", want, said) - } - } -} - -// numbered is an allotter for tests: one higher per call, as the inventory's is per machine. -func numbered() func(string) (order, error) { - var n int64 - return func(string) (order, error) { n++; return order{sequence: n}, nil } -} diff --git a/cmd/mesh-controller/readable.go b/cmd/mesh-controller/readable.go index df3179ea..c71d9b21 100644 --- a/cmd/mesh-controller/readable.go +++ b/cmd/mesh-controller/readable.go @@ -111,33 +111,6 @@ type meshStatus struct { // why they could not be read. Pending []pendingStatus `json:"pending,omitempty"` PendingUnread string `json:"pendingUnread,omitempty"` - // LastSent is every machine's last recorded send — who sent it, under which lease epoch, from which - // assignment generation, naming which modules — and what the machine answered of it (novox/hq issue 485). - // A machine the mesh has recorded no send to is absent. LastSentUnread says why it could not be read. - LastSent []machineLastSent `json:"lastSent"` - LastSentUnread string `json:"lastSentUnread,omitempty"` -} - -// machineLastSent is one machine's last send as status says it. -type machineLastSent struct { - Node string `json:"node"` - Sequence int64 `json:"sequence"` - SentAt time.Time `json:"sentAt"` - Sender string `json:"sender"` - Epoch int64 `json:"epoch"` - Generation int64 `json:"generation"` - Digest string `json:"digest"` - Modules []string `json:"modules"` - // Answer is what the machine said of it: applied, failed or refused, as its report has it; empty when it - // has not reported on it. Refused is its words when it refused it whole, Failed how many resources failed. - Answer string `json:"answer"` - Refused string `json:"refused,omitempty"` - Failed int `json:"failed,omitempty"` - AnsweredAt *time.Time `json:"answeredAt,omitempty"` - // RefusedForGeneration is when the machine refused it for coming from an older assignment generation than - // it applied, and AppliedGeneration that generation; absent when it did not. - RefusedForGeneration *time.Time `json:"refusedForGeneration,omitempty"` - AppliedGeneration int64 `json:"appliedGeneration,omitempty"` } // pendingStatus is one pending assignment as status says it. @@ -297,7 +270,6 @@ func statusAsJSON(asked answers) ([]byte, error) { out.Pending = append(out.Pending, pendingStatus{Node: p.Node, Module: p.Module, State: p.State, Build: p.Build, Repository: p.Repository, Path: p.Path, Since: p.Since, Settled: p.Settled, Note: p.Note}) } - out.LastSent, out.LastSentUnread = lastSentStatus(asked.lastSent), asked.lastSentUnread for name := range asked.refused { out.Unresolved = append(out.Unresolved, machineUnresolved{ Node: name, Problem: asked.refused[name]}) @@ -354,20 +326,3 @@ func say(value any) error { fmt.Println(string(body)) return nil } - -// lastSentStatus is every machine's last send as status --json says it, by machine name (novox/hq issue 485). -func lastSentStatus(sends map[string]inventory.Send) []machineLastSent { - out := []machineLastSent{} - for _, s := range sends { - modules := s.Modules - if modules == nil { - modules = []string{} - } - out = append(out, machineLastSent{Node: s.NodeName, Sequence: s.Sequence, SentAt: s.SentAt, Sender: s.Sender, - Epoch: s.Epoch, Generation: s.Generation, Digest: s.Digest, Modules: modules, Answer: s.Answer.Outcome, - Refused: s.Answer.Refused, Failed: s.Answer.Failed, AnsweredAt: s.Answer.At, - RefusedForGeneration: s.RefusedAt, AppliedGeneration: s.RefusedApplied}) - } - sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node }) - return out -} diff --git a/cmd/mesh-controller/reissue_hearing_test.go b/cmd/mesh-controller/reissue_hearing_test.go deleted file mode 100644 index 9375c583..00000000 --- a/cmd/mesh-controller/reissue_hearing_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import ( - "encoding/json" - "testing" - - "github.com/novox/mesh-controller/internal/broker" -) - -// A membership issued again for what it hears changes nothing else of it (novox/hq ADR 0363 §6). -func TestAMembershipIssuedAgainChangesOnlyWhatItHears(t *testing.T) { - issued := []byte(`{"node":"ace","module":"reader","receives":{"x":[1]},"emits":"mesh.mod.reader.event.{event}"}`) - hears := []broker.HeardIssued{{Stream: "MAIL", Consumer: "ace_reader"}} - body, changed, err := withHearing(issued, hears) - if err != nil || !changed { - t.Fatalf("changed %v, %v", changed, err) - } - var got map[string]any - if err := json.Unmarshal(body, &got); err != nil { - t.Fatal(err) - } - if got["receives"] == nil || got["emits"] != "mesh.mod.reader.event.{event}" || got["hears"] == nil { - t.Fatalf("issued again as %s", body) - } - if _, again, _ := withHearing(body, hears); again { - t.Fatal("the same hearing issued again counts as a change") - } - gone, changed, err := withHearing(body, nil) - if err != nil || !changed { - t.Fatalf("a withdrawal: %v, %v", changed, err) - } - got = nil - if err := json.Unmarshal(gone, &got); err != nil { - t.Fatal(err) - } - if _, still := got["hears"]; still { - t.Fatalf("a withdrawn hearing is still issued: %s", gone) - } -} diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index 0eaedffb..ddf3c875 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -99,7 +99,7 @@ var removedVerbs = map[string]bool{"build": true, "rebuild": true, "forget-check // Release 6 (ADR 0539 §5): a delivery is each machine following its wants, so the verbs that planned, started, stopped, // retried or closed one are gone. "plans": true, "plan": true, "push": true, "deliver": true, "delivery-stop": true, "delivery-backups-again": true, - "delivery-walks": true, "delivery-order": true, "upgrade": true} + "delivery-walks": true, "delivery-order": true} // controllerVerb is this binary's own definition of a verb: what it runs is what it declares, so the // arguments are checked against the table compiled beside argvFor, not a row a newer or older build @@ -618,6 +618,22 @@ func (a *verbArguments) commandLine() ([]string, error) { argv = append(argv, "--retired") } return argv, nil + case "upgrade": + m, policy := str("module"), str("policy") + if m == "" { + if policy != "" { + return nil, errors.New("upgrade takes a policy only for a module") + } + return []string{"upgrade"}, nil + } + argv := []string{"upgrade", m} + if policy != "" { + argv = append(argv, policy) + if w := str("why"); w != "" { + argv = append(argv, "--why", w) + } + } + return argv, nil case "bus": if !on("upgrade") { return []string{"bus"}, nil diff --git a/cmd/mesh-controller/send_stub.go b/cmd/mesh-controller/send_stub.go new file mode 100644 index 00000000..b263605e --- /dev/null +++ b/cmd/mesh-controller/send_stub.go @@ -0,0 +1,23 @@ +package main + +import "context" + +// What was once a send is no longer one (ADR 0539 §5, §8): the controller writes the compositions that changed instead +// (composeModules), and each machine follows its wants. The calls below stand where a send was asked until the part of the +// controller that asked it writes compositions in its place; the subtask that replaces the last caller deletes this file. + +// sendTo sends nothing; see sendToEach. +func sendTo(ctx context.Context, open *stores, names []string) error { + _, err := sendToEach(ctx, open, names) + return err +} + +// sendToEach answers the machines it was asked for, and sends them nothing. +func sendToEach(ctx context.Context, open *stores, names []string) ([]string, error) { + return names, nil +} + +// holdNodes held machines from composing to sending; nothing is sent, so it holds nothing. +func holdNodes(ctx context.Context, open *stores, names []string) (context.Context, func(), error) { + return ctx, func() {}, nil +} diff --git a/cmd/mesh-controller/sendable_integration_test.go b/cmd/mesh-controller/sendable_integration_test.go index ec51664d..ff78394f 100644 --- a/cmd/mesh-controller/sendable_integration_test.go +++ b/cmd/mesh-controller/sendable_integration_test.go @@ -89,16 +89,6 @@ func TestAnAdoptedDeclarationCarriesItsModeAndWhatWasTaken(t *testing.T) { t.Fatal("the digest the mesh compares is not of the declaration push sends") } - // plan --json prints that same envelope. - printed := stdoutOf(t, func() error { return planCommand(ctx, []string{"anchor", "--json"}) }) - var compact bytes.Buffer - if err := json.Compact(&compact, []byte(printed)); err != nil { - t.Fatalf("plan --json is not JSON: %v\n%s", err, printed) - } - if digestOf(compact.Bytes()) != digestOf(body) { - t.Fatalf("plan --json shows something other than what push sends:\n%s", printed) - } - // Taking the module moves its resources out of untaken. if err := open.inventory.Take(ctx, "anchor", "hello-web"); err != nil { t.Fatal(err) diff --git a/cmd/mesh-controller/sent_required_by_integration_test.go b/cmd/mesh-controller/sent_required_by_integration_test.go deleted file mode 100644 index 5bc15604..00000000 --- a/cmd/mesh-controller/sent_required_by_integration_test.go +++ /dev/null @@ -1,101 +0,0 @@ -//go:build integration - -package main - -import ( - "encoding/json" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -func TestAModulePulledInIsHandedOnAMachineThatCanTakeItAndMirrored(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - client := aClientModule() - moved, kept := aBot("moved-bot", true), aBot("kept-bot", false) - moved.Provides = []catalogue.Offer{{Name: "moved-bot-api"}} - kept.Provides = []catalogue.Offer{{Name: "kept-bot-api"}} - desk := catalogue.Manifest{Module: "desk", Version: "1", Requires: []string{"moved-bot-api", "kept-bot-api"}} - for _, m := range []catalogue.Manifest{client, moved, kept, desk} { - register(t, open, m) - } - for _, m := range []string{client.Module, desk.Module} { - if _, err := assign(ctx, open, "laptop", m); err != nil { - t.Fatal(err) - } - } - if err := issueProvisionerClient(ctx, inv, client, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil { - t.Fatal(err) - } - if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: link.HandedContract, - SaidAt: time.Now()}); err != nil { - t.Fatal(err) - } - - plan, settings, err := resolveNode(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - if plan.RequiredBy("moved-bot") != "desk" || plan.RequiredBy("kept-bot") != "desk" { - t.Fatalf("the bots are on laptop because %v", plan.Because) - } - declared, err := declarationFor(ctx, open, "laptop", plan, settings) - if err != nil { - t.Fatal(err) - } - var movedToken, keptToken, movedConf map[string]any - for _, r := range declared.Resources { - switch r["id"] { - case "moved-bot.needs-token": - movedToken = r - case "kept-bot.needs-token": - keptToken = r - case "moved-bot.conf": - movedConf = r - } - } - if movedToken == nil || movedToken["sealed"] != nil || movedToken["owner-at-runtime"] != "owned" { - t.Fatalf("the pulled-in moved module's secret file is sent as %v: placed, with no value", movedToken) - } - if movedConf == nil || movedConf["content"] != nil || movedConf["owner-at-runtime"] != "owned" { - t.Fatalf("the pulled-in moved module's templated file is sent as %v: placed, with no text", movedConf) - } - if keptToken == nil || keptToken["sealed"] == nil { - t.Fatalf("the pulled-in module not moved has its secret sent as %v: sealed, as before", keptToken) - } - if _, held, err := inv.ModuleSecretIfIssued(ctx, "laptop", "moved-bot", "token"); err != nil || held { - t.Fatalf("the send made the pulled-in handed module's secret in the controller's store (held %v, %v)", held, err) - } - if _, has := declared.plans["kept-bot"]; has || len(declared.plans) != 1 { - t.Fatalf("the send hands %v", declared.plans) - } - - record, err := inv.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, inventory.Send{Digest: "first", - Provisioning: sentRecordOf(ctx, declared).plans}); err != nil { - t.Fatal(err) - } - inv.SetProvisioningPlans(inv.SentProvisioningPlans) - mirror := inv.PlanValuesForTest(t) - var value inventory.AssignmentMirrored - if err := json.Unmarshal(mirror["laptop.moved-bot"], &value); err != nil { - t.Fatalf("no value for the pulled-in module: %s (%v)", mirror["laptop.moved-bot"], err) - } - if value.RequiredBy != "desk" || !value.Assigned.IsZero() || value.Provisioning == nil || - len(value.Provisioning.Files) != 1 || value.Provisioning.Files[0].Path != "/etc/moved-bot/bot.conf" || - len(value.Provisioning.Secrets) != 1 || value.Provisioning.Secrets[0].Name != "token" { - t.Fatalf("the pulled-in module is mirrored as %s", mirror["laptop.moved-bot"]) - } - if _, has := mirror["laptop.kept-bot"]; has { - t.Fatalf("a pulled-in module not handed is mirrored: %s", mirror["laptop.kept-bot"]) - } -} diff --git a/cmd/mesh-controller/sent_required_by_test.go b/cmd/mesh-controller/sent_required_by_test.go deleted file mode 100644 index eb5a1113..00000000 --- a/cmd/mesh-controller/sent_required_by_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package main - -import ( - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// A send keeps, beside each handed module's plan, what requires a module only pulled in (issue 765), so the -// assignments mirror gives that plan a value of its own; an assigned module's plan is kept as the bytes it was. -func TestASendKeepsWhatRequiresAPulledInModule(t *testing.T) { - plan := catalogue.Plan{Templates: catalogue.PlanTemplates{Source: "http://store:5000/v2/notify/files", - Digest: "sha256:" + strings.Repeat("d", 64)}, - Files: []catalogue.PlanFile{{ID: "rc", Path: "/home/op/rc", Template: "rc", OwnerAtRuntime: "owned"}}} - r := sentRecordOf(t.Context(), sendable{plans: map[string]catalogue.Plan{"dunst": plan, "notify": plan}, - requiredBy: map[string]string{"notify": "desk"}}) - kept := string(r.plans) - want := `{"dunst":{"templates":{"source":"http://store:5000/v2/notify/files","digest":"sha256:` + - strings.Repeat("d", 64) + `"},"files":[{"id":"rc","path":"/home/op/rc","template":"rc","owner-at-runtime":"owned"}]},` + - `"notify":{"templates":{"source":"http://store:5000/v2/notify/files","digest":"sha256:` + strings.Repeat("d", 64) + - `"},"files":[{"id":"rc","path":"/home/op/rc","template":"rc","owner-at-runtime":"owned"}],"required-by":"desk"}}` - if kept != want { - t.Fatalf("the send keeps\n%s\nwant\n%s", kept, want) - } -} - -// **A module only pulled in by another is handed on a machine that can take it** (issue 765): on a contract-11 -// node running a client, a moved module that nothing assigned there, required by one that is, has its files and its -// own secret handed like an assigned one's, no secret of it made or sealed by the controller, and its plan mirrored -// under its own key, saying what requires it. A module not moved, pulled in the same way, is sent as before. diff --git a/cmd/mesh-controller/status.go b/cmd/mesh-controller/status.go index 89504e45..55af4a6f 100644 --- a/cmd/mesh-controller/status.go +++ b/cmd/mesh-controller/status.go @@ -338,7 +338,6 @@ func printStatus(asked answers) error { // What each machine was last sent and by whom, one line each (novox/hq issue 485). Not a fault, so it does not // break "all well"; a record that could not be read does. - printLastSends(nodes, asked.lastSent, asked.lastSentUnread) if adopted := adoptedNodes(nodes); len(adopted) > 0 { // Said, because nothing forces the flip: a node left adopted is visible here rather than @@ -474,12 +473,6 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) { out.pendingUnread = err.Error() err = nil } - // And what each machine was last sent, by whom (novox/hq issue 485): the record of sends is read through - // the controller's verbs alone, the store's reader being shut out of its tables. Unreadable is said. - if out.lastSent, err = inv.LastSends(ctx); err != nil { - out.lastSentUnread = err.Error() - err = nil - } // And how many repairs were done by hand this week (novox/hq to-be 45 §7) — where there is a bus // to read the log from; a process with none has no log to count. if _, onBus := broker.BusAddress(); onBus == nil { @@ -604,8 +597,7 @@ func (a answers) well() bool { return len(a.wrong) == 0 && len(a.quiet) == 0 && len(a.behind) == 0 && len(a.waiting) == 0 && len(a.refused) == 0 && a.network == "" && len(a.untaken) == 0 && len(a.leftOut) == 0 && len(a.filtered) == 0 && len(a.unheld) == 0 && len(a.overflowing) == 0 && - len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending) && - a.lastSentUnread == "" + len(a.conditions) == 0 && a.conditionsUnread == "" && a.pendingUnread == "" && !pendingOpen(a.pending) } // pendingOpen is whether any pending assignment is still to be made. One that ended without being made is diff --git a/cmd/mesh-controller/status_summary.go b/cmd/mesh-controller/status_summary.go index c52f3c41..40ebc04e 100644 --- a/cmd/mesh-controller/status_summary.go +++ b/cmd/mesh-controller/status_summary.go @@ -196,11 +196,6 @@ func (l nudgingListener) Heard(ctx context.Context, report link.Report) (bool, e if report.Ordered() { link.StaleRefusals.Lifetime(report.Node, report.RefusedOlder, now) } - // A send refused for the generation it was composed from is kept on the send it refused, so S20 names - // its sender (novox/hq issue 234). - if report.OlderGeneration != nil { - heardGenerationRefusal(ctx, l.Enrolment.Inventory, report) - } // What the machine's witnesses put back and stand by (novox/hq ADR 0236): read by the gate and its // probe. Only from an account of the machine — not a word that a declaration was set aside, nor a rekey. if report.Superseded == "" && report.Rekey == nil && report.Node != "" { diff --git a/cmd/mesh-controller/unbound_test.go b/cmd/mesh-controller/unbound_test.go index 0114131b..a1638180 100644 --- a/cmd/mesh-controller/unbound_test.go +++ b/cmd/mesh-controller/unbound_test.go @@ -34,9 +34,6 @@ func TestPlanAndNodeListAnOptionalRequirementLeftUnbound(t *testing.T) { } } const said = "unbound influxdb-api, optional for grafana" - if shown := printed(t, func() error { return planCommand(ctx, []string{"laptop"}) }); !strings.Contains(shown, said) { - t.Errorf("plan laptop does not say influxdb-api is unbound:\n%s", shown) - } if shown := printed(t, func() error { return showNode(ctx, open, "laptop") }); !strings.Contains(shown, said) { t.Errorf("node show laptop does not say influxdb-api is unbound:\n%s", shown) } diff --git a/cmd/mesh-controller/upgrades.go b/cmd/mesh-controller/upgrades.go index fe2a4e39..1018cd91 100644 --- a/cmd/mesh-controller/upgrades.go +++ b/cmd/mesh-controller/upgrades.go @@ -3,6 +3,7 @@ package main import ( "context" "errors" + "flag" "fmt" "github.com/novox/mesh-controller/internal/gosource" "regexp" @@ -127,7 +128,7 @@ func (f following) Announceable(ctx context.Context) ([]link.Announcement, error // notNow marks a store that could not be read right now, or plans another controller is working (issue 921), so // the announcement is held rather than lost; anything else is returned as it was. func notNow(err error) error { - if inventory.Unreachable(err) || errors.Is(err, inventory.ErrPlansBusy) { + if inventory.Unreachable(err) { return fmt.Errorf("%w: %w", link.ErrTryAgain, err) } return err @@ -147,8 +148,9 @@ func unsaved(err error) error { // becomes one walk when its merge window closes: hearMerge, and cutBatchesHeld in batches.go. func (f following) SourceMoved(ctx context.Context, m link.SourceMoved) error { // A merge plans nothing here any more: mesh-delivery merges on the Approve, the indexer writes the fingerprints and - // the builder builds them (ADR 0539 §5, §17). The controller holds no batch for it. - return nil + // the builder builds them (ADR 0539 §5, §17). The controller keeps the announcement, for the build of a merge that + // moves a module between repositories to be shown the operator's Approve (ADR 0420 §7). + return keepForgeMerge(ctx, m) } // movesOfMerge is what one merge moves, acted on: every module recorded as built from that repository and @@ -587,26 +589,6 @@ func packagesSaid(read []inventory.ReadRepository, m link.SourceMoved) bool { return false } -// lookedAt is when a merge was last acted on for a module that packages another repository's source: its -// newest build, or the newest plan that built it or is still building it, whichever is later. A build asked -// after a merge clones that repository with the merge in it, so an older merge is history for it; a plan -// that closed without building it looked at nothing. -func lookedAt(name string, read []inventory.ReadRepository, plans []inventory.Plan) time.Time { - var at time.Time - for _, r := range read { - if r.Looked.After(at) { - at = r.Looked - } - } - for _, p := range plans { - s, in := p.Modules[name] - if in && (p.Open() || (s != nil && s.State == "built")) && p.Created.After(at) { - at = p.Created - } - } - return at -} - // readForPlanning is what each module's build read, as a change is mapped onto it — for a merge's reach, a pull request's // check and the facts alike (novox/hq ADR 0238), so they cannot disagree. No plan overtakes it any more (ADR 0539 §17). func readForPlanning(ctx context.Context, inv *inventory.Inventory) (map[string][]inventory.ReadRepository, error) { @@ -936,3 +918,111 @@ func reachableFrom(moved []string, edges []inventory.Edge) []string { // edgeChangeOrder is the edge kind a change order draws. const edgeChangeOrder = "change-order" + +// upgradeCommand says what should happen when a module's current version moves (ADR 0162 §3, ADR +// 0235): every module's policy and where it comes from; one module's; or a person's choice for one. +func upgradeCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("upgrade", flag.ContinueOnError) + together := set.Bool("together", false, + "send every machine running it at once, instead of one machine first") + why := set.String("why", "", "why this choice — required for record; kept and said with the policy") + positionals, err := parseAround(set, args) + if err != nil { + return err + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + inv := open.inventory + + if len(positionals) == 0 { + all, err := inv.Upgrades(ctx) + if err != nil { + return err + } + names := make([]string, 0, len(all)) + for n := range all { + names = append(names, n) + } + sort.Strings(names) + for _, n := range names { + u := all[n] + line := fmt.Sprintf("%-28s %-9s %s", n, u.Policy(), u.From) + if u.Why != "" { + line += ": " + u.Why + } + fmt.Println(line) + } + return nil + } + module := positionals[0] + if len(positionals) == 1 { + decision, err := inv.UpgradeOf(ctx, module) + if err != nil { + return err + } + fmt.Println(sayUpgrade(module, decision)) + return nil + } + + decision := inventory.Upgrade{Why: strings.TrimSpace(*why), By: link.Caller()} + switch positionals[1] { + case "roll-out", "roll": + decision.RollOut, decision.Together = true, *together + case "record": + if *together { + // Refused rather than ignored: --together only means anything for a roll-out, and + // accepting it here would store a preference that never applies and looks like it does. + return errors.New("`--together` says how to roll out, so it cannot be given with " + + "`record`, which is the choice not to") + } + if decision.Why == "" { + return fmt.Errorf("holding %s back from every merge is a choice a person reads later: --why "+ + "(novox/hq ADR 0236). Nothing was changed", module) + } + case "default": + if err := inv.ClearUpgradeOf(ctx, module); err != nil { + return err + } + decision, err := inv.UpgradeOf(ctx, module) + if err != nil { + return err + } + fmt.Println(sayUpgrade(module, decision)) + return nil + default: + return fmt.Errorf("upgrade roll-out|record|default — not %q", positionals[1]) + } + if err := inv.SetUpgradeOf(ctx, module, decision); err != nil { + return err + } + decision, err = inv.UpgradeOf(ctx, module) + if err != nil { + return err + } + fmt.Println(sayUpgrade(module, decision)) + return nil +} + +func sayUpgrade(module string, u inventory.Upgrade) string { + from := " (" + u.From + if u.By != "" { + from += ", " + u.By + } + if u.Why != "" { + from += ": " + u.Why + } + from += ")" + if !u.RollOut { + return fmt.Sprintf("when %s moves, the mesh records it and the machines running it are "+ + "reported as behind until a person pushes them%s", module, from) + } + if u.Together { + return fmt.Sprintf("when %s moves, every machine running it is sent the new version "+ + "together%s", module, from) + } + return fmt.Sprintf("when %s moves, one machine running it is sent the new version first and judged at "+ + "the gate; the rest follow once it passes, and a build that fails is put back there%s", module, from) +} diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index ec1325a7..9597148d 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -146,25 +146,6 @@ type machineFacts struct { // asleep says the machine said it would be away and has not said it is back (ADR 0211). func (m machineFacts) asleep() bool { return m.power.Away() } -type planFacts struct { - id, repository, commit string - tier, tiers int - entered time.Time - bound time.Duration - waiting string - bus bool -} - -// waitFacts is one walk waiting for its delivery's word: since its merge opened it. -type waitFacts struct { - id, repository, commit, awaits string - since time.Time - // modules are the modules the walk moves, as a person names the delivery. - modules []string - // merges are the merges it answers (novox/hq ADR 0276), as the deliveries to read are found. - merges []inventory.PlanMerge -} - type loopFacts struct { took time.Time pending uint64 @@ -415,14 +396,7 @@ func (w *watchdogs) gatherMachines(ctx context.Context, inv *inventory.Inventory if err != nil { return nil, fmt.Errorf("where the node tools run cannot be read: %w", err) } - applies, err := inv.Durations(ctx, inventory.DurationApply, now.Add(-7*24*time.Hour)) - if err != nil { - return nil, fmt.Errorf("the measured applies cannot be read: %w", err) - } lastApply := map[string]time.Duration{} - for _, d := range applies { // oldest first: the last one read is the newest - lastApply[d.Node] = d.Took - } var out []machineFacts anyPast := false for _, n := range nodes { diff --git a/internal/chain/chain_test.go b/internal/chain/chain_test.go index 42c216a0..68698bbc 100644 --- a/internal/chain/chain_test.go +++ b/internal/chain/chain_test.go @@ -49,23 +49,23 @@ func TestABuildIsKeptWholeAndReadBackWithItsReuseKeyAndWhatItPublished(t *testin func TestAWriteAgainstAMovedRevisionIsRefusedAndNothingIsLaidOver(t *testing.T) { ctx := context.Background() c := InMemory(nil) - first, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"}) + first, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", RequiredBy: "c1"}) if err != nil { t.Fatal(err) } if _, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes"}); !errors.Is(err, ErrExists) { t.Fatalf("a create of a key that holds a value answered %v", err) } - second, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c2"}, first) + second, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", RequiredBy: "c2"}, first) if err != nil { t.Fatal(err) } // A writer that read the first revision writes now. - if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "stale"}, first); !errors.Is(err, ErrMoved) { + if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", RequiredBy: "stale"}, first); !errors.Is(err, ErrMoved) { t.Fatalf("a write at a moved revision answered %v", err) } got, _, _ := c.Assignments.Get(ctx, "anchor.notes") - if got.Value.Commit != "c2" || got.Revision != second { + if got.Value.RequiredBy != "c2" || got.Revision != second { t.Fatalf("the key that moved was laid over: %+v at %d", got.Value, got.Revision) } } @@ -91,8 +91,8 @@ func TestNoKeyOfTheChainIsEverRemoved(t *testing.T) { } ctx := context.Background() c := InMemory(nil) - rev, _ := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"}) - if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Commit: "c1"}, rev); err != nil { + rev, _ := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", RequiredBy: "c1"}) + if _, err := c.Assignments.Update(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", RequiredBy: "c1"}, rev); err != nil { t.Fatal(err) } keys, err := c.Assignments.Keys(ctx) @@ -190,31 +190,36 @@ func TestARolloutNamesEveryAssignmentItChangedWithBuildBeforeAndAfter(t *testing } } -// **An assignment names its build, artifact or package, commit and rollout**, and keeps the fields its readers read -// under the names they read them by (ADR 0408 §3). -func TestAnAssignmentNamesBuildArtifactCommitAndRollout(t *testing.T) { +// **An assignment is placement only** (ADR 0539 §12): the node, the module, since when, what requires it and its +// provisioning plan; no build, artifact, package, commit, rollout or time since is written in it, and a value of before +// release 6 that names them is read with them ignored. +func TestAnAssignmentIsPlacementOnly(t *testing.T) { a := Assignment{Node: "anchor", Module: "notes", Assigned: at(3), RequiredBy: "wiki", - Provisioning: json.RawMessage(`{"plan":1}`), Build: "b2", Artifact: "sha256:bb", Commit: "c2", Rollout: "r1", Since: at(4)} + Provisioning: json.RawMessage(`{"plan":1}`)} raw, _ := json.Marshal(a) for _, want := range []string{`"node":"anchor"`, `"module":"notes"`, `"assigned":"2026-10-18T03:00:00Z"`, - `"required-by":"wiki"`, `"provisioning":{"plan":1}`, `"build":"b2"`, `"artifact":"sha256:bb"`, `"commit":"c2"`, - `"rollout":"r1"`, `"since":"2026-10-18T04:00:00Z"`} { + `"required-by":"wiki"`, `"provisioning":{"plan":1}`} { if !strings.Contains(string(raw), want) { t.Errorf("the assignment's value lacks %s: %s", want, raw) } } + for _, not := range []string{`"build"`, `"artifact"`, `"package"`, `"commit"`, `"rollout"`, `"since"`, `"pulled-in"`, `"sent-unknown"`} { + if strings.Contains(string(raw), not) { + t.Errorf("the assignment's value names %s: %s", not, raw) + } + } if a.Key() != "anchor.notes" { t.Errorf("key %s", a.Key()) } - // An assignment of today carries none of the new fields, and says none: a reader of today reads the bytes it did. - raw, _ = json.Marshal(Assignment{Node: "anchor", Module: "notes", Assigned: at(3)}) - if string(raw) != `{"node":"anchor","module":"notes","assigned":"2026-10-18T03:00:00Z"}` { - t.Errorf("an assignment without a chain is %s", raw) + // A value written before release 6 is read, its chain fields ignored, and written back without them. + var old Assignment + if err := json.Unmarshal([]byte(`{"node":"anchor","module":"notes","assigned":"2026-10-18T03:00:00Z","build":"b2","commit":"c2",`+ + `"rollout":"r1","since":"2026-10-18T04:00:00Z","pulled-in":true}`), &old); err != nil { + t.Fatal(err) } - // A package in place of an artifact. - raw, _ = json.Marshal(Assignment{Node: "a", Module: "m", Package: &Package{Name: "n", Version: "1"}}) - if !strings.Contains(string(raw), `"package":{"name":"n","version":"1"}`) { - t.Errorf("the package is %s", raw) + raw, _ = json.Marshal(old) + if string(raw) != `{"node":"anchor","module":"notes","assigned":"2026-10-18T03:00:00Z"}` { + t.Errorf("an old value written again is %s", raw) } } @@ -341,7 +346,7 @@ func TestAnEmptyBucketIsReadAsEmpty(t *testing.T) { func TestAnEndedAssignmentIsNoAssignmentAndStaysInTheBucket(t *testing.T) { ctx := context.Background() c := InMemory(nil) - rev, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Assigned: at(1), Commit: "c1"}) + rev, err := c.Assignments.Create(ctx, "anchor.notes", Assignment{Node: "anchor", Module: "notes", Assigned: at(1), RequiredBy: "c1"}) if err != nil { t.Fatal(err) } @@ -355,7 +360,7 @@ func TestAnEndedAssignmentIsNoAssignmentAndStaysInTheBucket(t *testing.T) { t.Fatal(err) } got, found, _ := c.Assignments.Get(ctx, "anchor.notes") - if !found || got.Value.Stands() || got.Value.Commit != "c1" || got.Value.Ended.IsZero() { + if !found || got.Value.Stands() || got.Value.RequiredBy != "c1" || got.Value.Ended.IsZero() { t.Fatalf("the ended value is %+v (found %v)", got.Value, found) } if history, _ := c.Assignments.History(ctx, "anchor.notes"); len(history) != 2 || !history[0].Value.Stands() { diff --git a/internal/chain/pending_test.go b/internal/chain/pending_test.go index 655aae40..c0e73f9d 100644 --- a/internal/chain/pending_test.go +++ b/internal/chain/pending_test.go @@ -61,27 +61,3 @@ func TestTheTransitionsOfAPendingAssignment(t *testing.T) { t.Error("a claim moves a wait to applying and back") } } - -// **A module only pulled in is a standing value marked pulled-in, never a state, and a hand send makes a key say its commit -// unknown with a time** (issue 1109): every reader treats a non-empty state as no assignment, so a pulled-in key must have -// none; and unknown is never written in `commit`. -func TestAPulledInValueStandsAndAHandSendIsATimeNotACommit(t *testing.T) { - at := time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC) - pulled := Assignment{Node: "anchor", Module: "mesh-wireguard", PulledIn: true, Commit: "c1", Build: "b1", Since: at} - if !pulled.Stands() || pulled.State != "" { - t.Fatalf("a pulled-in value does not stand: %+v", pulled) - } - raw, _ := json.Marshal(pulled) - if !strings.Contains(string(raw), `"pulled-in":true`) || strings.Contains(string(raw), `"state"`) || strings.Contains(string(raw), `"assigned"`) { - t.Fatalf("the pulled-in value is %s", raw) - } - unknown := Assignment{Node: "anchor", Module: "notes", Assigned: at, SentUnknown: at} - raw, _ = json.Marshal(unknown) - if !strings.Contains(string(raw), `"sent-unknown":"2026-10-18T03:00:00Z"`) || strings.Contains(string(raw), `"commit"`) || !unknown.Stands() { - t.Fatalf("the hand-sent value is %s", raw) - } - plain, _ := json.Marshal(Assignment{Node: "anchor", Module: "notes", Assigned: at}) - if strings.Contains(string(plain), "pulled-in") || strings.Contains(string(plain), "sent-unknown") { - t.Fatalf("a value that is neither says so: %s", plain) - } -} diff --git a/internal/chain/records.go b/internal/chain/records.go index 2738abd2..dcb33282 100644 --- a/internal/chain/records.go +++ b/internal/chain/records.go @@ -175,9 +175,10 @@ type Rollout struct { Links } -// Assignment is one module on one node, kept under `.`. The first five fields are the ones its readers -// have always read, with the names they have always had (novox/hq ADR 0408 §3); the rest name the chain: the build, the -// artifact (by digest) or the package it runs, its commit, the rollout that put it there and since when. +// Assignment is one module on one node, kept under `.`: placement only (novox/hq ADR 0539 §12). The first +// five fields are the ones its readers have always read, with the names they have always had (novox/hq ADR 0408 §3); +// no build, commit or rollout is written in it any more, and a value written before release 6 that names them is read +// with them ignored. type Assignment struct { Node string `json:"node"` Module string `json:"module"` @@ -185,13 +186,6 @@ type Assignment struct { RequiredBy string `json:"required-by,omitempty"` Provisioning json.RawMessage `json:"provisioning,omitempty"` - Build string `json:"build,omitempty"` - Artifact string `json:"artifact,omitempty"` - Package *Package `json:"package,omitempty"` - Commit string `json:"commit,omitempty"` - Rollout string `json:"rollout,omitempty"` - Since time.Time `json:"since,omitzero"` - // Ended is when the assignment ended (an unassignment): absent while it stands. **No key of the chain is removed** // (ADR 0408 §4), so an unassigned module keeps its key, saying so; every reader treats a value with Ended, or with a // State, as no assignment. State is StatePending while the key holds an assignment waiting for its build, or one that @@ -201,14 +195,6 @@ type Assignment struct { // Pending is the assignment's wait for its build (novox/hq ADR 0261), kept in the value of the key it waits for and, when // it is made, carried into the value that stands: the record of what became of the wait. Pending *Pending `json:"pending,omitempty"` - - // PulledIn marks a standing value of a module that is on the machine only because another requires it, or because a - // send carried it: no time assigned, but a key all the same, so what the machine was sent of it is kept and read - // (issue 1109). Never a state: every reader treats a non-empty state as no assignment, and would take it off the machine. - PulledIn bool `json:"pulled-in,omitempty"` - // SentUnknown is when a send by hand made what the machine runs unknown (a declaration sent with no builds known): the - // commit is cleared with it. Absent normally; a send that names its builds clears it. - SentUnknown time.Time `json:"sent-unknown,omitzero"` } // StatePending is the State of a value that holds no assignment but a pending one. diff --git a/internal/inventory/batches.go b/internal/inventory/batches.go deleted file mode 100644 index 8d5cf20e..00000000 --- a/internal/inventory/batches.go +++ /dev/null @@ -1,138 +0,0 @@ -package inventory - -import ( - "context" - "encoding/json" - "strings" - "time" -) - -// The merges a batch or walk answers (novox/hq ADR 0276): every merge the controller heard into a branch a -// module follows is kept here once, with the batch or walk that answers it. A merge is never dropped and never -// walked twice: its key is its repository and commit, and it belongs to one plan at a time. - -// BatchedMerge is one merge as the controller heard it. -type BatchedMerge struct { - Repository string - Branch string - Commit string - // Merged is when the forge made it: the branch's order. Heard is when the controller heard it: what the - // window is measured from. - Merged time.Time - Heard time.Time - // Event is the forge's announcement, what the cut plans from. - Event json.RawMessage - // Plan is the batch or walk answering it; empty while it waits to be walked alone. - Plan string - // Alone says it is walked on its own commit, after a failed walk that carried it in a later one. - Alone bool -} - -// AddMerge keeps a merge for a plan; false when the merge was kept already, for whatever plan. -func (i *Inventory) AddMerge(ctx context.Context, m BatchedMerge) (bool, error) { - c, err := i.theChain(ctx) - if err != nil { - return false, err - } - return i.chainAddMerge(ctx, c, m) -} - -// MergeOf is one kept merge; false when it was never heard. -func (i *Inventory) MergeOf(ctx context.Context, repository, commit string) (BatchedMerge, bool, error) { - c, err := i.theChain(ctx) - if err != nil { - return BatchedMerge{}, false, err - } - return i.chainMergeOf(ctx, c, repository, commit) -} - -// MergesOf is every merge a plan answers, oldest merge first. -func (i *Inventory) MergesOf(ctx context.Context, plan string) ([]BatchedMerge, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - return i.chainMergesOf(ctx, c, plan) -} - -// LaterMergesOf is every merge of a repository's branch made after a moment and answered by a plan, the -// newest first. -func (i *Inventory) LaterMergesOf(ctx context.Context, repository, branch string, after time.Time) ([]BatchedMerge, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - return i.chainLaterMergesOf(ctx, c, repository, branch, after) -} - -// KeptOnBranch says the controller keeps a merge of a repository's branch, and when it heard the branch's first: a merge of a branch it keeps merges of, made since then, is the record's to answer, whatever a -// module's last look says (novox/hq ADR 0276). -func (i *Inventory) KeptOnBranch(ctx context.Context, repository, branch string) (bool, time.Time, error) { - c, err := i.theChain(ctx) - if err != nil { - return false, time.Time{}, err - } - return i.chainKeptOnBranch(ctx, c, repository, branch) -} - -// EarliestKept says the controller keeps a merge of some repository's branch, and when it heard the first of all: where its record of -// merges begins on the whole mesh (novox/hq ADR 0463). A merge made before then is history on every trunk. -func (i *Inventory) EarliestKept(ctx context.Context) (bool, time.Time, error) { - c, err := i.theChain(ctx) - if err != nil { - return false, time.Time{}, err - } - return i.chainEarliestKept(ctx, c) -} - -// AloneMerges is every merge waiting to be walked on its own commit, the newest first (ADR 0276 decision 3). -func (i *Inventory) AloneMerges(ctx context.Context) ([]BatchedMerge, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - return i.chainAloneMerges(ctx, c) -} - -// WaitingMerges is every merge no plan answers that is not walked alone: kept, and waiting for something else to -// happen to it (a first merge into a repository, waiting for mesh-delivery's word on it, issue 1146). -func (i *Inventory) WaitingMerges(ctx context.Context) ([]BatchedMerge, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - return i.chainWaitingMerges(ctx, c) -} - -// AnswerMerge moves a merge to the plan that answers it now; an empty plan with alone leaves it waiting to -// be walked on its own commit. -func (i *Inventory) AnswerMerge(ctx context.Context, repository, commit, plan string, alone bool) error { - c, err := i.theChain(ctx) - if err != nil { - return err - } - return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Plan: plan, Alone: alone}}) -} - -// HoldMerge puts a merge back to waiting, no plan answering it, with the event it is kept under replaced (novox/hq issue 1173: -// a first merge held for a build-agent that reads its manifest, the hold said in its event). -func (i *Inventory) HoldMerge(ctx context.Context, repository, commit string, event json.RawMessage) error { - c, err := i.theChain(ctx) - if err != nil { - return err - } - return i.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: repository, Commit: commit, Event: event}}) -} - -// Batches is every batch not yet cut: assembling or queued, oldest first. One at a time is the rule; more -// are read so a fault is seen. -func (i *Inventory) Batches(ctx context.Context) ([]Plan, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.State == PlanAssembling || e.sum.State == PlanQueued }, sortCreated) -} - -// SameRepository says two spellings of a repository are one. -func SameRepository(a, b string) bool { return strings.EqualFold(a, b) } diff --git a/internal/inventory/build_lookups.go b/internal/inventory/build_lookups.go new file mode 100644 index 00000000..0f72e797 --- /dev/null +++ b/internal/inventory/build_lookups.go @@ -0,0 +1,34 @@ +package inventory + +import "context" + +// Lookups of builds the delivery plans once held, which other code still reads. + +// BuildOf is the id of the newest successful build of a module from a commit; empty when none is +// recorded (a manifest handed over by hand). +func (i *Inventory) BuildOf(ctx context.Context, module, commit string) (string, error) { + c, err := i.theChain(ctx) + if err != nil { + return "", err + } + return i.chainBuildOf(ctx, c, module, commit) +} + +// BuildFingerprints is, per commit, what the newest successful build of a module from it would put on a +// machine — its artifacts and its manifest, hashed — so a rebuild that changes nothing there is told +// from one that does (the bus's planned step, ADR 0236). +func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[string]string, error) { + c, err := i.theChain(ctx) + if err != nil { + return nil, err + } + return i.chainBuildFingerprints(ctx, c, module) +} + +// short is a commit as a person reads it. +func short(commit string) string { + if len(commit) > 8 { + return commit[:8] + } + return commit +} diff --git a/internal/inventory/busy_integration_test.go b/internal/inventory/busy_integration_test.go deleted file mode 100644 index 61283905..00000000 --- a/internal/inventory/busy_integration_test.go +++ /dev/null @@ -1,66 +0,0 @@ -//go:build integration - -package inventory - -import ( - "testing" - "time" -) - -func TestAReportOfABusyContainerIsKeptAndReadBack(t *testing.T) { - inv, node := aNodeWithModules(t) - ctx := t.Context() - record, err := inv.NodeByName(ctx, node) - if err != nil { - t.Fatal(err) - } - since := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC) - waiting := []BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: BusyWaiting, - Since: since, Until: since.Add(15 * time.Minute), Said: "busy: a check"}} - keep := func(busy []BusyWait) bool { - t.Helper() - news, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d1", Applied: 1, - Busy: busy}, ReportOrder{Epoch: 1, Sequence: 1}, func(ReportOrder) bool { return false }) - if err != nil { - t.Fatal(err) - } - return news - } - read := func() []BusyWait { - t.Helper() - reports, err := inv.LastReports(ctx) - if err != nil { - t.Fatal(err) - } - for _, r := range reports { - if r.Node == node { - return r.Busy - } - } - t.Fatal("the machine's report is not read back") - return nil - } - keep(nil) - if got := read(); len(got) != 0 { - t.Fatalf("a report with nothing busy reads back %+v", got) - } - if !keep(waiting) { - t.Error("a container starting to wait is not news") - } - got := read() - if b, ok := WaitingFor(got, "build-agent"); !ok || !b.Since.Equal(since) || b.Said != "busy: a check" { - t.Fatalf("the wait is not read back: %+v", got) - } - again := []BusyWait{waiting[0]} - again[0].Said = "busy: a check, a minute on" - if keep(again) { - t.Error("a wait whose output changed is news") - } - if !keep([]BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: "replaced-when-idle", - Since: since, Until: since.Add(15 * time.Minute)}}) { - t.Error("a container replaced after it waited is not news") - } - if _, ok := WaitingFor(read(), "build-agent"); ok { - t.Error("a container replaced still reads as waiting") - } -} diff --git a/internal/inventory/busy_test.go b/internal/inventory/busy_test.go deleted file mode 100644 index 7170119d..00000000 --- a/internal/inventory/busy_test.go +++ /dev/null @@ -1,140 +0,0 @@ -package inventory - -import ( - "strings" - "testing" - "time" -) - -// A walk whose first machine waited for a busy container counts the wait as a phase of its own, between the -// send to the first machine and the judging, and its delivery time holds it (novox/hq ADR 0335 §4): the -// recorded walk again, its judging started only at 17:54:07, a minute after the send, when the machine said -// the container replaced. -func TestABusyWaitIsAPhaseOfTheDeliveryTime(t *testing.T) { - raw := strings.Replace(recordedWalk, `"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z",`, - `"gate": {"machines": ["novox"], "since": "2026-10-10T17:54:07.287136827Z", - "busy": {"machine": "novox", "module": "mesh-delivery", "container": "mesh-delivery", - "since": "2026-10-10T17:53:07Z", "until": "2026-10-10T18:08:07Z", "ended": "2026-10-10T17:54:07.287136827Z"},`, 1) - w := walkOf(t, raw).WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil) - sumsUp(t, w) - if w.TotalMS != 282894 { - t.Fatalf("the wait changed the delivery's total: %dms", w.TotalMS) - } - busy := phase(w, PhaseBusy, 0) - if busy.State != PhaseMeasured || busy.TookMS != 60000 { - t.Fatalf("the busy wait is not a measured phase of a minute: %+v", busy) - } - if judged := phase(w, PhaseJudge, 0); judged.TookMS != 148607-60000 { - t.Fatalf("the judging still counts from the send, not from the replacement: %+v", judged) - } - var names []string - for _, ph := range w.Phases { - names = append(names, ph.Name) - } - if !strings.Contains(strings.Join(names, " "), PhaseSend+" "+PhaseBusy+" "+PhaseJudge) { - t.Fatalf("the busy wait is not between the send and the judging: %v", names) - } -} - -// While the wait goes on, the walk says it waits for a busy container there; a walk where nothing waited has no -// such phase. -func TestAnOpenBusyWaitIsSaid(t *testing.T) { - raw := strings.Replace(recordedWalk, `"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z", "passes": 3, - "verdict": "passed", "judged_at": "2026-10-10T17:55:35.894985053Z"}`, - `"gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z", - "busy": {"machine": "novox", "module": "mesh-delivery", "container": "mesh-delivery", - "since": "2026-10-10T17:53:07Z", "until": "2026-10-10T18:08:07Z"}}`, 1) - raw = strings.Replace(raw, `"state": "done"`, `"state": "rolling"`, 1) - p := walkOf(t, raw) - if !p.Modules["mesh-delivery"].Gate.Waiting() { - t.Fatal("a gate whose busy wait has not ended does not say it waits") - } - w := p.WalkPhases(time.Date(2026, 10, 10, 17, 58, 0, 0, time.UTC), nil) - busy := phase(w, PhaseBusy, 0) - if busy.State != PhaseUnknown || !strings.Contains(busy.Said, "waits for a busy container on novox") { - t.Fatalf("an open busy wait is not said: %+v", busy) - } - if plain := walkOf(t, recordedWalk).WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil); phase(plain, PhaseBusy, 0).Name != "" { - t.Fatalf("a walk where nothing waited has a busy phase: %+v", plain.Phases) - } -} - -// A container a report lists waiting is its module's by its id; its output alone is not a new account. -func TestABusyContainerIsItsModules(t *testing.T) { - list := []BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: BusyWaiting, Said: "a"}, - {ID: "web.server", State: "replaced-when-idle"}} - if b, ok := WaitingFor(list, "build-agent"); !ok || b.Container != "mesh-build-agent" { - t.Fatalf("build-agent's container is not found waiting: %+v", b) - } - if _, ok := WaitingFor(list, "web"); ok { - t.Error("a container replaced reads as waiting") - } - if _, ok := WaitingFor(list, "build"); ok { - t.Error("another module's container is found by a prefix of its name") - } - again := []BusyWait{{ID: "build-agent.server", State: BusyWaiting, Said: "b, a minute on"}, list[1]} - if !sameBusy(list, again) { - t.Error("a wait whose output changed is a new account") - } - if sameBusy(list, list[1:]) || !(Doing{Busy: list}).Waiting() || (Doing{Busy: list[1:]}).Waiting() { - t.Error("a wait that ended is not a new account") - } -} - -// A machine's report of a busy container is kept with its last account and read back beside it, and its -// starting and its end are news; its output alone is not (novox/hq ADR 0335). Needs the store. - -// A machine of the rest that answered and waits for a busy container is never read as silent, its wait is kept -// out of its apply time, and once replaced its report is the end of the wait (novox/hq ADR 0335). -func TestARestMachineWaitingForABusyContainerIsNeverSilent(t *testing.T) { - c := walk582(t) - shanks := c.walk.Modules["build-agent"].Rest["shanks"] - ref := shanks.Digest + "@" + shanks.At.Format("2006-01-02T15:04:05.000000") - answered := shanks.At.Add(15 * time.Second) - // shanks answered, and said a container waits: its apply row is the answer, marked busy. - c.rows["shanks"][0].Detail = ApplyBusy - at := c.judged.Add(20 * time.Minute) // past ApplySilentAfter from its send - w := c.walk.WalkPhases(at, c.lookup) - var got RestReport - for _, r := range w.Rest { - if r.Node == "shanks" { - got = r - } - } - if got.State != RestBusy || slicesContains(w.Silent, "shanks") { - t.Fatalf("a machine waiting for a busy container reads %q (silent: %v)", got.State, w.Silent) - } - if w.End != nil || !strings.Contains(phase(w, PhaseApply, -1).Said, "waits for a busy container on shanks") { - t.Fatalf("the walk ended, or does not say it waits: %+v", phase(w, PhaseApply, -1)) - } - - // 17 minutes on, shanks says it replaced the container: its wait is its own duration, and its report is then. - c.rows["shanks"] = append(c.rows["shanks"], Duration{Kind: DurationBusy, Subject: "shanks", Node: "shanks", Ref: ref, - Started: answered, Took: 17 * time.Minute}) - w = c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup) - sumsUp(t, w) - for _, r := range w.Rest { - if r.Node == "shanks" { - got = r - } - } - if got.State != RestApplied || got.Busy != "17m0s" || !got.Reported.Equal(answered.Add(17*time.Minute)) { - t.Fatalf("a machine replaced after its wait reads %+v", got) - } - if slicesContains(w.Silent, "shanks") || !w.End.Equal(answered.Add(17*time.Minute)) { - t.Fatalf("the walk read shanks as silent, or ended elsewhere: %v %v", w.Silent, w.End) - } - // And the apply time itself is the answer's, never the wait's. - if r, _ := FirstReportOf(c.rows["shanks"], shanks, c.walkSent); !r.Answered.Equal(answered) || r.Busy != 17*time.Minute { - t.Fatalf("the busy wait is not kept apart from the apply: %+v", r) - } -} - -func slicesContains(list []string, s string) bool { - for _, x := range list { - if x == s { - return true - } - } - return false -} diff --git a/internal/inventory/chain.go b/internal/inventory/chain.go index 0d297639..6f5d8af0 100644 --- a/internal/inventory/chain.go +++ b/internal/inventory/chain.go @@ -25,10 +25,6 @@ var ErrNoChain = errors.New("this kind of record lives on the bus, and this proc type chainState struct { mu sync.Mutex open ChainOpener - // view is the rollouts bucket kept in memory by one watch (issue 1092); pending are the merges added to a plan that - // is not saved yet. - view *rolloutView - pending map[string]pendingMerge // requireLayout is set for a chain on the bus: its builds are read only once the controller's start has laid them out // under their modules (issue 1121), and laid is that seen. requireLayout, laid bool @@ -75,10 +71,6 @@ func (i *Inventory) UseChain(c *chain.Chain) { i.state.mu.Lock() defer i.state.mu.Unlock() i.chain = c - if i.state.view != nil { - i.state.view.stop() - i.state.view = nil - } } // UseThrowawayChain gives the inventory a chain in memory that is its own, laid out already: for a mesh raised for a moment diff --git a/internal/inventory/chain_assignments.go b/internal/inventory/chain_assignments.go index ff7b8a02..79e9ece2 100644 --- a/internal/inventory/chain_assignments.go +++ b/internal/inventory/chain_assignments.go @@ -124,13 +124,6 @@ func (i *Inventory) isAssigned(ctx context.Context, node, module string) (bool, return false, nil } -// raiseGeneration raises the assignment generation, as the store's triggers do for a change of its table (migration -// 0094): in the chain nothing in the store changes with an assignment, so the writer says it. -func (i *Inventory) raiseGeneration(ctx context.Context) error { - _, err := i.store.Pool().Exec(ctx, `update assignment_generation set generation = generation + 1`) - return err -} - // chainAssign puts a module on a machine, and says whether that is new. func (i *Inventory) chainAssign(ctx context.Context, c *chain.Chain, node Node, module string) (bool, error) { var known bool @@ -154,7 +147,7 @@ func (i *Inventory) chainAssign(ctx context.Context, c *chain.Chain, node Node, switch { case found && cur.Value.Stands(): // A module only pulled in by another becomes an assignment, keeping what it holds. - cur.Value.Assigned, cur.Value.PulledIn = now, false + cur.Value.Assigned = now _, err = c.Assignments.Update(ctx, key, cur.Value, cur.Revision) case found: // One that ended is assigned again, and one that was pending is made: a fresh value over it, the earlier one @@ -170,9 +163,6 @@ func (i *Inventory) chainAssign(ctx context.Context, c *chain.Chain, node Node, if err != nil { return false, fmt.Errorf("%s is not assigned to %s on the bus: %w", module, node.Name, err) } - if err := i.raiseGeneration(ctx); err != nil { - return true, err - } return true, nil } return false, fmt.Errorf("the assignment of %s to %s kept moving under the write", module, node.Name) @@ -195,7 +185,7 @@ func (i *Inventory) chainUnassign(ctx context.Context, c *chain.Chain, node Node } else if err != nil { return fmt.Errorf("%s is not taken off %s on the bus: %w", module, node.Name, err) } - return i.raiseGeneration(ctx) + return nil } return fmt.Errorf("the assignment of %s to %s kept moving under the write", module, node.Name) } @@ -273,11 +263,11 @@ func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed i case keepSkip: continue case keepCreate: - next.Assigned, next.PulledIn = time.Time{}, true + next.Assigned = time.Time{} _, err = c.Assignments.Create(ctx, key, next) case keepFresh: // A value that ended, pulled in again: a fresh one over it, the ended one staying in history. - next.Assigned, next.PulledIn = time.Time{}, true + next.Assigned = time.Time{} _, err = c.Assignments.Update(ctx, key, next, cur.Revision) default: cur.Value.RequiredBy, cur.Value.Provisioning = next.RequiredBy, next.Provisioning @@ -292,8 +282,8 @@ func (i *Inventory) KeepAssignmentPlans(ctx context.Context) (written, removed i written++ } for key, cur := range have { - if _, wanted := want[key]; wanted || !cur.Value.Assigned.IsZero() || !cur.Value.Stands() || cur.Value.Commit != "" { - continue // one a send still carries is on the machine, and ends when a send does not carry it any more + if _, wanted := want[key]; wanted || !cur.Value.Assigned.IsZero() || !cur.Value.Stands() { + continue } // A value of a module no longer pulled in, which no assignment names: it ends, and stays. cur.Value.Ended = time.Now().UTC() diff --git a/internal/inventory/chain_builds.go b/internal/inventory/chain_builds.go index 03517eac..4794adc2 100644 --- a/internal/inventory/chain_builds.go +++ b/internal/inventory/chain_builds.go @@ -810,33 +810,11 @@ func fillManifests(ctx context.Context, c *chain.Chain, builds []Build) error { return nil } -// failedAtAGate is the builds that failed their gate. The gate's verdicts stay in their table (issue 1097). -func (i *Inventory) failedAtAGate(ctx context.Context) (map[string]bool, error) { - rows, err := i.store.Pool().Query(ctx, `select build from build_gate where verdict = 'failed'`) - if err != nil { - return nil, err - } - defer rows.Close() - out := map[string]bool{} - for rows.Next() { - var id string - if err := rows.Scan(&id); err != nil { - return nil, err - } - out[id] = true - } - return out, rows.Err() -} - func (i *Inventory) chainSourceRows(ctx context.Context, c *chain.Chain, module string) (map[string][]sourceRow, []string, error) { all, err := chainBuildsOf(ctx, c, module) if err != nil { return nil, nil, err } - failed, err := i.failedAtAGate(ctx) - if err != nil { - return nil, nil, err - } ok := worked(all) byModule(ok) out := map[string][]sourceRow{} @@ -853,7 +831,7 @@ func (i *Inventory) chainSourceRows(ctx context.Context, c *chain.Chain, module order = append(order, b.Module) } out[b.Module] = append(out[b.Module], sourceRow{id: b.ID, commit: b.Commit, fingerprint: b.SourceFingerprint, - made: made, manifest: b.Manifest, manifestDigest: b.ManifestDigest, failedGate: failed[b.ID]}) + made: made, manifest: b.Manifest, manifestDigest: b.ManifestDigest}) } return out, order, nil } @@ -1104,8 +1082,7 @@ func (i *Inventory) chainEveryReferenceMade(ctx context.Context, c *chain.Chain) // chainHeldMade is, per module, what the build standing for its newest successful build made. Read from the pointers: the // module's `worked` (its newest success) and `held` (the first build of the run of one source that it stands on), one record -// of each. A build that failed its gate ends a run (issue 280), and the pointer cannot say where; then the module's builds -// are read. +// of each. func (i *Inventory) chainHeldMade(ctx context.Context, c *chain.Chain) (map[string][]Artifact, error) { worked, err := i.pointedBuilds(ctx, c, "*.worked") if err != nil { @@ -1115,29 +1092,12 @@ func (i *Inventory) chainHeldMade(ctx context.Context, c *chain.Chain) (map[stri if err != nil { return nil, err } - failed, err := i.failedAtAGate(ctx) - if err != nil { - return nil, err - } out := map[string][]Artifact{} for module, w := range worked { stand := w - if h, ok := held[module]; ok && !failed[w.ID] { + if h, ok := held[module]; ok { stand = h } - if failed[stand.ID] && stand.ID != w.ID { - rows, _, err := i.chainSourceRows(ctx, c, module) - if err != nil { - return nil, err - } - if builds := rows[module]; len(builds) > 0 { - var made []Artifact - if err := json.Unmarshal(builds[standing(builds, 0)].made, &made); err == nil { - out[module] = made - } - } - continue - } out[module] = fromChainBuild(stand).Made } return out, nil diff --git a/internal/inventory/chain_builds_test.go b/internal/inventory/chain_builds_test.go index b1ef3314..0d8c5ddb 100644 --- a/internal/inventory/chain_builds_test.go +++ b/internal/inventory/chain_builds_test.go @@ -123,7 +123,6 @@ func TestTheBusBeingAwayIsNotNowLikeTheStoreBeingAway(t *testing.T) { for name, err := range map[string]error{ "a value too large": chain.ErrTooLarge, "a key not taken": chain.ErrBadKey, - "a plan moved": ErrPlanMoved, "a key that exists": chain.ErrExists, } { if Unreachable(err) { diff --git a/internal/inventory/chain_first_read_lays_out_test.go b/internal/inventory/chain_first_read_lays_out_test.go index 7b80c80f..ef11f3ad 100644 --- a/internal/inventory/chain_first_read_lays_out_test.go +++ b/internal/inventory/chain_first_read_lays_out_test.go @@ -23,3 +23,13 @@ func TestTheFirstReadOfAChainNotLaidOutLaysItOut(t *testing.T) { t.Fatalf("the read did not lay the builds out: found %v, %v", found, err) } } + +// anInventoryOnTheChain is an inventory with no store and a chain in memory: the chain's own paths, which never reach the +// store. +func anInventoryOnTheChain(t *testing.T) (*Inventory, *chain.Chain) { + t.Helper() + c := chain.InMemory(nil) + inv := &Inventory{state: &chainState{}} + inv.UseChain(c) + return inv, c +} diff --git a/internal/inventory/chain_links.go b/internal/inventory/chain_links.go index 37f8f3b7..9bd85ce7 100644 --- a/internal/inventory/chain_links.go +++ b/internal/inventory/chain_links.go @@ -2,15 +2,7 @@ package inventory import ( "context" - "encoding/json" - "errors" - "fmt" - "maps" - "slices" "sort" - "time" - - "github.com/jackc/pgx/v5" "github.com/novox/mesh-controller/internal/chain" ) @@ -21,44 +13,6 @@ import ( // release_plan.go). One read of an assignment then names the commit, the build and the rollout that put it there, and one read // of the rollout lists its assignments. -type rolloutKey struct{} - -// RolloutModule is what a rollout sends of one module: the commit and the build of it. -type RolloutModule struct{ Commit, Build string } - -type rolloutOf struct { - id string - modules map[string]RolloutModule -} - -// WithRollout is a context for a send a rollout makes: what RecordSent writes in the assignments of the modules it carries -// names this rollout and the build it sent. A module the send carries beside these, that another walk is moving, is named -// by the build the mesh holds of its commit and by no rollout. -func WithRollout(ctx context.Context, id string, modules map[string]RolloutModule) context.Context { - return context.WithValue(ctx, rolloutKey{}, rolloutOf{id: id, modules: modules}) -} - -// AssignmentRuns is what each module assigned to a machine runs there, as its assignment names it: the build and the -// commit. Empty when the assignments are not on the bus, and for a module not sent yet. -func (i *Inventory) AssignmentRuns(ctx context.Context, node string) (map[string]chain.Ref, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - all, err := c.Assignments.All(ctx) - if err != nil { - return nil, err - } - out := map[string]chain.Ref{} - for _, k := range all { - a := k.Value - if a.Node == node && a.Stands() && (a.Build != "" || a.Commit != "") { - out[a.Module] = chain.Ref{Build: a.Build, Commit: a.Commit} - } - } - return out, nil -} - // AssignmentsOn are the assignments of a machine as the chain names them, by module: the standing ones, with their links. func (i *Inventory) AssignmentsOn(ctx context.Context, node string) ([]chain.Assignment, error) { c, err := i.theChain(ctx) @@ -78,285 +32,3 @@ func (i *Inventory) AssignmentsOn(ctx context.Context, node string) ([]chain.Ass sort.Slice(out, func(a, b int) bool { return out[a].Module < out[b].Module }) return out, nil } - -// recordRuns writes what a send put on a machine into the assignments, once the send is recorded, and it is how the mesh -// knows what a machine was sent (SentBuilds). **A send that names its builds** writes, for each module it carried a build of -// — by the commit — the build, what it published, the commit, the rollout and since when, on the module's key: filled where -// the key names nothing, replaced when the commit moved, never blanked by a push by hand; a module nothing assigns, carried -// because another requires it, gets a standing key marked pulled-in; and every other key of the machine loses what it named, -// for the machine is not sent it any more. **A send that names none** (a declaration by hand) makes what the machine runs -// unknown: every key of it says so (sent-unknown) and names no commit. **An error here fails the record of the send**, for -// what a machine was sent is read from these keys and from nowhere else. -func (i *Inventory) recordRuns(ctx context.Context, nodeID string, builds map[string]string, at time.Time) error { - c, err := i.theChain(ctx) - if err != nil { - return fmt.Errorf("what the machine runs cannot be written: %w", err) - } - var node string - if err := i.store.Pool().QueryRow(ctx, `select name from node where id = $1`, nodeID).Scan(&node); errors.Is(err, pgx.ErrNoRows) { - return nil - } else if err != nil { - return err - } - rollout, _ := ctx.Value(rolloutKey{}).(rolloutOf) - all, err := c.Assignments.All(ctx) - if err != nil { - return fmt.Errorf("the assignments of %s cannot be read: %w", node, err) - } - var failed []error - for _, k := range sortedValues(all) { - a := k.Value - if a.Node != node || !a.Stands() { - continue - } - if builds[a.Module] != "" { - continue // carried, and its commit is written below - } - if err := i.forgetRun(ctx, c, k, builds == nil, at); err != nil { - failed = append(failed, fmt.Errorf("%s on %s: %w", a.Module, node, err)) - } - } - modules := make([]string, 0, len(builds)) - for m := range builds { - modules = append(modules, m) - } - sort.Strings(modules) - for _, m := range modules { - if builds[m] == "" { - continue - } - if err := i.recordRun(ctx, c, node, m, builds[m], rollout, at); err != nil { - failed = append(failed, fmt.Errorf("%s on %s: %w", m, node, err)) - } - } - return errors.Join(failed...) -} - -func sortedValues(all map[string]chain.Kept[chain.Assignment]) []chain.Kept[chain.Assignment] { - out := make([]chain.Kept[chain.Assignment], 0, len(all)) - for key, k := range all { - k.Key = key - out = append(out, k) - } - sort.Slice(out, func(a, b int) bool { return out[a].Key < out[b].Key }) - return out -} - -// forgetRun writes that a machine is no longer sent what a key named: its links are cleared, and for a send by hand the key -// says what the machine runs is unknown. -func (i *Inventory) forgetRun(ctx context.Context, c *chain.Chain, k chain.Kept[chain.Assignment], hand bool, at time.Time) error { - for try := 0; try < casTries; try++ { - next := k.Value - next.Build, next.Artifact, next.Package, next.Commit, next.Rollout, next.Since = "", "", nil, "", "", time.Time{} - next.SentUnknown = time.Time{} - if hand { - next.SentUnknown = at.UTC() - } - if next.Build == k.Value.Build && next.Commit == k.Value.Commit && next.Rollout == k.Value.Rollout && - next.Artifact == k.Value.Artifact && next.Package == k.Value.Package && next.Since.Equal(k.Value.Since) && - next.SentUnknown.Equal(k.Value.SentUnknown) { - return nil - } - if _, err := c.Assignments.Update(ctx, k.Key, next, k.Revision); errors.Is(err, chain.ErrMoved) { - cur, found, gerr := c.Assignments.Get(ctx, k.Key) - if gerr != nil { - return gerr - } - if !found || !cur.Value.Stands() { - return nil - } - k = chain.Kept[chain.Assignment]{Key: k.Key, Value: cur.Value, Revision: cur.Revision} - continue - } else { - return err - } - } - return fmt.Errorf("the assignment kept moving under the write") -} - -func (i *Inventory) recordRun(ctx context.Context, c *chain.Chain, node, module, commit string, rollout rolloutOf, at time.Time) error { - key := assignmentKey(node, module) - var build, rolloutID string - if sent, ours := rollout.modules[module]; ours && sent.Commit == commit { - build, rolloutID = sent.Build, rollout.id - } - var published *chain.Published - resolved := false - for try := 0; try < casTries; try++ { - cur, found, err := c.Assignments.Get(ctx, key) - if err != nil { - return err - } - if found && cur.Value.State != "" { - return nil // a wait for a build: the module is not on the machine by it - } - standing := found && cur.Value.Stands() - same := standing && cur.Value.Commit == commit - if same && cur.Value.Build != "" && cur.Value.SentUnknown.IsZero() && (rolloutID == "" || cur.Value.Rollout != "") { - return nil // a send that carries what the machine runs already says nothing new - } - if !resolved { - // The build of the commit is looked up once, and only for a module that moved or names no build. - if build == "" { - if build, err = i.BuildOf(ctx, module, commit); err != nil { - return err - } - } - if build != "" { - if b, found, err := i.BuildByID(ctx, build); err != nil { - return err - } else if found { - published = publishedOf(b.Made) - } - } - resolved = true - } - next := cur.Value - switch { - case !standing: - // Carried and assigned to nobody here: a standing key, no time assigned, marked pulled in. An ended one is - // written over, the ended one staying in history. - next = chain.Assignment{Node: node, Module: module, PulledIn: true, Commit: commit, Build: build, Rollout: rolloutID, - Since: at.UTC()} - case same: - // Already running it: fill what is missing, blank nothing. - if next.Build == "" { - next.Build = build - } - if next.Rollout == "" { - next.Rollout = rolloutID - } - if next.Since.IsZero() { - next.Since = at.UTC() - } - default: - next.Commit, next.Build, next.Rollout, next.Since = commit, build, rolloutID, at.UTC() - next.Artifact, next.Package = "", nil - } - next.SentUnknown = time.Time{} - if published != nil && next.Artifact == "" && next.Package == nil && next.Build == build { - next.Artifact, next.Package = published.Artifact, published.Package - } - if standing && next.Build == cur.Value.Build && next.Commit == cur.Value.Commit && next.Rollout == cur.Value.Rollout && - next.Artifact == cur.Value.Artifact && (next.Package == nil) == (cur.Value.Package == nil) && - next.Since.Equal(cur.Value.Since) && next.SentUnknown.Equal(cur.Value.SentUnknown) { - return nil - } - if !found { - if _, err := c.Assignments.Create(ctx, key, next); errors.Is(err, chain.ErrExists) { - continue - } else { - return err - } - } - if _, err := c.Assignments.Update(ctx, key, next, cur.Revision); errors.Is(err, chain.ErrMoved) { - continue - } else { - return err - } - } - return fmt.Errorf("the assignment kept moving under the write") -} - -// CopySentBuilds copies what delivery 1 recorded in node.sent_builds onto the keys of the assignments, once, at the start of -// the controller that holds the lease and has attached the chain (issue 1109, review round 1): delivery 1 wrote no commit onto -// the key of a module an assignment pulled in, so without it those modules would be missing from SentBuilds although the -// machine is known to have been sent. A key that names a commit is never overwritten, so it is idempotent; once a node's keys -// are written its column is emptied, so a later restart cannot lay a stale commit over a key a later send cleared or ended. -// It answers how many keys it wrote; an error is raised by the caller as a condition. -func (i *Inventory) CopySentBuilds(ctx context.Context) (int, error) { - c, err := i.theChain(ctx) - if err != nil { - return 0, err - } - rows, err := i.store.Pool().Query(ctx, `select name, sent_builds from node where sent_builds is not null`) - if err != nil { - return 0, err - } - type kept struct { - name string - sent map[string]string - } - var all []kept - for rows.Next() { - var name string - var raw []byte - if err := rows.Scan(&name, &raw); err != nil { - rows.Close() - return 0, err - } - sent := map[string]string{} - if err := json.Unmarshal(raw, &sent); err != nil { - rows.Close() - return 0, fmt.Errorf("what %s was sent, in the store, cannot be read: %w", name, err) - } - all = append(all, kept{name, sent}) - } - rows.Close() - if err := rows.Err(); err != nil { - return 0, err - } - now := time.Now().UTC() - for _, k := range all { - if err := i.copySentBuildsOf(ctx, c, k.name, k.sent, now); err != nil { - return 0, fmt.Errorf("what %s was sent cannot be copied onto its assignments: %w", k.name, err) - } - if _, err := i.store.Pool().Exec(ctx, `update node set sent_builds = null where name = $1`, k.name); err != nil { - return 0, err - } - } - return len(all), nil -} - -// copySentBuildsOf is CopySentBuilds for one node, over a chain: for each module the node was sent, a standing key that -// names no commit gets it (and the build, when one is known; no rollout), and where no key exists a standing pulled-in key is -// made. A key with a commit, an ended key, a hand send's unknown and a wait for a build are left as they are. -func (i *Inventory) copySentBuildsOf(ctx context.Context, c *chain.Chain, node string, sent map[string]string, at time.Time) error { - for _, module := range slices.Sorted(maps.Keys(sent)) { - commit := sent[module] - if commit == "" { - continue - } - key := assignmentKey(node, module) - build, err := i.chainBuildOf(ctx, c, module, commit) - if err != nil { - return err - } - for try := 0; ; try++ { - if try == casTries { - return fmt.Errorf("the assignment %s kept moving under the write", key) - } - cur, found, err := c.Assignments.Get(ctx, key) - if err != nil { - return err - } - if found && (cur.Value.State != "" || !cur.Value.Ended.IsZero() || cur.Value.Commit != "" || !cur.Value.SentUnknown.IsZero()) { - break // a wait, an ended key (never revived), one that names a commit, or a hand send's unknown - } - next := cur.Value - if !found { - next = chain.Assignment{Node: node, Module: module, PulledIn: true, Since: at} - } - next.Commit, next.Build = commit, build - if next.Since.IsZero() { - next.Since = at - } - if !found { - _, err = c.Assignments.Create(ctx, key, next) - if errors.Is(err, chain.ErrExists) { - continue - } - } else { - _, err = c.Assignments.Update(ctx, key, next, cur.Revision) - if errors.Is(err, chain.ErrMoved) { - continue - } - } - if err != nil { - return err - } - break - } - } - return nil -} diff --git a/internal/inventory/chain_pending_test.go b/internal/inventory/chain_pending_test.go index 3d48ba19..b18acaa0 100644 --- a/internal/inventory/chain_pending_test.go +++ b/internal/inventory/chain_pending_test.go @@ -77,9 +77,6 @@ func TestARecordedPendingIsSkippedByAssignedAndTheJoins(t *testing.T) { if running, _ := inv.Running(ctx, "web"); len(running) != 0 { t.Fatalf("running %v", running) } - if runs, _ := inv.AssignmentRuns(ctx, node); len(runs) != 0 { - t.Fatalf("runs %v", runs) - } entries, _ := inv.Catalogued(ctx) for _, e := range entries { if e.Manifest.Module == "web" && len(e.On) != 0 { diff --git a/internal/inventory/chain_rollouts.go b/internal/inventory/chain_rollouts.go deleted file mode 100644 index 9bcfa996..00000000 --- a/internal/inventory/chain_rollouts.go +++ /dev/null @@ -1,986 +0,0 @@ -package inventory - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "sort" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/novox/mesh-controller/internal/chain" -) - -// Rollouts on the bus (novox/hq ADR 0408, ADR 0276, issue 1092): once the kind `rollouts` is moved (chain.go) every plan -// and batch is a key of the rollouts bucket under its id, written by compare-and-set, and the merges a batch or walk -// answers are held in its record. The tables are read when the kind is not moved, by the methods these stand beside. -// -// **Two counters.** The bucket's revision is the compare-and-set of every write. The plan's own Revision, kept in its value, -// counts the saves of the plan: a save refuses (ErrPlanMoved) a plan another save moved since it was read, and takes no -// notice of a merge moving in or out of the record, which a copy read before could not know of. Held is written by the -// merge operations alone; a save keeps what the record holds. - -// rolloutView is the rollouts bucket in memory, kept current by one watch (issue 1092): the survey of the plans, the -// batches and the merges is read from here, never from the bucket one key at a time. A write is still made by -// compare-and-set on the bucket, so a stale view costs a refused write and a read again, never a wrong one. -type rolloutView struct { - chain *chain.Chain - mu sync.RWMutex - entries map[string]viewEntry - ready chan struct{} - once sync.Once - err error - dead atomic.Bool - cancel context.CancelFunc -} - -// viewEntry is a record as the bucket holds it, with what the surveys ask of it read out. -type viewEntry struct { - raw []byte - rev uint64 - sum planSummary -} - -// planSummary is the part of a plan the surveys filter and sort by. -type planSummary struct { - ID string `json:"id"` - State string `json:"state"` - Created time.Time `json:"created"` - Repository string `json:"repository"` - Branch string `json:"branch"` - Merged time.Time `json:"merged"` - Release json.RawMessage `json:"release"` - Held []chain.HeldMerge `json:"held"` -} - -func (s planSummary) open() bool { return s.State == PlanBuilding || s.State == PlanRolling } - -func (s planSummary) isRelease() bool { - r := strings.TrimSpace(string(s.Release)) - return r != "" && r != "null" -} - -func newRolloutView(c *chain.Chain) *rolloutView { - return &rolloutView{chain: c, entries: map[string]viewEntry{}, ready: make(chan struct{})} -} - -func (v *rolloutView) start() { - ctx, cancel := context.WithCancel(context.Background()) - v.cancel = cancel - go func() { - w, err := chain.RolloutsOf[Plan](v.chain).Watch(ctx) - if err != nil { - v.fail(fmt.Errorf("the rollouts cannot be watched: %w", err)) - return - } - for { - select { - case <-ctx.Done(): - v.fail(ctx.Err()) - return - case e, open := <-w: - if !open { - v.fail(errors.New("the rollouts bucket stopped answering")) - return - } - if e.Caught { - v.once.Do(func() { close(v.ready) }) - continue - } - v.apply(e.Key, e.Value, e.Revision) - } - } - }() -} - -// fail ends the view: it answers an error to whoever waits for it to fill, and is made again by the next read. -func (v *rolloutView) fail(err error) { - v.mu.Lock() - if v.err == nil { - v.err = err - } - v.mu.Unlock() - v.dead.Store(true) - v.once.Do(func() { close(v.ready) }) -} - -func (v *rolloutView) stop() { - if v.cancel != nil { - v.cancel() - } -} - -// apply keeps a record the watch handed over or a write of this process made, unless the view holds a newer one. -func (v *rolloutView) apply(key string, raw []byte, rev uint64) { - var sum planSummary - if err := json.Unmarshal(raw, &sum); err != nil || sum.ID == "" { - sum = planSummary{ID: key} - } - v.mu.Lock() - defer v.mu.Unlock() - if old, has := v.entries[key]; has && old.rev >= rev { - return - } - v.entries[key] = viewEntry{raw: append([]byte(nil), raw...), rev: rev, sum: sum} -} - -func (v *rolloutView) list() []viewEntry { - v.mu.RLock() - defer v.mu.RUnlock() - out := make([]viewEntry, 0, len(v.entries)) - for _, e := range v.entries { - out = append(out, e) - } - return out -} - -// rolloutView is the view of the chain's rollouts, started at the first read and ready when it holds the bucket whole. -func (i *Inventory) rolloutView(ctx context.Context, c *chain.Chain) (*rolloutView, error) { - i.state.mu.Lock() - v := i.state.view - if v == nil || v.chain != c || v.dead.Load() { - if v != nil { - v.stop() - } - v = newRolloutView(c) - i.state.view = v - v.start() - } - i.state.mu.Unlock() - select { - case <-v.ready: - case <-ctx.Done(): - return nil, ctx.Err() - } - v.mu.RLock() - defer v.mu.RUnlock() - if v.err != nil && len(v.entries) == 0 { - return nil, fmt.Errorf("the rollouts cannot be read from the bus: %w", v.err) - } - return v, nil -} - -// stopViews ends the view's watch: the inventory is closing. -func (i *Inventory) stopViews() { - i.state.mu.Lock() - defer i.state.mu.Unlock() - if i.state.view != nil { - i.state.view.stop() - i.state.view = nil - } -} - -func normalizePlan(p *Plan) { - if p.Modules == nil { - p.Modules = map[string]*PlanModule{} - } - if p.TierEntered.IsZero() { - p.TierEntered = p.Created - } - if !p.Merged.IsZero() { - p.Merged = p.Merged.UTC() - } -} - -func planOf(e viewEntry) (Plan, error) { - var p Plan - if err := json.Unmarshal(e.raw, &p); err != nil { - return Plan{}, fmt.Errorf("the plan %s on the bus cannot be read: %w", e.sum.ID, err) - } - normalizePlan(&p) - return p, nil -} - -func sortCreated(entries []viewEntry) { - sort.SliceStable(entries, func(a, b int) bool { - if !entries[a].sum.Created.Equal(entries[b].sum.Created) { - return entries[a].sum.Created.Before(entries[b].sum.Created) - } - return entries[a].sum.ID < entries[b].sum.ID - }) -} - -func sortCreatedNewest(entries []viewEntry) { - sort.SliceStable(entries, func(a, b int) bool { - if !entries[a].sum.Created.Equal(entries[b].sum.Created) { - return entries[a].sum.Created.After(entries[b].sum.Created) - } - return entries[a].sum.ID > entries[b].sum.ID - }) -} - -// chainPlans is the plans that pass keep, in an order, read from the view. -func (i *Inventory) chainPlans(ctx context.Context, c *chain.Chain, keep func(viewEntry) bool, order func([]viewEntry)) ([]Plan, error) { - v, err := i.rolloutView(ctx, c) - if err != nil { - return nil, err - } - var chosen []viewEntry - for _, e := range v.list() { - if e.sum.ID != WaitingRecord && keep(e) { - chosen = append(chosen, e) - } - } - order(chosen) - out := make([]Plan, 0, len(chosen)) - for _, e := range chosen { - p, err := planOf(e) - if err != nil { - return nil, err - } - out = append(out, p) - } - return out, nil -} - -func (i *Inventory) chainNewestMergeOf(ctx context.Context, c *chain.Chain, repository, branch string) (Plan, bool, error) { - v, err := i.rolloutView(ctx, c) - if err != nil { - return Plan{}, false, err - } - var chosen []viewEntry - for _, e := range v.list() { - if strings.EqualFold(e.sum.Repository, repository) && e.sum.Branch == branch && !e.sum.Merged.IsZero() && !e.sum.isRelease() { - chosen = append(chosen, e) - } - } - sort.SliceStable(chosen, func(a, b int) bool { - if !chosen[a].sum.Merged.Equal(chosen[b].sum.Merged) { - return chosen[a].sum.Merged.After(chosen[b].sum.Merged) - } - return chosen[a].sum.Created.After(chosen[b].sum.Created) - }) - if len(chosen) == 0 { - return Plan{}, false, nil - } - p, err := planOf(chosen[0]) - return p, err == nil, err -} - -// chainPlanByID is one plan, read from the bucket itself: the caller of the verb wants the plan as it is, and the view -// learns of it. -func (i *Inventory) chainPlanByID(ctx context.Context, c *chain.Chain, id string) (Plan, error) { - if !chain.ValidKey(id) { - // No plan has an id the bus takes no spelling of; it is a plain id no plan has. - return Plan{}, fmt.Errorf("no plan %s", id) - } - e, found, err := chain.RolloutsOf[Plan](c).GetRaw(ctx, id) - if err != nil { - return Plan{}, err - } - if !found { - return Plan{}, fmt.Errorf("no plan %s", id) - } - if v, err := i.rolloutView(ctx, c); err == nil { - v.apply(id, e.Value, e.Revision) - } - return planOf(viewEntry{raw: e.Value, rev: e.Revision, sum: planSummary{ID: id}}) -} - -// ---- the merges - -// mergeHolder is a record's copy of a merge. -type mergeHolder struct { - plan string - m chain.HeldMerge - rev uint64 -} - -func mergeKey(repository, commit string) string { return strings.ToLower(repository) + "|" + commit } - -// mergeHolders is every record's copy of each merge, by the merge's key. -func mergeHolders(entries []viewEntry) map[string][]mergeHolder { - out := map[string][]mergeHolder{} - for _, e := range entries { - for _, m := range e.sum.Held { - k := mergeKey(m.Repository, m.Commit) - out[k] = append(out[k], mergeHolder{plan: e.sum.ID, m: m, rev: e.rev}) - } - } - return out -} - -// winner is the copy of a merge that counts: the one that entered its record last (ADR 0408, issue 1092 — a merge in two -// records is the half-done move of one, and the record it entered last is the one it moved to), a merge a plan answers -// before one no plan does, then the record written last. -func winner(copies []mergeHolder) mergeHolder { - best := copies[0] - for _, c := range copies[1:] { - switch { - case c.m.Placed.After(best.m.Placed): - best = c - case c.m.Placed.Equal(best.m.Placed) && best.m.Waiting && !c.m.Waiting: - best = c - case c.m.Placed.Equal(best.m.Placed) && best.m.Waiting == c.m.Waiting && c.rev > best.rev: - best = c - } - } - return best -} - -func winners(entries []viewEntry) map[string]mergeHolder { - out := map[string]mergeHolder{} - for k, copies := range mergeHolders(entries) { - out[k] = winner(copies) - } - return out -} - -func batchedOf(h mergeHolder) BatchedMerge { - b := BatchedMerge{Repository: strings.ToLower(h.m.Repository), Branch: h.m.Branch, Commit: h.m.Commit, - Heard: h.m.Heard.UTC(), Event: h.m.Event, Plan: h.plan, Alone: h.m.Alone} - if !h.m.Merged.IsZero() { - b.Merged = h.m.Merged.UTC() - } - if h.m.Waiting { - b.Plan = "" - } - return b -} - -func heldOf(m BatchedMerge, now time.Time) chain.HeldMerge { - h := chain.HeldMerge{Repository: strings.ToLower(m.Repository), Branch: m.Branch, Commit: m.Commit, Heard: m.Heard.UTC(), - Event: m.Event, Alone: m.Alone, Placed: now.UTC()} - if !m.Merged.IsZero() { - h.Merged = m.Merged.UTC() - } - return h -} - -func (i *Inventory) chainWinners(ctx context.Context, c *chain.Chain) (map[string]mergeHolder, error) { - v, err := i.rolloutView(ctx, c) - if err != nil { - return nil, err - } - return winners(v.list()), nil -} - -func (i *Inventory) chainMergeOf(ctx context.Context, c *chain.Chain, repository, commit string) (BatchedMerge, bool, error) { - w, err := i.chainWinners(ctx, c) - if err != nil { - return BatchedMerge{}, false, err - } - h, ok := w[mergeKey(repository, commit)] - if !ok { - return BatchedMerge{}, false, nil - } - return batchedOf(h), true, nil -} - -// merged orders merges by the time they were made, those whose time is not known last, then by when they were heard. -func sortMerges(ms []BatchedMerge, newest bool) { - sort.SliceStable(ms, func(a, b int) bool { - x, y := ms[a], ms[b] - switch { - case x.Merged.IsZero() != y.Merged.IsZero(): - return !x.Merged.IsZero() - case !x.Merged.Equal(y.Merged): - if newest { - return x.Merged.After(y.Merged) - } - return x.Merged.Before(y.Merged) - } - if newest { - return x.Heard.After(y.Heard) - } - return x.Heard.Before(y.Heard) - }) -} - -func (i *Inventory) chainMergesOf(ctx context.Context, c *chain.Chain, plan string) ([]BatchedMerge, error) { - w, err := i.chainWinners(ctx, c) - if err != nil { - return nil, err - } - var out []BatchedMerge - for _, h := range w { - if h.plan == plan && !h.m.Waiting { - out = append(out, batchedOf(h)) - } - } - sortMerges(out, false) - return out, nil -} - -func (i *Inventory) chainLaterMergesOf(ctx context.Context, c *chain.Chain, repository, branch string, after time.Time) ([]BatchedMerge, error) { - w, err := i.chainWinners(ctx, c) - if err != nil { - return nil, err - } - var out []BatchedMerge - for _, h := range w { - if !h.m.Waiting && strings.EqualFold(h.m.Repository, repository) && h.m.Branch == branch && h.m.Merged.After(after) { - out = append(out, batchedOf(h)) - } - } - sort.SliceStable(out, func(a, b int) bool { return out[a].Merged.After(out[b].Merged) }) - return out, nil -} - -func (i *Inventory) chainKeptOnBranch(ctx context.Context, c *chain.Chain, repository, branch string) (bool, time.Time, error) { - w, err := i.chainWinners(ctx, c) - if err != nil { - return false, time.Time{}, err - } - var first time.Time - for _, h := range w { - if strings.EqualFold(h.m.Repository, repository) && h.m.Branch == branch && (first.IsZero() || h.m.Heard.Before(first)) { - first = h.m.Heard - } - } - if first.IsZero() { - return false, time.Time{}, nil - } - return true, first.UTC(), nil -} - -// chainEarliestKept is when the controller heard the first merge it holds a record of, on any repository and branch. -func (i *Inventory) chainEarliestKept(ctx context.Context, c *chain.Chain) (bool, time.Time, error) { - w, err := i.chainWinners(ctx, c) - if err != nil { - return false, time.Time{}, err - } - var first time.Time - for _, h := range w { - if first.IsZero() || h.m.Heard.Before(first) { - first = h.m.Heard - } - } - if first.IsZero() { - return false, time.Time{}, nil - } - return true, first.UTC(), nil -} - -func (i *Inventory) chainAloneMerges(ctx context.Context, c *chain.Chain) ([]BatchedMerge, error) { - w, err := i.chainWinners(ctx, c) - if err != nil { - return nil, err - } - var out []BatchedMerge - for _, h := range w { - if h.m.Waiting && h.m.Alone { - out = append(out, batchedOf(h)) - } - } - sortMerges(out, true) - return out, nil -} - -// chainWaitingMerges is every merge no plan answers that waits and is not walked alone, the newest first. -func (i *Inventory) chainWaitingMerges(ctx context.Context, c *chain.Chain) ([]BatchedMerge, error) { - w, err := i.chainWinners(ctx, c) - if err != nil { - return nil, err - } - var out []BatchedMerge - for _, h := range w { - if h.m.Waiting && !h.m.Alone { - out = append(out, batchedOf(h)) - } - } - sortMerges(out, true) - return out, nil -} - -// pendingMerge is a merge added to a plan that is not saved yet: carried until that plan's save writes it. -type pendingMerge struct { - plan string - m chain.HeldMerge -} - -// chainAddMerge keeps a merge for a plan. **When the plan has a record the merge is written into it at once**; when it has -// none — a new batch, saved by the caller next — the merge is carried and written by that plan's save. The forge's event is -// acknowledged only after the caller's write succeeded, and handed over again if it did not (ADR 0407 §3). -func (i *Inventory) chainAddMerge(ctx context.Context, c *chain.Chain, m BatchedMerge) (bool, error) { - waiting := m.Plan == "" - if waiting { - // A merge no plan answers is held, flagged, by the record kept for those. - m.Plan = WaitingRecord - if err := i.ensureWaitingRecord(ctx, c); err != nil { - return false, err - } - } - v, err := i.rolloutView(ctx, c) - if err != nil { - return false, err - } - key := mergeKey(m.Repository, m.Commit) - if _, known := winners(v.list())[key]; known { - return false, nil - } - held := heldOf(m, time.Now()) - held.Waiting = waiting - found, err := i.editRecord(ctx, c, m.Plan, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) { - for _, x := range h { - if mergeKey(x.Repository, x.Commit) == key { - return h, false - } - } - return append(h, held), true - }) - if err != nil { - return false, err - } - if !found { - i.state.mu.Lock() - if i.state.pending == nil { - i.state.pending = map[string]pendingMerge{} - } - i.state.pending[key] = pendingMerge{plan: m.Plan, m: held} - i.state.mu.Unlock() - } - return true, nil -} - -// WaitingRecord is the record that holds the merges no plan has ever answered: a merge kept with no plan, which a store -// kept as a row with a null plan. It is no plan: the surveys of plans leave it out. -const WaitingRecord = "waiting-merges" - -// ensureWaitingRecord makes the record of the merges no plan answers, once. -func (i *Inventory) ensureWaitingRecord(ctx context.Context, c *chain.Chain) error { - b := chain.RolloutsOf[Plan](c) - if _, found, err := b.GetRaw(ctx, WaitingRecord); err != nil || found { - return err - } - body, err := json.Marshal(Plan{ID: WaitingRecord, State: "merges", Created: time.Unix(0, 0).UTC(), Revision: 1, - Tiers: [][]string{}, Modules: map[string]*PlanModule{}, Note: "the merges no plan answers"}) - if err != nil { - return err - } - rev, err := b.CreateRaw(ctx, WaitingRecord, body) - if errors.Is(err, chain.ErrExists) { - return nil - } - if err != nil { - return err - } - if v, verr := i.rolloutView(ctx, c); verr == nil { - v.apply(WaitingRecord, body, rev) - } - return nil -} - -// ErrMergeUnsaved is a merge moved to a plan that has no record to hold it. -var ErrMergeUnsaved = errors.New("a merge was moved to a plan that has not been saved") - -// editRecord changes the merges a plan's record holds and nothing else of it, by compare-and-set on the bucket, read again -// when it moved. It answers false when the plan has no record. -func (i *Inventory) editRecord(ctx context.Context, c *chain.Chain, id string, edit func([]chain.HeldMerge) ([]chain.HeldMerge, bool)) (bool, error) { - b := chain.RolloutsOf[Plan](c) - for try := 0; try < casTries; try++ { - e, found, err := b.GetRaw(ctx, id) - if err != nil || !found { - return false, err - } - var record map[string]json.RawMessage - if err := json.Unmarshal(e.Value, &record); err != nil { - return true, fmt.Errorf("the plan %s on the bus cannot be read: %w", id, err) - } - var held []chain.HeldMerge - if raw, has := record["held"]; has { - if err := json.Unmarshal(raw, &held); err != nil { - return true, fmt.Errorf("the merges of plan %s on the bus cannot be read: %w", id, err) - } - } - next, changed := edit(held) - if !changed { - return true, nil - } - if len(next) == 0 { - delete(record, "held") - } else { - raw, err := json.Marshal(next) - if err != nil { - return true, err - } - record["held"] = raw - } - body, err := json.Marshal(record) - if err != nil { - return true, err - } - rev, err := b.UpdateRaw(ctx, id, body, e.Revision) - if errors.Is(err, chain.ErrMoved) { - continue - } - if err != nil { - return true, fmt.Errorf("the merges of plan %s are not written: %w", id, err) - } - if v, verr := i.rolloutView(ctx, c); verr == nil { - v.apply(id, body, rev) - } - return true, nil - } - return true, fmt.Errorf("the merges of plan %s kept moving under the write", id) -} - -// recordEdit is what the merges of one record gain, lose and have flagged in one act. -type recordEdit struct { - add []chain.HeldMerge - drop map[string]bool - flag map[string]chain.HeldMerge -} - -func (e *recordEdit) apply(held []chain.HeldMerge) []chain.HeldMerge { - out := make([]chain.HeldMerge, 0, len(held)+len(e.add)) - for _, h := range held { - k := mergeKey(h.Repository, h.Commit) - if e.drop[k] { - continue - } - if f, has := e.flag[k]; has { - h.Waiting, h.Alone = f.Waiting, f.Alone - // A merge put back to waiting may carry a new announcement (HoldMerge); every other flag copies its own. - if len(f.Event) > 0 { - h.Event = f.Event - } - } - out = append(out, h) - } - for _, a := range e.add { - k := mergeKey(a.Repository, a.Commit) - dup := false - for _, h := range out { - if mergeKey(h.Repository, h.Commit) == k { - dup = true - break - } - } - if !dup { - out = append(out, a) - } - } - return out -} - -func (e *recordEdit) empty() bool { return len(e.add) == 0 && len(e.drop) == 0 && len(e.flag) == 0 } - -// ---- the save - -// tierLeftOf is the measurement a plan's save makes when it leaves a tier (planTierLeft, over the record the save -// replaces): when the plan entered the tier it is now at, and the duration of the tier it left, if it left one. -func tierLeftOf(old *Plan, p Plan, now time.Time) (time.Time, *Duration) { - if old == nil { - return now, nil // a new plan enters its first tier now - } - if old.State == PlanAssembling || old.State == PlanQueued { - return now, nil // a batch cut into a walk enters its first tier now (novox/hq ADR 0276) - } - since := old.TierEntered - if since.IsZero() { - since = old.Created - } - wasOpen := old.State == PlanBuilding || old.State == PlanRolling - if !wasOpen && p.Open() { - // **A walk set open again enters its tier anew** (issue 1886): a resume or a retry at the tier it ended at. Kept, - // the entry of before its stop measured the tier from hours before the resume — R4e read "waiting … for 5h48m53s" - // seven minutes after it was resumed — and the watchdog of its progress (S3) found it late at once. - return now, nil - } - if !wasOpen || (old.Tier == p.Tier && p.Open()) { - return since, nil // still in the tier, or already ended - } - var modules []string - if old.Tier >= 0 && old.Tier < len(p.Tiers) { - modules = p.Tiers[old.Tier] - } - detail := fmt.Sprintf("plan %s, tier %d of %d (%v), left %s", p.ID, old.Tier, len(p.Tiers), modules, p.State) - if p.Open() { - detail = fmt.Sprintf("plan %s, tier %d of %d (%v), moved on to tier %d", p.ID, old.Tier, len(p.Tiers), modules, p.Tier) - } - return now, &Duration{Kind: DurationPlanTier, Subject: p.Repository, Ref: fmt.Sprintf("%s/tier-%d", p.ID, old.Tier), - Started: since, Took: now.Sub(since), Detail: detail} -} - -// chainSavePlans writes plans, as SavePlan does each, and gives merges to the plans that answer them. **Each plan by -// compare-and-set on its own Revision**, a copy read before another save refused with ErrPlanMoved before anything is -// written; **a merge moved to a plan is written into the receiving record first, then taken from the giving one**, so a -// crash between leaves it in both and never in neither, and the one it entered last counts (ADR 0408, issue 1092). -func (i *Inventory) chainSavePlans(ctx context.Context, c *chain.Chain, plans []*Plan, moves []MergeMove) (err error) { - epoch, err := i.actingEpoch(ctx) - if err != nil { - if len(plans) > 0 { - return fmt.Errorf("the plan for %s %s is not written: %w", plans[0].Repository, plans[0].Commit, err) - } - return err - } - v, err := i.rolloutView(ctx, c) - if err != nil { - return err - } - b := chain.RolloutsOf[Plan](c) - marked := time.Now() - undo := make([]func(), 0, len(plans)) - defer func() { - if err != nil { - for _, u := range undo { - u() - } - } - }() - for _, p := range plans { - undo = append(undo, p.markWalk(marked)) - } - - // Every plan still at the revision it was read at, before anything is written. - cur := make(map[string]chain.Kept[Plan], len(plans)) - for _, p := range plans { - k, found, err := b.Get(ctx, p.ID) - if err != nil { - return err - } - if found { - if k.Value.Revision != p.Revision { - return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved) - } - cur[p.ID] = k - } - } - - // What each record gains, loses and has flagged. - now := time.Now().UTC() - edits := map[string]*recordEdit{} - editOf := func(id string) *recordEdit { - if edits[id] == nil { - edits[id] = &recordEdit{drop: map[string]bool{}, flag: map[string]chain.HeldMerge{}} - } - return edits[id] - } - copies := mergeHolders(v.list()) - for _, m := range moves { - key := mergeKey(m.Repository, m.Commit) - held, has := copies[key] - if !has { - // A merge carried for a plan not saved yet goes where it is moved. - i.state.mu.Lock() - if pm, pending := i.state.pending[key]; pending && m.Plan != "" { - pm.plan, pm.m.Alone = m.Plan, m.Alone - i.state.pending[key] = pm - } - i.state.mu.Unlock() - continue - } - w := winner(held) - if m.Plan == "" { - f := w.m - f.Waiting, f.Alone = true, m.Alone - if m.Event != nil { - f.Event = m.Event - } - editOf(w.plan).flag[key] = f - continue - } - if w.plan == m.Plan { - f := w.m - f.Waiting, f.Alone = false, m.Alone - editOf(w.plan).flag[key] = f - continue - } - moved := w.m - moved.Waiting, moved.Alone, moved.Placed = false, m.Alone, now - editOf(m.Plan).add = append(editOf(m.Plan).add, moved) - for _, h := range held { - editOf(h.plan).drop[key] = true - } - } - - // The records not among the plans: receivers now, givers after. - for id, e := range edits { - if _, written := cur[id]; written || planNamed(plans, id) || len(e.add) == 0 { - continue - } - e := e - found, err := i.editRecord(ctx, c, id, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) { - return (&recordEdit{add: e.add}).apply(h), true - }) - if err != nil { - return err - } - if !found { - // Nothing is taken from the record that holds a merge to give it to a plan that has none. - return fmt.Errorf("%w: %s has no record to hold %d merge(s) moved to it", ErrMergeUnsaved, id, len(e.add)) - } - } - - // The plans, those that gain a merge first. - order := append([]*Plan(nil), plans...) - sort.SliceStable(order, func(a, b int) bool { - x, y := edits[order[a].ID], edits[order[b].ID] - return x != nil && len(x.add) > 0 && (y == nil || len(y.add) == 0) - }) - type written struct { - revision int64 - entered time.Time - held []chain.HeldMerge - duration *Duration - } - done := map[string]written{} - for _, p := range order { - var w written - for try := 0; ; try++ { - if try == casTries { - return fmt.Errorf("the plan for %s %s (%s) kept moving under the write", p.Repository, short(p.Commit), p.ID) - } - k, found := cur[p.ID] - if try > 0 { - fresh, still, gerr := b.Get(ctx, p.ID) - if gerr != nil { - return gerr - } - if still && fresh.Value.Revision != p.Revision { - return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved) - } - k, found = fresh, still - cur[p.ID] = k - } - record := *p - var old *Plan - var held []chain.HeldMerge - if found { - o := k.Value - old, held = &o, append([]chain.HeldMerge(nil), k.Value.Held...) - record.Revision = k.Value.Revision + 1 - } else { - record.Revision = 1 - } - if e := edits[p.ID]; e != nil { - held = e.apply(held) - } - i.state.mu.Lock() - for key, pm := range i.state.pending { - if pm.plan == p.ID { - held = (&recordEdit{add: []chain.HeldMerge{pm.m}}).apply(held) - _ = key - } - } - i.state.mu.Unlock() - record.Held = held - entered, duration := tierLeftOf(old, *p, time.Now()) - record.TierEntered, record.Updated = entered, time.Now().UTC() - if epoch != nil { - record.Epoch = uint64(*epoch) - } else { - record.Epoch = 0 - } - body, merr := json.Marshal(record) - if merr != nil { - return merr - } - var rev uint64 - var werr error - if found { - rev, werr = b.UpdateRaw(ctx, p.ID, body, k.Revision) - } else { - rev, werr = createRaw(ctx, c, p.ID, body) - } - if errors.Is(werr, chain.ErrMoved) || (!found && errors.Is(werr, chain.ErrExists)) { - if !found { - return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, ErrPlanMoved) - } - continue - } - if werr != nil { - return fmt.Errorf("the plan for %s %s (%s) is not written: %w", p.Repository, short(p.Commit), p.ID, werr) - } - v.apply(p.ID, body, rev) - w = written{revision: record.Revision, entered: entered, held: held, duration: duration} - break - } - done[p.ID] = w - } - - // The records that gave a merge up, now it is held by the one it went to. - for id, e := range edits { - if planNamed(plans, id) || (len(e.drop) == 0 && len(e.flag) == 0) { - continue - } - e := e - if _, err := i.editRecord(ctx, c, id, func(h []chain.HeldMerge) ([]chain.HeldMerge, bool) { - return (&recordEdit{drop: e.drop, flag: e.flag}).apply(h), true - }); err != nil { - return err - } - } - - i.state.mu.Lock() - for key, pm := range i.state.pending { - if _, saved := done[pm.plan]; saved { - delete(i.state.pending, key) - } - } - i.state.mu.Unlock() - for _, p := range plans { - w := done[p.ID] - p.Revision, p.TierEntered, p.Held = w.revision, w.entered, w.held - if epoch != nil { - p.Epoch = uint64(*epoch) - } else { - p.Epoch = 0 - } - if w.duration != nil { - // After the write: the key of the duration is the plan and its tier, so a measurement made twice is one. - record := i.RecordDuration - if i.recorded != nil { - record = i.recorded - } - if derr := record(ctx, *w.duration); derr != nil { - fmt.Printf("the time plan %s spent in a tier is not recorded: %v\n", p.ID, derr) - } - } - if PlanSaved != nil { - PlanSaved(i, *p) - } - } - return nil -} - -func planNamed(plans []*Plan, id string) bool { - for _, p := range plans { - if p.ID == id { - return true - } - } - return false -} - -// createRaw writes a plan's first record. -func createRaw(ctx context.Context, c *chain.Chain, id string, body []byte) (uint64, error) { - return chain.RolloutsOf[json.RawMessage](c).CreateRaw(ctx, id, body) -} - -// RepairMerges takes a merge found in two records out of the one it entered first: the half-done move of a crash, repaired -// at the next cut (ADR 0408, issue 1092). It says how many it repaired. -func (i *Inventory) RepairMerges(ctx context.Context) (int, error) { - c, err := i.theChain(ctx) - if err != nil { - return 0, err - } - v, err := i.rolloutView(ctx, c) - if err != nil { - return 0, err - } - repaired := 0 - for key, copies := range mergeHolders(v.list()) { - if len(copies) < 2 { - continue - } - w := winner(copies) - for _, h := range copies { - if h.plan == w.plan { - continue - } - key := key - if _, err := i.editRecord(ctx, c, h.plan, func(held []chain.HeldMerge) ([]chain.HeldMerge, bool) { - return (&recordEdit{drop: map[string]bool{key: true}}).apply(held), true - }); err != nil { - return repaired, err - } - repaired++ - } - } - return repaired, nil -} diff --git a/internal/inventory/chain_rollouts_test.go b/internal/inventory/chain_rollouts_test.go deleted file mode 100644 index d7e8b7f3..00000000 --- a/internal/inventory/chain_rollouts_test.go +++ /dev/null @@ -1,374 +0,0 @@ -package inventory - -import ( - "context" - "errors" - "sync/atomic" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/chain" -) - -// anInventoryOnTheChain is an inventory with no store and a chain in memory: the chain's own paths, which never reach the -// store but for the durations a save measures, taken here by nothing. -func anInventoryOnTheChain(t *testing.T) (*Inventory, *chain.Chain) { - t.Helper() - c := chain.InMemory(nil) - inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }} - inv.UseChain(c) - t.Cleanup(inv.stopViews) - return inv, c -} - -var plansAt = time.Date(2026, 10, 18, 3, 0, 0, 0, time.UTC) - -func aBatch(id string, at time.Time) *Plan { - return &Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: "c-" + id, Created: at, State: PlanAssembling, - Tiers: [][]string{}, Modules: map[string]*PlanModule{}} -} - -func aMerge(commit, plan string) BatchedMerge { - return BatchedMerge{Repository: "Novox/Mesh-Catalog", Branch: "main", Commit: commit, Merged: plansAt, Heard: plansAt.Add(time.Second), - Event: []byte(`{"commit":"` + commit + `"}`), Plan: plan} -} - -// **A plan is kept under its id; a save of a copy another save has moved since is refused and writes nothing** (ADR 0408 -// §2, issue 1092): the plan's own Revision counts its saves, 1 for the first. -func TestAPlanIsKeptUnderItsIdAndASaveOfAStaleCopyIsRefused(t *testing.T) { - inv, c := anInventoryOnTheChain(t) - ctx := context.Background() - p := aBatch("plan-1", plansAt) - if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil { - t.Fatal(err) - } - if p.Revision != 1 { - t.Fatalf("a new plan is at revision %d", p.Revision) - } - stale := *p - p.Note = "moved on" - if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil || p.Revision != 2 { - t.Fatalf("the second save: revision %d, %v", p.Revision, err) - } - stale.Note = "laid over" - if err := inv.chainSavePlans(ctx, c, []*Plan{&stale}, nil); !errors.Is(err, ErrPlanMoved) { - t.Fatalf("a stale copy was saved: %v", err) - } - got, err := inv.chainPlanByID(ctx, c, "plan-1") - if err != nil || got.Note != "moved on" || got.Revision != 2 { - t.Fatalf("the plan is %+v: %v", got, err) - } - // A plan new to the bucket that is already there is refused too. - again := aBatch("plan-1", plansAt) - if err := inv.chainSavePlans(ctx, c, []*Plan{again}, nil); !errors.Is(err, ErrPlanMoved) { - t.Fatalf("a new plan over a kept one: %v", err) - } - if _, err := inv.chainPlanByID(ctx, c, "plan-nobody"); err == nil || err.Error() != "no plan plan-nobody" { - t.Fatalf("a plan nobody kept: %v", err) - } -} - -// **A save keeps the merges the record holds, not the ones its copy read** (issue 1092): a merge that moved into the record -// after the copy was read is not laid back by the copy's save. -func TestASaveKeepsTheMergesTheRecordHoldsAndNotTheCopys(t *testing.T) { - inv, c := anInventoryOnTheChain(t) - ctx := context.Background() - batch := aBatch("plan-1", plansAt) - if err := inv.chainSavePlans(ctx, c, []*Plan{batch}, nil); err != nil { - t.Fatal(err) - } - if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-1")); err != nil || !added { - t.Fatalf("added %v: %v", added, err) - } - // The copy was read before the merge came in; saving it neither fails nor loses the merge. - batch.Note = "window closed" - if err := inv.chainSavePlans(ctx, c, []*Plan{batch}, nil); err != nil { - t.Fatalf("a copy read before a merge came in: %v", err) - } - merges, err := inv.chainMergesOf(ctx, c, "plan-1") - if err != nil || len(merges) != 1 || merges[0].Commit != "m1" || merges[0].Repository != "novox/mesh-catalog" { - t.Fatalf("the merges of the plan: %+v %v", merges, err) - } - if string(merges[0].Event) != `{"commit":"m1"}` || merges[0].Heard.IsZero() || merges[0].Plan != "plan-1" { - t.Fatalf("the merge came back as %+v", merges[0]) - } - // A merge heard is not heard twice, whatever the spelling of its repository. - if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-1")); err != nil || added { - t.Fatalf("added again %v: %v", added, err) - } -} - -// **A merge added to a plan that is not saved is written by that plan's save, and by nothing else** (issue 1092): until -// then it is in no record — so the event that brought it is handed over again if the save fails. -func TestAMergeAddedToAPlanNotSavedYetIsWrittenByItsSave(t *testing.T) { - inv, c := anInventoryOnTheChain(t) - ctx := context.Background() - if added, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-new")); err != nil || !added { - t.Fatalf("added %v: %v", added, err) - } - if _, known, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); known { - t.Fatal("a merge for a plan not saved is in a record") - } - // The save is refused: nothing is written, the merge is still carried. - held := false - away := chain.New(chain.NewMemory(), chain.NewMemory(), chain.NewMemory(), func(context.Context) error { - if held { - return nil - } - return errors.New("the lease is not held") - }) - other := &Inventory{state: &chainState{}, recorded: inv.recorded} - other.UseChain(away) - t.Cleanup(other.stopViews) - other.state.pending = inv.state.pending - p := aBatch("plan-new", plansAt) - if err := other.chainSavePlans(ctx, away, []*Plan{p}, nil); !errors.Is(err, chain.ErrNotHeld) { - t.Fatalf("the save answered %v", err) - } - if _, known, _ := other.chainMergeOf(ctx, away, "novox/mesh-catalog", "m1"); known { - t.Fatal("a refused save left the merge in a record") - } - held = true - if err := other.chainSavePlans(ctx, away, []*Plan{p}, nil); err != nil { - t.Fatal(err) - } - if m, known, _ := other.chainMergeOf(ctx, away, "novox/mesh-catalog", "m1"); !known || m.Plan != "plan-new" { - t.Fatalf("the save did not write the merge: %+v %v", m, known) - } -} - -// failingOn is a bucket that refuses to write one key: the second half of a move, interrupted. -type failingOn struct { - chain.Backend - key string - fail *atomic.Bool -} - -func (f failingOn) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) { - if key == f.key && f.fail.Load() { - return 0, errors.New("the bus went away") - } - return f.Backend.Update(ctx, key, value, revision) -} - -// **A merge moved to a plan is written to the receiving record first, then taken from the giving one; one found in both is -// read from the one it entered last and repaired** (issue 1092, ADR 0299 §7). -func TestAMergeMovedToAPlanIsWrittenToItFirstAndRepairedIfTheMoveStopped(t *testing.T) { - c := chain.InMemory(nil) - var fail atomic.Bool - rollouts := failingOn{Backend: chain.NewMemory(), key: "plan-a", fail: &fail} - c = chain.New(chain.NewMemory(), rollouts, chain.NewMemory(), func(context.Context) error { return nil }) - inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }} - inv.UseChain(c) - t.Cleanup(inv.stopViews) - ctx := context.Background() - a, b := aBatch("plan-a", plansAt), aBatch("plan-b", plansAt.Add(time.Minute)) - for _, p := range []*Plan{a, b} { - if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil { - t.Fatal(err) - } - } - if _, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-a")); err != nil { - t.Fatal(err) - } - - // The move stops before the giving record is written: the merge is in both. - fail.Store(true) - err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-b"}}) - if err == nil { - t.Fatal("the move was said to be made with the giving record unwritten") - } - fail.Store(false) - if m, _, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); m.Plan != "plan-b" { - t.Fatalf("a merge in two records is read from %q, not the one it moved to", m.Plan) - } - if got, _ := inv.chainMergesOf(ctx, c, "plan-a"); len(got) != 0 { - t.Fatalf("the giving plan still answers %+v", got) - } - // The next cut repairs it. - v, _ := inv.rolloutView(ctx, c) - if n := len(mergeHolders(v.list())["novox/mesh-catalog|m1"]); n != 2 { - t.Fatalf("the merge is held by %d records, want the two of a stopped move", n) - } - repaired := 0 - for k, copies := range mergeHolders(v.list()) { - if len(copies) < 2 { - continue - } - w := winner(copies) - for _, h := range copies { - if h.plan != w.plan { - key := k - if _, err := inv.editRecord(ctx, c, h.plan, func(held []chain.HeldMerge) ([]chain.HeldMerge, bool) { - return (&recordEdit{drop: map[string]bool{key: true}}).apply(held), true - }); err != nil { - t.Fatal(err) - } - repaired++ - } - } - } - if repaired != 1 { - t.Fatalf("repaired %d", repaired) - } - waitFor(t, func() bool { - v, _ := inv.rolloutView(ctx, c) - return len(mergeHolders(v.list())["novox/mesh-catalog|m1"]) == 1 - }) -} - -// waitFor polls a condition that a watch makes true, for a moment. -func waitFor(t *testing.T, ok func() bool) { - t.Helper() - for end := time.Now().Add(3 * time.Second); time.Now().Before(end); time.Sleep(5 * time.Millisecond) { - if ok() { - return - } - } - t.Fatal("the condition did not come true") -} - -// **A merge no plan answers stays, flagged, in the record that held it last, and is walked alone** (issue 1092): not in -// the merges of the plan, listed among the merges waiting, and answered again by the plan it is moved to. -func TestAWaitingMergeStaysInTheRecordThatHeldItLast(t *testing.T) { - inv, c := anInventoryOnTheChain(t) - ctx := context.Background() - a, b := aBatch("plan-a", plansAt), aBatch("plan-b", plansAt.Add(time.Minute)) - for _, p := range []*Plan{a, b} { - if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil { - t.Fatal(err) - } - } - if _, err := inv.chainAddMerge(ctx, c, aMerge("m1", "plan-a")); err != nil { - t.Fatal(err) - } - if err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "", Alone: true}}); err != nil { - t.Fatal(err) - } - if got, _ := inv.chainMergesOf(ctx, c, "plan-a"); len(got) != 0 { - t.Fatalf("a waiting merge is answered by %+v", got) - } - alone, err := inv.chainAloneMerges(ctx, c) - if err != nil || len(alone) != 1 || alone[0].Commit != "m1" || alone[0].Plan != "" || !alone[0].Alone { - t.Fatalf("waiting: %+v %v", alone, err) - } - if m, known, _ := inv.chainMergeOf(ctx, c, "novox/mesh-catalog", "m1"); !known || m.Plan != "" { - t.Fatalf("a waiting merge reads as %+v %v", m, known) - } - if later, _ := inv.chainLaterMergesOf(ctx, c, "novox/mesh-catalog", "main", plansAt.Add(-time.Hour)); len(later) != 0 { - t.Fatalf("a waiting merge is among those a plan answers: %+v", later) - } - if kept, first, _ := inv.chainKeptOnBranch(ctx, c, "novox/mesh-catalog", "main"); !kept || first.IsZero() { - t.Fatalf("a waiting merge is not kept on its branch: %v %v", kept, first) - } - // Walked alone: the plan it goes to answers it, and the record that held it lets it go. - if err := inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-b", Alone: true}}); err != nil { - t.Fatal(err) - } - if got, _ := inv.chainMergesOf(ctx, c, "plan-b"); len(got) != 1 || !got[0].Alone { - t.Fatalf("plan-b answers %+v", got) - } - if left, _ := inv.chainAloneMerges(ctx, c); len(left) != 0 { - t.Fatalf("still waiting: %+v", left) - } - waitFor(t, func() bool { - v, _ := inv.rolloutView(ctx, c) - return len(mergeHolders(v.list())["novox/mesh-catalog|m1"]) == 1 - }) - // A move to a plan with no record is refused and takes nothing from the one that holds the merge. - err = inv.chainSavePlans(ctx, c, nil, []MergeMove{{Repository: "novox/mesh-catalog", Commit: "m1", Plan: "plan-nowhere"}}) - if !errors.Is(err, ErrMergeUnsaved) { - t.Fatalf("a move to a plan with no record: %v", err) - } - if got, _ := inv.chainMergesOf(ctx, c, "plan-b"); len(got) != 1 { - t.Fatalf("the merge was taken from plan-b: %+v", got) - } -} - -// countingBackend counts the reads a bucket answers. -type countingBackend struct { - chain.Backend - gets, keys atomic.Int32 -} - -func (c *countingBackend) Get(ctx context.Context, key string) (chain.Entry, bool, error) { - c.gets.Add(1) - return c.Backend.Get(ctx, key) -} - -func (c *countingBackend) Keys(ctx context.Context) ([]string, error) { - c.keys.Add(1) - return c.Backend.Keys(ctx) -} - -// **The surveys are read from a view kept by one watch, never from the bucket key by key** (issue 1092): the plans and the -// merges are asked for many times and the bucket is read for none of them; a write by another process reaches the view -// through its watch. -func TestThePlansAreReadFromAViewKeptByOneWatch(t *testing.T) { - counting := &countingBackend{Backend: chain.NewMemory()} - c := chain.New(chain.NewMemory(), counting, chain.NewMemory(), func(context.Context) error { return nil }) - inv := &Inventory{state: &chainState{}, recorded: func(context.Context, Duration) error { return nil }} - inv.UseChain(c) - t.Cleanup(inv.stopViews) - ctx := context.Background() - for k, id := range []string{"plan-1", "plan-2", "plan-3"} { - p := aBatch(id, plansAt.Add(time.Duration(k)*time.Minute)) - if k > 0 { - p.State = PlanBuilding - } - if err := inv.chainSavePlans(ctx, c, []*Plan{p}, nil); err != nil { - t.Fatal(err) - } - } - before := counting.gets.Load() - for n := 0; n < 20; n++ { - open, err := inv.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated) - if err != nil || len(open) != 2 || open[0].ID != "plan-2" { - t.Fatalf("open plans %+v: %v", open, err) - } - if _, err := inv.chainAloneMerges(ctx, c); err != nil { - t.Fatal(err) - } - } - if counting.gets.Load() != before || counting.keys.Load() != 0 { - t.Fatalf("the bucket was read %d times and its keys %d times for the surveys", counting.gets.Load()-before, counting.keys.Load()) - } - // Another process writes; the view follows. - other := &Inventory{state: &chainState{}, recorded: inv.recorded} - other.UseChain(c) - t.Cleanup(other.stopViews) - late := aBatch("plan-4", plansAt.Add(time.Hour)) - late.State = PlanRolling - if err := other.chainSavePlans(ctx, c, []*Plan{late}, nil); err != nil { - t.Fatal(err) - } - waitFor(t, func() bool { - open, _ := inv.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated) - return len(open) == 3 - }) -} - -// Where the record of merges begins is the earliest merge kept on any repository and branch (novox/hq ADR 0463 §2). -func TestTheRecordOfMergesBeginsAtTheEarliestMergeKeptOnAnyTrunk(t *testing.T) { - inv, c := anInventoryOnTheChain(t) - ctx := context.Background() - if kept, _, err := inv.chainEarliestKept(ctx, c); err != nil || kept { - t.Fatalf("an inventory with no merge kept says it keeps one: %v %v", kept, err) - } - if err := inv.chainSavePlans(ctx, c, []*Plan{aBatch("plan-1", plansAt), aBatch("plan-2", plansAt)}, nil); err != nil { - t.Fatal(err) - } - later := aMerge("m-late", "plan-1") - later.Repository, later.Branch, later.Heard = "novox/mesh-sdk", "main", plansAt.Add(48*time.Hour) - earlier := aMerge("m-early", "plan-2") - earlier.Repository, earlier.Branch, earlier.Heard = "novox/photos", "nox-mesh", plansAt.Add(time.Hour) - for _, m := range []BatchedMerge{later, earlier} { - if added, err := inv.chainAddMerge(ctx, c, m); err != nil || !added { - t.Fatalf("%s: %v %v", m.Commit, added, err) - } - } - kept, first, err := inv.chainEarliestKept(ctx, c) - if err != nil || !kept || !first.Equal(plansAt.Add(time.Hour)) { - t.Fatalf("the record begins at %v (%v, %v), want %v", first, kept, err, plansAt.Add(time.Hour)) - } -} diff --git a/internal/inventory/chain_sent_copy_test.go b/internal/inventory/chain_sent_copy_test.go deleted file mode 100644 index 3920c150..00000000 --- a/internal/inventory/chain_sent_copy_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package inventory - -import ( - "testing" - "time" - - "github.com/novox/mesh-controller/internal/chain" -) - -// **What delivery 1 recorded in node.sent_builds is copied onto the keys of the assignments, once** (issue 1109, review -// round 1): a module pulled in by an assignment had no commit on its key, so it would fall out of SentBuilds. The copy -// creates a standing pulled-in key where none stands, fills a standing key that names no commit (commit, and the build -// when one is known, no rollout), and overwrites nothing: a key with a commit, a hand send's unknown, a wait for a build. -func TestWhatTheColumnRecordedIsCopiedOntoTheKeysAndOverwritesNothing(t *testing.T) { - inv, c := anInventoryOnTheChain(t) - ctx := t.Context() - at := time.Date(2026, 10, 18, 4, 0, 0, 0, time.UTC) - if err := inv.chainRecordBuild(ctx, c, Build{ID: "b-wg", Module: "mesh-wireguard", Commit: "c-wg"}); err != nil { - t.Fatal(err) - } - put := func(module string, a chain.Assignment) { - a.Node, a.Module = "laptop", module - if _, err := c.Assignments.Create(ctx, assignmentKey("laptop", module), a); err != nil { - t.Fatal(err) - } - } - put("notes", chain.Assignment{Assigned: at}) // assigned, no commit yet - put("web", chain.Assignment{Assigned: at, Commit: "c-web-new", Build: "b1"}) // already has one - put("hand", chain.Assignment{Assigned: at, SentUnknown: at}) // a hand send since - put("waits", chain.Assignment{Assigned: at, State: chain.StatePending}) // a wait for a build - put("ended", chain.Assignment{Assigned: at, Ended: at}) // unassigned - sent := map[string]string{"notes": "c-notes", "web": "c-web-old", "mesh-wireguard": "c-wg", "hand": "c-hand", - "waits": "c-waits", "ended": "c-ended"} - - for pass := 0; pass < 2; pass++ { // and the second pass changes nothing - if err := inv.copySentBuildsOf(ctx, c, "laptop", sent, at); err != nil { - t.Fatal(err) - } - get := func(module string) chain.Assignment { - t.Helper() - k, found, err := c.Assignments.Get(ctx, assignmentKey("laptop", module)) - if err != nil || !found { - t.Fatalf("%s: %v %v", module, found, err) - } - return k.Value - } - if a := get("notes"); a.Commit != "c-notes" || a.Rollout != "" || !a.Stands() || a.PulledIn { - t.Errorf("notes: %+v", a) - } - if a := get("web"); a.Commit != "c-web-new" || a.Build != "b1" { - t.Errorf("a key with a commit was overwritten: %+v", a) - } - if a := get("hand"); a.Commit != "" || a.SentUnknown.IsZero() { - t.Errorf("a hand send's unknown was overwritten: %+v", a) - } - if a := get("waits"); a.Commit != "" || a.State != chain.StatePending { - t.Errorf("a wait was overwritten: %+v", a) - } - if a := get("mesh-wireguard"); a.Commit != "c-wg" || a.Build != "b-wg" || !a.PulledIn || !a.Stands() || a.Rollout != "" { - t.Errorf("the pulled-in module has no standing key of its own: %+v", a) - } - if a := get("ended"); a.Commit != "" || a.Stands() { - t.Errorf("an ended key was revived or given a commit: %+v", a) - } - } -} - -// The copy reads the column of the store, writes the keys, and empties the column it has copied: SentBuilds then answers the -// pulled-in module, and a second copy finds nothing to do. -func TestTheColumnOfASentMachineIsCopiedOntoItsKeysAndEmptied(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - if _, err := inv.AddNode(ctx, "laptop"); err != nil { - t.Fatal(err) - } - inv.ExecForTest(t, `update node set sent_builds = '{"mesh-wireguard": "c-wg"}'::jsonb, sent_at = now() where name = 'laptop'`) - if n, err := inv.CopySentBuilds(ctx); err != nil || n != 1 { - t.Fatalf("the copy did %d nodes (%v)", n, err) - } - got, known, err := inv.SentBuilds(ctx, "laptop") - if err != nil || !known || got["mesh-wireguard"] != "c-wg" { - t.Fatalf("SentBuilds answers %v, known %v (%v)", got, known, err) - } - if rows := inv.QueryForTest(t, `select count(*) from node where sent_builds is not null`); rows[0][0] != "0" { - t.Fatalf("the column was not emptied: %v", rows) - } - if n, err := inv.CopySentBuilds(ctx); err != nil || n != 0 { - t.Fatalf("a second copy did %d nodes (%v)", n, err) - } -} - -// A machine delivery 2 has sent has its column emptied by the send itself, so a copy after it leaves a key the send cleared -// without a commit. -func TestASendEmptiesTheColumnSoALaterCopyLeavesAClearedKeyBare(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - record, err := inv.AddNode(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - inv.ExecForTest(t, `update node set sent_builds = '{"notes": "c-old"}'::jsonb where name = 'laptop'`) - if err := inv.RecordSentUnder(ctx, record.ID, "d1", map[string]string{}, 1); err != nil { - t.Fatal(err) - } - if rows := inv.QueryForTest(t, `select count(*) from node where sent_builds is not null`); rows[0][0] != "0" { - t.Fatalf("a send left the column: %v", rows) - } - if n, err := inv.CopySentBuilds(ctx); err != nil || n != 0 { - t.Fatalf("the copy did %d nodes (%v)", n, err) - } -} diff --git a/internal/inventory/chain_sent_test.go b/internal/inventory/chain_sent_test.go deleted file mode 100644 index a62ea017..00000000 --- a/internal/inventory/chain_sent_test.go +++ /dev/null @@ -1,116 +0,0 @@ -//go:build integration - -package inventory - -import ( - "context" - "errors" - "reflect" - "testing" - - "github.com/novox/mesh-controller/internal/chain" -) - -// **What a machine was sent is read from the keys of its assignments that name a commit** (ADR 0408, issue 1109): an assigned -// module's, and a module only pulled in, whose key is a standing value marked pulled-in and never a state. -func TestSentBuildsAnswersFromTheAssignmentsKeysOfTheNode(t *testing.T) { - inv, node := aNodeWithModules(t, "web", "cache") - ctx := t.Context() - record, _ := inv.NodeByName(ctx, node) - if _, err := inv.Assign(ctx, node, "web"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1", "cache": "c2"}); err != nil { - t.Fatal(err) - } - if builds, known, err := inv.SentBuilds(ctx, node); err != nil || !known || !reflect.DeepEqual(builds, map[string]string{"web": "c1", "cache": "c2"}) { - t.Fatalf("sent %v %v %v", builds, known, err) - } - pulled, found, _ := inv.AssignmentValue(ctx, node, "cache") - if !found || !pulled.Stands() || pulled.State != "" || !pulled.PulledIn || !pulled.Assigned.IsZero() || pulled.Commit != "c2" { - t.Fatalf("the module only carried has the key %+v", pulled) - } - if on, _ := inv.Assigned(ctx, node); !reflect.DeepEqual(on, []string{"web"}) { - t.Fatalf("a module only pulled in is assigned: %v", on) - } - // Assigned later, it keeps what it was sent and is pulled in no longer. - if _, err := inv.Assign(ctx, node, "cache"); err != nil { - t.Fatal(err) - } - made, _, _ := inv.AssignmentValue(ctx, node, "cache") - if made.PulledIn || made.Assigned.IsZero() || made.Commit != "c2" { - t.Fatalf("assigned over a pulled-in key: %+v", made) - } -} - -// **A send replaces the whole map of the node, and a hand send makes it unknown until the next send that names its builds** -// (issue 1109): a module no longer carried loses its commit, the node reads unknown when any key says so. -func TestASendReplacesTheWholeMapOfTheNodeAndAHandSendMakesItUnknown(t *testing.T) { - inv, node := aNodeWithModules(t, "web", "cache") - ctx := t.Context() - record, _ := inv.NodeByName(ctx, node) - if _, err := inv.Assign(ctx, node, "web"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1", "cache": "c2"}); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, record.ID, "d2", map[string]string{"web": "c3"}); err != nil { - t.Fatal(err) - } - if builds, known, _ := inv.SentBuilds(ctx, node); !known || !reflect.DeepEqual(builds, map[string]string{"web": "c3"}) { - t.Fatalf("the second send did not replace the map: %v %v", builds, known) - } - if old, _, _ := inv.AssignmentValue(ctx, node, "cache"); old.Commit != "" || old.Build != "" { - t.Fatalf("a module no longer carried keeps %+v", old) - } - // By hand: unknown, with a time and no commit anywhere. - if err := inv.RecordSent(ctx, record.ID, "d3", nil); err != nil { - t.Fatal(err) - } - if builds, known, _ := inv.SentBuilds(ctx, node); known || builds != nil { - t.Fatalf("a hand send reads as %v %v", builds, known) - } - web, _, _ := inv.AssignmentValue(ctx, node, "web") - if web.SentUnknown.IsZero() || web.Commit != "" || !web.Stands() { - t.Fatalf("web after a hand send: %+v", web) - } - if sent, _ := inv.Outstanding(ctx, node); sent != "d3" { - t.Fatalf("the digest of the hand send: %q", sent) - } - // A send that names its builds makes it known again. - if err := inv.RecordSent(ctx, record.ID, "d4", map[string]string{"web": "c4"}); err != nil { - t.Fatal(err) - } - if builds, known, _ := inv.SentBuilds(ctx, node); !known || !reflect.DeepEqual(builds, map[string]string{"web": "c4"}) { - t.Fatalf("after a send that names its builds: %v %v", builds, known) - } - if again, _, _ := inv.AssignmentValue(ctx, node, "web"); !again.SentUnknown.IsZero() { - t.Fatalf("web still says unknown: %+v", again) - } -} - -// failingToWrite is an assignments bucket that fails every write. -type failingToWrite struct{ chain.Backend } - -func (f failingToWrite) Create(context.Context, string, []byte) (uint64, error) { - return 0, errors.New("the bus went away") -} - -func (f failingToWrite) Update(context.Context, string, []byte, uint64) (uint64, error) { - return 0, errors.New("the bus went away") -} - -// **A links write that fails fails the record of the send, loudly** (issue 1109): what a machine was sent is read from the -// keys, so a send recorded without them would read as one that never happened. -func TestAFailedWriteOfTheLinksFailsTheRecordOfTheSend(t *testing.T) { - inv, node := aNodeWithModules(t, "web") - ctx := t.Context() - record, _ := inv.NodeByName(ctx, node) - inv.UseChain(chain.New(chain.NewMemory(), chain.NewMemory(), failingToWrite{chain.NewMemory()}, - func(context.Context) error { return nil })) - err := inv.RecordSent(ctx, record.ID, "d1", map[string]string{"web": "c1"}) - if err == nil { - t.Fatal("a send whose links could not be written was recorded") - } -} diff --git a/internal/inventory/chain_store_test.go b/internal/inventory/chain_store_test.go index d05be0d2..a05465f3 100644 --- a/internal/inventory/chain_store_test.go +++ b/internal/inventory/chain_store_test.go @@ -4,6 +4,7 @@ package inventory import ( "context" + "encoding/json" "errors" "reflect" "slices" @@ -72,10 +73,6 @@ func TestBuildsOnTheBusAreReadBackByEveryReader(t *testing.T) { if err != nil || len(announced) != 3 { t.Fatalf("announceable %d: %v", len(announced), err) } - prints, err := inv.Fingerprints(ctx) - if err != nil || prints["notes"]["c1"] == "" || prints["notes"]["c2"] == "" || prints["wiki"]["c0ffeebuild-4"] == "" { - t.Fatalf("fingerprints %v: %v", prints, err) - } } // A build request row as the store held it before the move. @@ -166,7 +163,7 @@ func TestARequestIsTheBuildInStateAskedAndItsOutcomeTakesItsPlace(t *testing.T) // **An assignment is a key of the assignments bucket; an unassignment writes `ended` and keeps the key and its values; a // repeat is absorbed** (ADR 0408, issue 1093): the readers of the store — what a machine runs, where a module runs, whether a module -// may be forgotten, what is behind — read the bucket, the generation rises with each change, and a seat held by the +// may be forgotten, what is behind — read the bucket, and a seat held by the // assignment goes with it. func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) { inv, node := aNodeWithModules(t, "postgres", "web") @@ -174,7 +171,6 @@ func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) { if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { t.Fatal(err) } - start := generationNow(t, inv) if added, err := inv.Assign(ctx, node, "postgres"); err != nil || !added { t.Fatalf("assigned %v: %v", added, err) } @@ -185,8 +181,24 @@ func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) { if err != nil || !found || kept.Value.Node != "anchor" || kept.Value.Module != "postgres" || kept.Value.Assigned.IsZero() { t.Fatalf("the assignment on the bus: %+v %v %v", kept.Value, found, err) } - if generationNow(t, inv) <= start { - t.Fatal("an assignment left the generation where it was") + // **Placement only** (ADR 0539 §12): the value on the bus names the node, the module and since when, as mesh-config + // reads them (unknown fields ignored there), and no build, commit or rollout. + kv, err := inv.KeptAssignments(ctx) + if err != nil || len(kv["anchor.postgres"].Value) == 0 { + t.Fatalf("the kept assignments: %v %v", kv, err) + } + var read struct { + Node string `json:"node"` + Module string `json:"module"` + Assigned time.Time `json:"assigned"` + } + if err := json.Unmarshal(kv["anchor.postgres"].Value, &read); err != nil || read.Node != "anchor" || read.Module != "postgres" || read.Assigned.IsZero() { + t.Fatalf("a reader of placement reads %+v from %s: %v", read, kv["anchor.postgres"].Value, err) + } + for _, not := range []string{`"build"`, `"commit"`, `"rollout"`, `"artifact"`, `"package"`, `"since"`} { + if strings.Contains(string(kv["anchor.postgres"].Value), not) { + t.Errorf("an assignment names %s: %s", not, kv["anchor.postgres"].Value) + } } if _, err := inv.Assign(ctx, node, "no-such-module"); !errors.Is(err, ErrNoSuchModule) { t.Fatalf("a module the mesh does not know: %v", err) @@ -224,13 +236,9 @@ func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) { t.Fatalf("a seat handed to an assignment: %v", err) } - before := generationNow(t, inv) if err := inv.Unassign(ctx, node, "postgres"); err != nil { t.Fatal(err) } - if generationNow(t, inv) <= before { - t.Fatal("an unassignment left the generation where it was") - } // No key is removed (ADR 0408 §4): the value says it ended, and the one before it is in the key's history. gone, found, err := inv.chain.Assignments.Get(ctx, "anchor.postgres") if err != nil || !found || gone.Value.Ended.IsZero() || gone.Value.Stands() || gone.Value.Assigned.IsZero() { @@ -278,12 +286,12 @@ func TestAnAssignmentIsAKeyOfTheBucketAndAnUnassignmentEndsIt(t *testing.T) { } // **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the -// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay, +// plan of a module the machine was sent is written into its assignment, another field of the value stays, // a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here // ends an assignment. // **The provisioner's plans are kept in the assignments' values, one field at a time** (issue 1093, design 57 P6): the -// plan of a module the machine was sent is written into its assignment, the build and commit another write put there stay, +// plan of a module the machine was sent is written into its assignment, another field of the value stays, // a module only pulled in by another has a value of its own that ends when it is no longer pulled in, and nothing here // ends an assignment. func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testing.T) { @@ -300,23 +308,14 @@ func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testi Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc", Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}} raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"notify":` + strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`) - if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil { - t.Fatal(err) - } - // A build and a commit put in the value by another write, after the send: a send that carries no builds (a hand send, as this - // one) forgets what each assignment of its machine runs (issue 1109), so they are written once it is recorded. - cur, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst") - cur.Value.Build, cur.Value.Commit = "build-1", "c1" - if _, err := inv.chain.Assignments.Update(ctx, "anchor.dunst", cur.Value, cur.Revision); err != nil { - t.Fatal(err) - } + sentPlans(t, inv, record.ID, raw) inv.SetProvisioningPlans(inv.SentProvisioningPlans) written, removed, err := inv.KeepAssignmentPlans(ctx) if err != nil || written != 2 || removed != 0 { t.Fatalf("kept: %d written, %d ended: %v", written, removed, err) } dunst, _, _ := inv.chain.Assignments.Get(ctx, "anchor.dunst") - if dunst.Value.Provisioning == nil || dunst.Value.Build != "build-1" || dunst.Value.Commit != "c1" || dunst.Value.Assigned.IsZero() { + if dunst.Value.Provisioning == nil || dunst.Value.Assigned.IsZero() { t.Fatalf("dunst's value is %+v", dunst.Value) } if web, _, _ := inv.chain.Assignments.Get(ctx, "anchor.web"); web.Value.Provisioning != nil { @@ -334,9 +333,7 @@ func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testi } // Sent again without the pulled-in module: its value ends, the assignments stay. only := []byte(`{"dunst":` + mustJSON(t, plan) + `}`) - if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second", Provisioning: only}); err != nil { - t.Fatal(err) - } + sentPlans(t, inv, record.ID, only) if _, removed, err := inv.KeepAssignmentPlans(ctx); err != nil || removed != 1 { t.Fatalf("ended %d: %v", removed, err) } @@ -351,3 +348,20 @@ func TestThePlansAreKeptInTheAssignmentsValuesAndNothingElseOfThemMoves(t *testi // **The move carries every assignment with what its machine was last sent, and keeps the plan the mirror put in the value** // (ADR 0408 §8, issue 1093): the commit, the build of it and what it published, since when; a value that names its build // already is left as it is. + +// sentPlans gives a machine the plans it was last sent, as the column the mirror reads keeps them. +func sentPlans(t *testing.T, inv *Inventory, id string, plans []byte) { + t.Helper() + if _, err := inv.store.Pool().Exec(t.Context(), `update node set sent_provisioning = $2 where id = $1`, id, plans); err != nil { + t.Fatal(err) + } +} + +func mustJSON(t *testing.T, v any) string { + t.Helper() + raw, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return string(raw) +} diff --git a/internal/inventory/collection.go b/internal/inventory/collection.go index 2edf865e..d6272be0 100644 --- a/internal/inventory/collection.go +++ b/internal/inventory/collection.go @@ -458,7 +458,7 @@ func keptLately(builds []Build, planned map[string]bool, now time.Time, within t return out } -// keptLatelyOrPlanned is keptLately of every build the records hold, against the plans that have not ended. +// keptLatelyOrPlanned is keptLately of every build the records hold, (no plan names a build any more, ADR 0539 §17). func (i *Inventory) keptLatelyOrPlanned(ctx context.Context) (map[string][]string, error) { c, err := i.theChain(ctx) if err != nil { @@ -468,27 +468,7 @@ func (i *Inventory) keptLatelyOrPlanned(ctx context.Context) (map[string][]strin if err != nil { return nil, err } - plans, err := i.OpenPlans(ctx) - if err != nil { - return nil, err - } - return keptLately(all, plannedBuilds(plans), time.Now().UTC(), i.lately()), nil -} - -// plannedBuilds is every build id these plans name: a module's build, and the builds a rollout carried. -func plannedBuilds(plans []Plan) map[string]bool { - out := map[string]bool{} - for _, p := range plans { - for _, m := range p.Modules { - if m != nil && m.Build != "" { - out[m.Build] = true - } - } - for _, id := range p.Builds { - out[id] = true - } - } - return out + return keptLately(all, nil, time.Now().UTC(), i.lately()), nil } // digestIn is the `sha256:` a reference names, empty when it names none. diff --git a/internal/inventory/durations.go b/internal/inventory/durations.go deleted file mode 100644 index 5611cef8..00000000 --- a/internal/inventory/durations.go +++ /dev/null @@ -1,134 +0,0 @@ -package inventory - -import ( - "context" - "time" -) - -// The durations the core's bounds are set from (novox/hq to-be 45 Phase 0): see migration 0066. - -// The kinds of duration recorded. -const ( - DurationApply = "apply" - DurationPlanTier = "plan-tier" - DurationBuild = "build" - // DurationWalkPhase is one phase of an ended walk, per class (novox/hq ADR 0282 decision 6): subject - // "/", and "/total" for its merge to every machine running it. - DurationWalkPhase = "walk-phase" - // DurationBusy is a machine's wait for a busy container (novox/hq ADR 0335): from its first report of a - // send, which said a container waits because it is busy, to the report that says it replaced — the same - // send's ref as its `apply` row, whose detail is ApplyBusy. Its own measure, never in the apply's. - DurationBusy = "busy-wait" -) - -// ApplyBusy is the detail of an `apply` duration whose report said a container waits because it is busy: the -// machine answered, and is not yet running what it was sent (novox/hq ADR 0335). -const ApplyBusy = "busy" - -// DurationKinds are every kind, in the order `durations` shows them. -var DurationKinds = []string{DurationApply, DurationPlanTier, DurationBuild, DurationWalkPhase, - DurationBusy} - -// DurationsKeptFor is how long a duration is kept: long enough to set a bound from, and to correct it -// in Phase 1's first live week. -const DurationsKeptFor = 30 * 24 * time.Hour - -// Duration is one measurement. -type Duration struct { - Kind string `json:"kind"` - Subject string `json:"subject"` - Node string `json:"node,omitempty"` - Ref string `json:"ref"` - Started time.Time `json:"started"` - Took time.Duration `json:"took"` - Detail string `json:"detail,omitempty"` -} - -// RecordDuration keeps one measurement, once: the same thing measured again is not a second row. -func (i *Inventory) RecordDuration(ctx context.Context, d Duration) error { - if d.Took < 0 { - return nil // a clock that went back measures nothing - } - _, err := i.store.Pool().Exec(ctx, - `insert into duration (kind, subject, node, ref, started, took_ms, detail) - values ($1, $2, $3, $4, $5, $6, $7) on conflict do nothing`, - d.Kind, d.Subject, d.Node, d.Ref, d.Started, d.Took.Milliseconds(), d.Detail) - return err -} - -// RecordApplyDuration measures a machine's report of the declaration it was last sent: from the send -// to the first report of it. A report of anything else, or of a send already measured, measures -// nothing — a machine reconciling reports the same declaration every few minutes. -func (i *Inventory) RecordApplyDuration(ctx context.Context, node, declared, outcome string) error { - if declared == "" { - return nil - } - _, err := i.store.Pool().Exec(ctx, - `insert into duration (kind, subject, node, ref, started, took_ms, detail) - select $1, n.name, n.name, n.sent || '@' || to_char(n.sent_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US'), - n.sent_at, (extract(epoch from (now() - n.sent_at)) * 1000)::bigint, $4 - from node n - where n.name = $2 and n.sent = $3 and n.sent_at is not null - on conflict do nothing`, - DurationApply, node, declared, outcome) - return err -} - -// RecordBusyDuration measures a machine's wait for a busy container, at the report that says it replaced -// (novox/hq ADR 0335): from the end of its `apply` duration of the send it was last sent, which said it waited, to -// now. Nothing when that send's first report did not say a wait, and once per send. -func (i *Inventory) RecordBusyDuration(ctx context.Context, node, declared string) error { - if declared == "" { - return nil - } - _, err := i.store.Pool().Exec(ctx, - `insert into duration (kind, subject, node, ref, started, took_ms, detail) - select $1, d.subject, d.node, d.ref, d.started + d.took_ms * interval '1 millisecond', - greatest(0, (extract(epoch from (now() - (d.started + d.took_ms * interval '1 millisecond'))) * 1000)::bigint), '' - from duration d join node n on n.name = d.node - where d.kind = $2 and d.subject = $3 and d.detail = $5 and n.sent = $4 and n.sent_at is not null - and d.ref = n.sent || '@' || to_char(n.sent_at at time zone 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US') - on conflict do nothing`, - DurationBusy, DurationApply, node, declared, ApplyBusy) - return err -} - -// Durations is every measurement of a kind since a moment, oldest first; every kind when kind is empty. -func (i *Inventory) Durations(ctx context.Context, kind string, since time.Time) ([]Duration, error) { - rows, err := i.store.Pool().Query(ctx, - `select kind, subject, node, ref, started, took_ms, detail from duration - where ($1 = '' or kind = $1) and recorded >= $2 order by recorded`, kind, since) - if err != nil { - return nil, err - } - defer rows.Close() - var out []Duration - for rows.Next() { - var d Duration - var ms int64 - if err := rows.Scan(&d.Kind, &d.Subject, &d.Node, &d.Ref, &d.Started, &ms, &d.Detail); err != nil { - return nil, err - } - d.Took = time.Duration(ms) * time.Millisecond - out = append(out, d) - } - return out, rows.Err() -} - -// WalkPhasesKept says whether any phase of a walk is kept as a walk-phase duration, and whether its total is. -func (i *Inventory) WalkPhasesKept(ctx context.Context, plan string) (anyKept, totalKept bool, err error) { - err = i.store.Pool().QueryRow(ctx, - `select count(*) > 0, count(*) filter (where ref = $2) > 0 from duration where kind = $1 and ref like $3`, - DurationWalkPhase, plan+"/total", plan+"/%").Scan(&anyKept, &totalKept) - return anyKept, totalKept, err -} - -// ForgetOldDurations removes what is older than DurationsKeptFor, and says how many. -func (i *Inventory) ForgetOldDurations(ctx context.Context) (int64, error) { - tag, err := i.store.Pool().Exec(ctx, `delete from duration where recorded < $1`, - time.Now().Add(-DurationsKeptFor)) - if err != nil { - return 0, err - } - return tag.RowsAffected(), nil -} diff --git a/internal/inventory/durations_test.go b/internal/inventory/durations_test.go deleted file mode 100644 index 639866dc..00000000 --- a/internal/inventory/durations_test.go +++ /dev/null @@ -1,80 +0,0 @@ -//go:build integration - -package inventory - -import ( - "testing" - "time" -) - -// **The durations the bounds are set from are recorded once each** (novox/hq to-be 45 Phase 0): a -// send measured at its first report and not again at every reconcile that repeats it; a send of -// something else measures nothing; a new send is a new measurement. -func TestAnApplyIsMeasuredOncePerSend(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - node, err := inv.AddNode(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, node.ID, "d1", nil); err != nil { - t.Fatal(err) - } - for range 3 { // the report, then two reconciles saying the same - if err := inv.RecordApplyDuration(ctx, "anchor", "d1", OutcomeApplied); err != nil { - t.Fatal(err) - } - } - if err := inv.RecordApplyDuration(ctx, "anchor", "d0", OutcomeApplied); err != nil { - t.Fatal(err) - } - ds, err := inv.Durations(ctx, DurationApply, time.Now().Add(-time.Hour)) - if err != nil || len(ds) != 1 || ds[0].Subject != "anchor" || ds[0].Detail != OutcomeApplied || ds[0].Took < 0 { - t.Fatalf("%v %+v", err, ds) - } - time.Sleep(10 * time.Millisecond) - if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil { - t.Fatal(err) - } - if err := inv.RecordApplyDuration(ctx, "anchor", "d2", OutcomeFailed); err != nil { - t.Fatal(err) - } - if ds, _ = inv.Durations(ctx, "", time.Now().Add(-time.Hour)); len(ds) != 2 { - t.Fatalf("a second send was not measured: %+v", ds) - } -} - -// A plan's tier is measured when the plan leaves it — moving on, or ending — and only then. -func TestAPlansTierIsMeasuredWhenItIsLeft(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - p := Plan{ID: "plan-1", Repository: "novox/mesh-tools", Commit: "abc", Created: time.Now().UTC(), - State: PlanBuilding, Tiers: [][]string{{"mesh-tools"}, {"builder"}}, - Modules: map[string]*PlanModule{"mesh-tools": {}, "builder": {}}} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - p.State = PlanRolling // the same tier, saved again - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - if ds, _ := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour)); len(ds) != 0 { - t.Fatalf("a tier not left was measured: %+v", ds) - } - p.Tier = 1 - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - p.State = PlanDone - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - if err := inv.SavePlan(ctx, &p); err != nil { // saved again once ended: nothing more to measure - t.Fatal(err) - } - ds, err := inv.Durations(ctx, DurationPlanTier, time.Now().Add(-time.Hour)) - if err != nil || len(ds) != 2 || ds[0].Subject != "novox/mesh-tools" || ds[0].Ref != "plan-1/tier-0" || - ds[1].Ref != "plan-1/tier-1" { - t.Fatalf("%v %+v", err, ds) - } -} diff --git a/internal/inventory/gate.go b/internal/inventory/gate.go deleted file mode 100644 index f83e09c7..00000000 --- a/internal/inventory/gate.go +++ /dev/null @@ -1,335 +0,0 @@ -package inventory - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "strings" - "time" - - "github.com/jackc/pgx/v5" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// The gate's verdicts (novox/hq ADR 0236, to-be 45 §8): what a build did on its first machine, and, -// for one that failed there, how it was put back. One row per build, written by the plan that rolled it -// out, under the lease. - -// The gate's verdicts and a failed build's rollback. -const ( - GatePassed = "passed" - GateFailed = "failed" - // GateSuperseded is a judging ended by a later send to the judged machine that moved the module to - // another build (novox/hq issue 352): no verdict on the build, and nothing put back. - GateSuperseded = "superseded" - - RollingBack = "rolling-back" - RolledBack = "rolled-back" - NotRolledBack = "not-rolled-back" -) - -// GateVerdict is one build's verdict at its gate. -type GateVerdict struct { - Build string - Module string - Commit string - Previous string - Plan string - Machines []string - Verdict string - Rollback string - Why string - Component string - // JudgingFrom is when the first machine reported the build applied and the judging began. - JudgingFrom *time.Time - JudgedAt time.Time - Epoch uint64 -} - -// RecordGate writes a build's verdict. A build that passed on one machine may still fail on the next one -// judged; **a failed build's row is written once**: any later verdict for it is refused with ErrGateKept, which is what keeps a rollback to one per build — the row -// is written before the rollback's send, and a controller replaced in between finds it. -func (i *Inventory) RecordGate(ctx context.Context, v GateVerdict) error { - epoch, err := i.actingEpoch(ctx) - if err != nil { - return fmt.Errorf("the gate's verdict on %s is not written: %w", v.Build, err) - } - if v.Machines == nil { - v.Machines = []string{} - } - tag, err := i.store.Pool().Exec(ctx, - `insert into build_gate (build, module, commit_hash, previous, plan, machines, verdict, rollback, why, - component, judging_from, judged_at, epoch) - values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now(), $12) - on conflict (build) do update set verdict = excluded.verdict, rollback = excluded.rollback, - why = excluded.why, previous = excluded.previous, machines = excluded.machines, - judged_at = now(), epoch = excluded.epoch - where build_gate.verdict = 'passed'`, - v.Build, v.Module, v.Commit, v.Previous, v.Plan, v.Machines, v.Verdict, v.Rollback, v.Why, v.Component, - v.JudgingFrom, epoch) - if err != nil { - return err - } - if tag.RowsAffected() == 0 { - return fmt.Errorf("%w: %s", ErrGateKept, v.Build) - } - return nil -} - -// ErrGateKept is a verdict already kept for the build, which is not written over. -var ErrGateKept = errors.New("this build's verdict at its gate is already kept") - -// SetRollback records how a failed build's rollback went. -func (i *Inventory) SetRollback(ctx context.Context, build, rollback, why string) error { - if _, err := i.actingEpoch(ctx); err != nil { - return err - } - _, err := i.store.Pool().Exec(ctx, - `update build_gate set rollback = $2, why = $3, judged_at = now() where build = $1 and verdict = 'failed'`, - build, rollback, why) - return err -} - -// GateOf is a build's verdict, and whether it has one. -func (i *Inventory) GateOf(ctx context.Context, build string) (GateVerdict, bool, error) { - rows, err := i.store.Pool().Query(ctx, gateSelect+` where build = $1`, build) - if err != nil { - return GateVerdict{}, false, err - } - list, err := scanGates(rows) - if err != nil || len(list) == 0 { - return GateVerdict{}, false, err - } - return list[0], true, nil -} - -// GateFailed is whether a build failed its gate: one the mesh never registers or sends again on its own. -func (i *Inventory) GateFailed(ctx context.Context, build string) (bool, error) { - v, found, err := i.GateOf(ctx, build) - return found && v.Verdict == GateFailed, err -} - -// LatestGates is the newest verdict of every module that has one: what the gate probe (DG) reads. -func (i *Inventory) LatestGates(ctx context.Context) ([]GateVerdict, error) { - rows, err := i.store.Pool().Query(ctx, `select distinct on (module) build, module, commit_hash, previous, plan, - machines, verdict, rollback, why, component, judging_from, judged_at, coalesce(epoch, 0) - from build_gate order by module, judged_at desc`) - if err != nil { - return nil, err - } - return scanGates(rows) -} - -// Gates is the newest verdicts, newest first: what `plans gates` lists. -func (i *Inventory) Gates(ctx context.Context, limit int) ([]GateVerdict, error) { - rows, err := i.store.Pool().Query(ctx, gateSelect+` order by judged_at desc limit $1`, limit) - if err != nil { - return nil, err - } - return scanGates(rows) -} - -const gateSelect = `select build, module, commit_hash, previous, plan, machines, verdict, rollback, why, component, - judging_from, judged_at, coalesce(epoch, 0) from build_gate` - -func scanGates(rows pgx.Rows) ([]GateVerdict, error) { - defer rows.Close() - var out []GateVerdict - for rows.Next() { - var v GateVerdict - var epoch int64 - if err := rows.Scan(&v.Build, &v.Module, &v.Commit, &v.Previous, &v.Plan, &v.Machines, &v.Verdict, - &v.Rollback, &v.Why, &v.Component, &v.JudgingFrom, &v.JudgedAt, &epoch); err != nil { - return nil, err - } - v.Epoch = uint64(epoch) - out = append(out, v) - } - return out, rows.Err() -} - -// PreviousBuild is the build a module goes back to when a build of it fails its gate: the newest build -// that worked, made from the commit the first machine ran before, asked before the failed one, and not -// itself failed at a gate. Among the builds whose artifacts the mesh keeps (KeptBuilds): an older one -// may already be gone from the artifact store. False when there is none to go back to. -func (i *Inventory) PreviousBuild(ctx context.Context, module, commit string, failed Build) (Build, bool, error) { - builds, err := i.Builds(ctx, module, 50) - if err != nil { - return Build{}, false, err - } - kept := 0 - for _, b := range builds { - if !b.Worked() { - continue - } - kept++ - if kept > KeptBuilds { - break - } - if b.ID == failed.ID || (commit != "" && b.Commit != commit) { - continue - } - // Only a build of the repository the failed build was made from — the module's, since its take-in - // registered it (novox/hq ADR 0266): a build of the module's name from another repository is recorded - // and was never registered, and putting it back would register it now. - if failed.Repository != "" && !sameRepositoryAs(b.Repository, failed.Repository) { - continue - } - if !failed.AskedOrAt().IsZero() && !b.AskedOrAt().Before(failed.AskedOrAt()) { - continue - } - if bad, err := i.GateFailed(ctx, b.ID); err != nil { - return Build{}, false, err - } else if bad { - continue - } - manifest, err := i.manifestOf(ctx, b.ID) - if err != nil { - return Build{}, false, err - } - if len(manifest) == 0 || string(manifest) == "null" { - continue - } - b.Manifest = manifest - return b, true, nil - } - return Build{}, false, nil -} - -// sameRepositoryAs says two recorded repositories are one, however their case or `.git` is spelled. -func sameRepositoryAs(a, b string) bool { - trim := func(s string) string { - return strings.TrimSuffix(strings.TrimRight(strings.ToLower(strings.TrimSpace(s)), "/"), ".git") - } - return trim(a) == trim(b) -} - -// RestoreModule puts a module's registered build back to an earlier one: its manifest, the commit it -// was built from, and when it was asked — as now, so the build that failed its gate, asked before, can -// never register over it again (issue 219's order). The source's head is left where the merge moved it: -// the module IS behind its source, and `status` says so. -func (i *Inventory) RestoreModule(ctx context.Context, b Build) error { - if _, err := i.actingEpoch(ctx); err != nil { - return fmt.Errorf("%s is not put back: %w", b.Module, err) - } - m, err := catalogue.ParseManifest(b.Manifest) - if err != nil { - return fmt.Errorf("%s's build %s is not a manifest the mesh can register again: %w", b.Module, b.ID, err) - } - raw, err := json.Marshal(m) - if err != nil { - return err - } - tag, err := i.store.Pool().Exec(ctx, - `update module set manifest = $2, version = nullif($3, ''), built_from = nullif($4, ''), - built_asked = now(), registered = now() - where name = $1`, b.Module, raw, m.Version, b.Commit) - if err != nil { - return err - } - if tag.RowsAffected() == 0 { - return fmt.Errorf("%w: %s", ErrNoSuchModule, b.Module) - } - return nil -} - -// BuildFingerprints is, per commit, what the newest successful build of a module from it would put on a -// machine — its artifacts and its manifest, hashed — so a rebuild that changes nothing there is told -// from one that does (the bus's planned step, ADR 0236). -func (i *Inventory) BuildFingerprints(ctx context.Context, module string) (map[string]string, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - return i.chainBuildFingerprints(ctx, c, module) -} - -// Fingerprints is BuildFingerprints for every module at once: module → commit → fingerprint. -func (i *Inventory) Fingerprints(ctx context.Context) (map[string]map[string]string, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - return i.chainFingerprints(ctx, c) -} - -// PassedCommits is, per module, the commits a build of which passed its gate on some machine. -func (i *Inventory) PassedCommits(ctx context.Context) (map[string]map[string]bool, error) { - rows, err := i.store.Pool().Query(ctx, `select module, commit_hash from build_gate where verdict = 'passed'`) - if err != nil { - return nil, err - } - defer rows.Close() - out := map[string]map[string]bool{} - for rows.Next() { - var module, commit string - if err := rows.Scan(&module, &commit); err != nil { - return nil, err - } - if out[module] == nil { - out[module] = map[string]bool{} - } - out[module][commit] = true - } - return out, rows.Err() -} - -// BuildOf is the id of the newest successful build of a module from a commit; empty when none is -// recorded (a manifest handed over by hand). -func (i *Inventory) BuildOf(ctx context.Context, module, commit string) (string, error) { - c, err := i.theChain(ctx) - if err != nil { - return "", err - } - return i.chainBuildOf(ctx, c, module, commit) -} - -// ManifestAt is the manifest a module was registered with at a commit (or a commit it abbreviates): the -// newest successful build from it, with the artifacts of the build standing for it when its source was -// unchanged (issue 280) — what a machine last sent that build runs. False when no such build, or none -// with a manifest, is kept (novox/hq issue 295: a recorded module is composed at the build its machine -// runs until a person's push moves it). -func (i *Inventory) ManifestAt(ctx context.Context, module, commit string) (catalogue.Manifest, bool, error) { - if commit == "" { - return catalogue.Manifest{}, false, nil - } - id, raw, found, err := i.newestManifestAt(ctx, module, commit) - if err != nil { - return catalogue.Manifest{}, false, err - } - if !found { - return catalogue.Manifest{}, false, nil - } - if stands, same, err := i.StandingBuild(ctx, module, id); err != nil { - return catalogue.Manifest{}, false, err - } else if stands != "" && stands != id && len(same) > 0 { - raw = same - } - m, err := catalogue.ParseManifest(raw) - if err != nil { - return catalogue.Manifest{}, false, fmt.Errorf("%s's build from %s is not a manifest the mesh can compose: %w", - module, commit, err) - } - return m, true, nil -} - -// manifestOf is the manifest a build was recorded with; nil when it kept none. -func (i *Inventory) manifestOf(ctx context.Context, id string) ([]byte, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - return i.chainManifestOf(ctx, c, id) -} - -// newestManifestAt is the newest successful build of a module from a commit, or one it abbreviates, that kept a manifest: -// its id and manifest; false when there is none. -func (i *Inventory) newestManifestAt(ctx context.Context, module, commit string) (string, []byte, bool, error) { - c, err := i.theChain(ctx) - if err != nil { - return "", nil, false, err - } - return i.chainManifestBuild(ctx, c, module, commit) -} diff --git a/internal/inventory/heartbeat_gap_gone_test.go b/internal/inventory/heartbeat_gap_gone_test.go deleted file mode 100644 index 1fcce32c..00000000 --- a/internal/inventory/heartbeat_gap_gone_test.go +++ /dev/null @@ -1,12 +0,0 @@ -package inventory - -import "testing" - -// 1183/AC3: the heartbeat-gap duration is no kind the mesh measures. -func TestNoHeartbeatGapIsMeasured(t *testing.T) { // 1183/AC3 - for _, k := range DurationKinds { - if k == "heartbeat-gap" { - t.Fatalf("%s is still a kind of duration: %v", k, DurationKinds) - } - } -} diff --git a/internal/inventory/heldhere_test.go b/internal/inventory/heldhere_test.go deleted file mode 100644 index fe812c12..00000000 --- a/internal/inventory/heldhere_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package inventory - -import ( - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" -) - -// novox/hq issue 218: a seat held once for the mesh is granted and issued only to the holder on record; -// a node seat to every machine's claimant; a mesh seat with no holder on record as derived. -func TestOnlyTheRecordedHolderHoldsAMeshSeat(t *testing.T) { - claimed := []broker.Seat{ - {Name: "mesh-store", Scope: catalogue.ScopeMesh}, - {Name: "node-packet-filter", Scope: catalogue.ScopeNode}, - {Name: "unrecorded", Scope: catalogue.ScopeMesh}, - } - holdings := []catalogue.Held{{Claim: "mesh-store", Scope: catalogue.ScopeMesh, Node: "control", Module: "postgres"}} - names := func(ss []broker.Seat) (out []string) { - for _, s := range ss { - out = append(out, s.Name) - } - return - } - if got := names(heldHere(claimed, holdings, "control", "postgres")); len(got) != 3 { - t.Errorf("the holder lost a seat: %v", got) - } - got := names(heldHere(claimed, holdings, "other", "postgres")) - if len(got) != 2 || got[0] != "node-packet-filter" || got[1] != "unrecorded" { - t.Errorf("a claimant on another machine holds %v; want the node seat and the unrecorded one, not the store", got) - } -} diff --git a/internal/inventory/hold.go b/internal/inventory/hold.go deleted file mode 100644 index 272a40c7..00000000 --- a/internal/inventory/hold.go +++ /dev/null @@ -1,149 +0,0 @@ -package inventory - -import ( - "context" - "errors" - "fmt" - "slices" - "strings" - "sync" - "time" -) - -// ErrNodeBusy is a node another act kept holding for longer than a caller waits. -var ErrNodeBusy = errors.New("another act is composing or sending that node's declaration") - -// HoldWaitFor is how long HoldNodes waits for a node another act holds, and HoldPoll how often it -// looks again. Variables so a test need not wait minutes. -var ( - HoldWaitFor = 2 * time.Minute - HoldPoll = 250 * time.Millisecond -) - -// HoldNodes serialises composing and sending a declaration per node (novox/hq ADR 0100): while -// one caller holds a node, another asking for it waits. Without it a push that composed a node as -// adopted could send that declaration after `converge --yes` sent the converged one, and the node -// would return to adopted with nobody having asked. -// -// Session-level advisory locks on one connection, all or none: a set not wholly free is given back -// at once, so two callers holding overlapping sets never each wait on the other. **A waiter pins -// no connection.** It looks again every HoldPoll with a connection borrowed for the look, and gives -// up after HoldWaitFor with ErrNodeBusy naming the node — so a stuck holder costs the pool one -// connection, never one per caller queued behind it. Release gives every one back, and may be -// called more than once. -func (i *Inventory) HoldNodes(ctx context.Context, names []string) (func(), error) { - sorted := slices.Clone(names) - slices.Sort(sorted) - sorted = slices.Compact(sorted) - deadline := time.Now().Add(HoldWaitFor) - for { - release, busy, err := i.tryHold(ctx, sorted) - if err != nil || busy == "" { - return release, err - } - if time.Now().After(deadline) { - return nil, fmt.Errorf("%w: %s has been held for over %s — try again once it is done", - ErrNodeBusy, busy, HoldWaitFor) - } - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(HoldPoll): - } - } -} - -// tryHold takes every named node's lock or none, and says which node was busy when it took none. -func (i *Inventory) tryHold(ctx context.Context, sorted []string) (func(), string, error) { - conn, err := i.store.Pool().Acquire(ctx) - if err != nil { - return nil, "", err - } - var once sync.Once - release := func() { - once.Do(func() { - // Unlocking all of this session's advisory locks, then handing the connection back: a - // connection returned still holding one would hold it for whoever borrows it next. - _, err := conn.Exec(context.WithoutCancel(ctx), `select pg_advisory_unlock_all()`) - if err != nil { - // The session's locks die with the session: close it rather than return it. - _ = conn.Conn().Close(context.WithoutCancel(ctx)) - } - conn.Release() - }) - } - for _, name := range sorted { - var took bool - if err := conn.QueryRow(ctx, - `select pg_try_advisory_lock(hashtext('mesh-node-declaration:' || $1)::bigint)`, - name).Scan(&took); err != nil { - release() - return nil, "", err - } - if !took { - release() - return nil, strings.TrimSpace(name), nil - } - } - return release, "", nil -} - -// ErrPlansBusy is the plans held by another act — on a machine replacing its controller, the other -// controller — for longer than a caller waits, or at all for one that does not wait. -var ErrPlansBusy = errors.New("another controller is working the plans") - -// HoldPlans makes working the plans one act at a time, across every controller on the store -// (novox/hq issue 213). A plan is read, changed and written whole; two controllers doing that at -// once — the old and the new for the moment a machine hands its controller over, or a controller -// and a person's `plans stop` — each act on what the other has not saved yet: a tier asked twice, -// an outcome written over. A session-level advisory lock on one connection, released by the -// returned function and by the session ending, so a controller that dies holding it holds nothing. -// -// wait false gives ErrPlansBusy at once when another holds them — the timer's way: the holder is -// moving the plans already; and a build outcome's on the event loop, which the build records keep -// for the look to settle (novox/hq issue 1708). wait true looks again every HoldPoll for up to -// HoldWaitFor — a merge's way, which must be written. -func (i *Inventory) HoldPlans(ctx context.Context, wait bool) (func(), error) { - deadline := time.Now().Add(HoldWaitFor) - for { - release, took, err := i.tryLock(ctx, "mesh-plans") - if err != nil || took { - return release, err - } - if !wait || time.Now().After(deadline) { - return nil, ErrPlansBusy - } - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(HoldPoll): - } - } -} - -// tryLock takes one named advisory lock on a connection of its own, or gives the connection back. -func (i *Inventory) tryLock(ctx context.Context, key string) (func(), bool, error) { - conn, err := i.store.Pool().Acquire(ctx) - if err != nil { - return nil, false, err - } - var once sync.Once - release := func() { - once.Do(func() { - if _, err := conn.Exec(context.WithoutCancel(ctx), `select pg_advisory_unlock_all()`); err != nil { - _ = conn.Conn().Close(context.WithoutCancel(ctx)) - } - conn.Release() - }) - } - var took bool - if err := conn.QueryRow(ctx, `select pg_try_advisory_lock(hashtext($1)::bigint)`, key).Scan(&took); err != nil { - release() - return nil, false, err - } - if !took { - release() - return nil, false, nil - } - return release, true, nil -} diff --git a/internal/inventory/hold_plans_test.go b/internal/inventory/hold_plans_test.go deleted file mode 100644 index daa4744a..00000000 --- a/internal/inventory/hold_plans_test.go +++ /dev/null @@ -1,40 +0,0 @@ -//go:build integration - -package inventory - -import ( - "errors" - "testing" - "time" -) - -// novox/hq issue 213: while a machine hands its controller over from the container to the process, -// two controllers run on one store for a moment. Working the plans is one act at a time across them. -func TestThePlansAreWorkedByOneControllerAtATime(t *testing.T) { - first := ForTest(t) - // A second controller: its own connections to the same store. - second, err := Open(t.Context()) - if err != nil { - t.Fatal(err) - } - t.Cleanup(second.Close) - - release, err := first.HoldPlans(t.Context(), false) - if err != nil { - t.Fatalf("the plans could not be held when nobody held them: %v", err) - } - t.Cleanup(release) // a pool closing waits for a connection still held; release is idempotent - if _, err := second.HoldPlans(t.Context(), false); !errors.Is(err, ErrPlansBusy) { - t.Fatalf("a second controller held the plans while the first did: %v", err) - } - // A waiter gets them once they are let go. - was := HoldPoll - HoldPoll = 10 * time.Millisecond - defer func() { HoldPoll = was }() - go func() { time.Sleep(50 * time.Millisecond); release() }() - again, err := second.HoldPlans(t.Context(), true) - if err != nil { - t.Fatalf("a waiting controller never got the plans once they were let go: %v", err) - } - again() -} diff --git a/internal/inventory/hold_test.go b/internal/inventory/hold_test.go deleted file mode 100644 index 19cc4132..00000000 --- a/internal/inventory/hold_test.go +++ /dev/null @@ -1,93 +0,0 @@ -//go:build integration - -package inventory - -import ( - "errors" - "strings" - "testing" - "time" -) - -// Composing and sending one node's declaration is serialised: a second holder waits for the first. -func TestHoldingANodeMakesTheNextHolderWait(t *testing.T) { - inv := fresh(t) - ctx := t.Context() - release, err := inv.HoldNodes(ctx, []string{"anchor", "laptop"}) - if err != nil { - t.Fatal(err) - } - got := make(chan func(), 1) - go func() { - second, err := inv.HoldNodes(ctx, []string{"laptop"}) - if err != nil { - t.Error(err) - got <- func() {} - return - } - got <- second - }() - select { - case <-got: - t.Fatal("a node held by one caller was held by another at the same time") - case <-time.After(300 * time.Millisecond): - } - // Another node is not held up. - other, err := inv.HoldNodes(ctx, []string{"joiner"}) - if err != nil { - t.Fatal(err) - } - other() - release() - select { - case second := <-got: - second() - case <-time.After(5 * time.Second): - t.Fatal("releasing the node did not let the next holder in") - } -} - -// A waiter pins no pool connection while it waits, and gives up after a bounded wait saying which -// node is busy. -func TestAWaiterPinsNoConnectionAndGivesUp(t *testing.T) { - inv := fresh(t) - ctx := t.Context() - release, err := inv.HoldNodes(ctx, []string{"anchor"}) - if err != nil { - t.Fatal(err) - } - defer release() - savedWait, savedPoll := HoldWaitFor, HoldPoll - HoldWaitFor, HoldPoll = 1500*time.Millisecond, 50*time.Millisecond - defer func() { HoldWaitFor, HoldPoll = savedWait, savedPoll }() - - pool := inv.store.Pool() - base := pool.Stat().AcquiredConns() - const waiters = 3 - done := make(chan error, waiters) - for range waiters { - go func() { - _, err := inv.HoldNodes(ctx, []string{"anchor"}) - done <- err - }() - } - // While they wait, the pool lends nothing to them for longer than a look: **pinned is held throughout**, - // so at some moment of the wait fewer than all of them hold one. Asked of the least seen over many - // looks, not of the most: a look that lands while every waiter is mid-query is a busy machine, not a - // pinned connection, and on a loaded one (the suite's packages at once, the race detector) it did land. - least := waiters - for range 100 { - time.Sleep(10 * time.Millisecond) - if n := int(pool.Stat().AcquiredConns() - base); n < least { - least = n - } - } - if least >= waiters { - t.Fatalf("%d waiters held %d connections at every look across their wait", waiters, least) - } - for range waiters { - if err := <-done; !errors.Is(err, ErrNodeBusy) || !strings.Contains(err.Error(), "anchor") { - t.Fatalf("a waiter did not give up naming the busy node: %v", err) - } - } -} diff --git a/internal/inventory/migrations/0108-the-plans-leave-the-store.sql b/internal/inventory/migrations/0108-the-plans-leave-the-store.sql new file mode 100644 index 00000000..6d3ab18c --- /dev/null +++ b/internal/inventory/migrations/0108-the-plans-leave-the-store.sql @@ -0,0 +1,16 @@ +-- The tables of the delivery plans' machinery leave the store (novox/hq ADR 0539 §17, design 47 release 6, issue 1945). +-- +-- A machine follows its wants and applies a module from the composition the controller writes for it (ADR 0539 §1, §2), so +-- the controller keeps no record of what it sent, no counter of the assignments a send was composed from, no verdict of a +-- gate that judged a build on its first machine and no measure of how long a plan's tier or a machine's apply took. This +-- drops the four tables that kept them: declaration_send (0094), assignment_generation (0094), build_gate (0073) and duration +-- (0066, 0075). +-- +-- **It is one-way**: a controller older than this schema is refused by the put-back gate (the highest migration it carries is +-- recorded by its version, issue 352). What stays: the node row and its columns (the digest of the last send, which the old +-- reads of `status` still name until they go with their verbs), the bus step, the core box and the refused registrations. +-- The function raise_assignment_generation and its triggers went with 0105. +drop table if exists declaration_send; +drop table if exists assignment_generation; +drop table if exists build_gate; +drop table if exists duration; diff --git a/internal/inventory/nodes.go b/internal/inventory/nodes.go index 0014e244..ebc6ac2d 100644 --- a/internal/inventory/nodes.go +++ b/internal/inventory/nodes.go @@ -37,8 +37,6 @@ type Inventory struct { // chain is the controller's chain on the bus (novox/hq ADR 0408), once attached (UseChain). chain *chain.Chain state *chainState - // recorded, when set, takes the durations a save of the chain measures in place of the store: a test with no store. - recorded func(context.Context, Duration) error // throwaway is set on an inventory raised for a moment beside the real mesh (the merge gate's): nothing it // composes is to be written to the store or the bus. throwaway bool @@ -73,7 +71,6 @@ func Open(ctx context.Context) (*Inventory, error) { } func (i *Inventory) Close() { - i.stopViews() i.store.Close() } @@ -1283,8 +1280,9 @@ func (i *Inventory) RecordSentUnder(ctx context.Context, node, digest string, bu `update node set sent = $2, sent_at = now(), sent_builds = null, sent_epoch = $3 where id = $1`, node, digest, sentEpoch); err != nil { return err } - // What it was sent of each module is the assignments' (SentBuilds); a write of it that fails fails this record. - return i.recordRuns(ctx, node, builds, time.Now()) + // What it was sent of each module is no longer kept in the assignments: they hold placement only (ADR 0539 §12). + _ = builds + return nil } // AwaitingSince is since when a machine has had something waiting for its next send (novox/hq issue @@ -1301,36 +1299,8 @@ func (i *Inventory) AwaitingSince(ctx context.Context, name string) (time.Time, // assigned module and one only pulled in (ADR 0408, issue 1109). Known is false when that was not kept: a machine never sent // anything, or last sent a declaration by hand, which makes every key of it say its commit unknown. func (i *Inventory) SentBuilds(ctx context.Context, name string) (builds map[string]string, known bool, err error) { - var sentAt *time.Time - err = i.store.Pool().QueryRow(ctx, `select sent_at from node where name = $1`, name).Scan(&sentAt) - if errors.Is(err, pgx.ErrNoRows) { - return nil, false, nil - } - if err != nil || sentAt == nil { - return nil, false, err - } - c, err := i.theChain(ctx) - if err != nil { - return nil, false, err - } - all, err := c.Assignments.All(ctx) - if err != nil { - return nil, false, fmt.Errorf("what %s was sent cannot be read from the bus: %w", name, err) - } - builds = map[string]string{} - for _, k := range all { - a := k.Value - if a.Node != name || !a.Stands() { - continue - } - if !a.SentUnknown.IsZero() { - return nil, false, nil - } - if a.Commit != "" { - builds[a.Module] = a.Commit - } - } - return builds, true, nil + // Never known: the assignments hold placement only (ADR 0539 §12), and what a machine runs is its have. + return nil, false, nil } // RecordSentBusUsers keeps a digest of the bus's user list a machine was just sent, by its name diff --git a/internal/inventory/older_build_test.go b/internal/inventory/older_build_test.go deleted file mode 100644 index eab509ef..00000000 --- a/internal/inventory/older_build_test.go +++ /dev/null @@ -1,126 +0,0 @@ -//go:build integration - -package inventory - -import ( - "context" - "errors" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// novox/hq 04-ISSUES/219: two builds of one module in flight together, the one asked first heard -// last. The newer request stood on the newer base; the older one's late outcome is recorded and is -// not what the module is. - -func postgresBuild(id string, asked time.Time, image string) Build { - b := aBuild(id, "postgres", "") - b.Asked = asked - b.Against = []string{"mesh-tools/runtime@sha256:" + id} - b.Made = []Artifact{{Name: "server", Kind: "image", Reference: "postgres@sha256:" + image}} - return b -} - -func TestAnOlderRequestFinishingLaterIsNotWhatTheModuleHolds(t *testing.T) { - inv := fresh(t) - ctx := context.Background() - older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC) - newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC) - - // The newer request finishes first, the older one last — recorded in that order. - if err := inv.RecordBuild(ctx, postgresBuild("newer", newer, "4bcd5f73")); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuild(ctx, postgresBuild("older", older, "0ab07fa9")); err != nil { - t.Fatal(err) - } - - held, err := inv.Held(ctx) - if err != nil { - t.Fatal(err) - } - if got := held["postgres/server"]; got != "postgres@sha256:4bcd5f73" { - t.Errorf("postgres holds %q; want the newer request's image 4bcd5f73", got) - } - against, err := inv.BuiltAgainst(ctx) - if err != nil { - t.Fatal(err) - } - if got := against["postgres"]; len(got) != 1 || got[0] != "mesh-tools/runtime@sha256:newer" { - t.Errorf("postgres stands on %v; want what the newer request stood on", got) - } - - // Both are still recorded, the late one first as what happened lately. - builds, err := inv.Builds(ctx, "postgres", 5) - if err != nil { - t.Fatal(err) - } - if len(builds) != 2 || builds[0].ID != "older" || !builds[0].Asked.Equal(older) { - t.Fatalf("both builds, newest heard first, with when they were asked: %+v", builds) - } -} - -func TestABuildWithNoKnownRequestTimeIsOrderedByWhenItWasRecorded(t *testing.T) { - // What the mesh did before it kept the request time, so a row from before still answers. - inv := fresh(t) - ctx := context.Background() - for _, id := range []string{"first", "second"} { - b := aBuild(id, "shell", "") - b.Made = []Artifact{{Name: "config", Kind: "archive", Reference: "…/" + id}} - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - held, err := inv.Held(ctx) - if err != nil { - t.Fatal(err) - } - if got := held["shell/config"]; got != "…/second" { - t.Errorf("shell holds %q; want the one recorded last", got) - } -} - -func TestARegistrationFromAnOlderRequestDoesNotReplaceANewerOne(t *testing.T) { - inv := fresh(t) - ctx := context.Background() - older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC) - newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC) - from := func(asked time.Time) Source { - return Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/postgres", - BuiltFrom: "efff5415", Asked: asked} - } - - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "fixed"}, from(newer)); err != nil { - t.Fatal(err) - } - err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "stale"}, from(older)) - if !errors.Is(err, ErrSuperseded) { - t.Fatalf("an older request's registration was not refused as superseded: %v", err) - } - shelf, err := inv.Catalogue(ctx) - if err != nil { - t.Fatal(err) - } - if got := shelf["postgres"].Version; got != "fixed" { - t.Fatalf("postgres is %q; want the newer request's manifest", got) - } - - // A later request, and a manifest handed over by hand — asked when it is handed over — both - // replace it as before. - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "later"}, - from(newer.Add(time.Minute))); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "postgres", Version: "by-hand"}, Source{}); err != nil { - t.Fatal(err) - } - if shelf, _ := inv.Catalogue(ctx); shelf["postgres"].Version != "by-hand" { - t.Fatalf("postgres is %q; want the manifest handed over by hand", shelf["postgres"].Version) - } - src, err := inv.SourceOf(ctx, "postgres") - if err != nil || src.Repository != "novox/mesh-catalog" { - t.Fatalf("a hand registration erased the provenance: %+v %v", src, err) - } -} diff --git a/internal/inventory/order_test.go b/internal/inventory/order_test.go deleted file mode 100644 index 910d783a..00000000 --- a/internal/inventory/order_test.go +++ /dev/null @@ -1,128 +0,0 @@ -//go:build integration - -package inventory - -import ( - "context" - "errors" - "testing" - "time" -) - -// Order and one writer, as the store keeps them (novox/hq to-be 45 §6). - -// **A plan is written by compare-and-set on its revision, carrying the epoch**: a write against a plan -// another writer moved since is refused, and nothing is written by a process that may not act. -func TestAPlanIsWrittenByCompareAndSetUnderTheLease(t *testing.T) { - inv := ForTest(t) - ctx := context.Background() - epoch := uint64(57) - inv.ActsUnder(func(context.Context) (uint64, error) { return epoch, nil }) - - p := Plan{ID: "plan-cas", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(), State: PlanBuilding, - Tiers: [][]string{{"app"}}, Modules: map[string]*PlanModule{"app": {}}} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - if p.Revision != 1 || p.Epoch != 57 { - t.Fatalf("a new plan was written at revision %d, epoch %d", p.Revision, p.Epoch) - } - // Two readers of revision 1: the first write wins, the second is refused and writes nothing. - first, err := inv.PlanByID(ctx, "plan-cas") - if err != nil { - t.Fatal(err) - } - second := first - first.Note = "the newer word" - if err := inv.SavePlan(ctx, &first); err != nil { - t.Fatal(err) - } - second.State = PlanFailed - if err := inv.SavePlan(ctx, &second); !errors.Is(err, ErrPlanMoved) { - t.Fatalf("a write against a plan moved since it was read was not refused: %v", err) - } - kept, _ := inv.PlanByID(ctx, "plan-cas") - if kept.State != PlanBuilding || kept.Note != "the newer word" || kept.Revision != 2 { - t.Fatalf("the refused write reached the plan: %+v", kept) - } - // The writer saves its own plan again without reading it: its revision moved with its write. - first.Tier = 0 - if err := inv.SavePlan(ctx, &first); err != nil { - t.Fatalf("a writer could not save its own plan twice: %v", err) - } - // A new plan under an id already written is refused, not laid over it. - again := Plan{ID: "plan-cas", Repository: "novox/app", Commit: "deadbeef", Created: time.Now(), State: PlanBuilding} - if err := inv.SavePlan(ctx, &again); !errors.Is(err, ErrPlanMoved) { - t.Fatalf("a second plan under one id was written: %v", err) - } - // And a process that does not hold the lease writes nothing. - inv.ActsUnder(func(context.Context) (uint64, error) { return 0, errors.New("this controller lost the lease") }) - first.Note = "from a controller that lost the lease" - if err := inv.SavePlan(ctx, &first); err == nil { - t.Fatal("a plan was written by a controller that does not hold the lease") - } - if kept, _ := inv.PlanByID(ctx, "plan-cas"); kept.Note != "the newer word" || kept.Epoch != 57 { - t.Fatalf("a controller without the lease wrote the plan: %+v", kept) - } -} - -// The epochs the mesh issued: the highest is the floor; taking one ends every earlier one nobody gave -// back as found expired, and says which; one given back says so and is not said again. -func TestTheEpochsIssuedAreKept(t *testing.T) { - inv := ForTest(t) - ctx := context.Background() - if highest, err := inv.HighestEpoch(ctx); err != nil || highest != 0 { - t.Fatalf("a mesh that issued none has %d (%v)", highest, err) - } - if _, err := inv.TookEpoch(ctx, Epoch{Epoch: 41, Instance: "a", Taken: time.Now()}); err != nil { - t.Fatal(err) - } - // a stopped renewing and said nothing; b takes the lease. - expired, err := inv.TookEpoch(ctx, Epoch{Epoch: 57, Instance: "b", Taken: time.Now()}) - if err != nil || len(expired) != 1 || expired[0].Epoch != 41 || expired[0].How != EpochExpired { - t.Fatalf("taking 57 ended %+v (%v), want 41 found expired", expired, err) - } - if err := inv.EndEpoch(ctx, 57, EpochReleased); err != nil { - t.Fatal(err) - } - expired, err = inv.TookEpoch(ctx, Epoch{Epoch: 60, Instance: "c", Taken: time.Now()}) - if err != nil || len(expired) != 0 { - t.Fatalf("a lease given back was found expired: %+v (%v)", expired, err) - } - if highest, _ := inv.HighestEpoch(ctx); highest != 60 { - t.Fatalf("the highest epoch issued is %d, want 60", highest) - } - e, found, err := inv.EpochOf(ctx, 57) - if err != nil || !found || e.Instance != "b" || e.How != EpochReleased || e.Ended == nil { - t.Fatalf("epoch 57 reads %+v (%v, %v)", e, found, err) - } - recent, err := inv.EpochsSince(ctx, time.Now().Add(-time.Hour)) - if err != nil || len(recent) != 3 || recent[0].Epoch != 60 || recent[0].Ended != nil { - t.Fatalf("the epochs of the last hour read %+v (%v)", recent, err) - } -} - -// What a machine was sent under, and whether it reads an epoch. -func TestTheEpochAMachineWasSentIsKept(t *testing.T) { - inv := ForTest(t) - ctx := context.Background() - node, err := inv.AddNode(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - if err := inv.RecordSentUnder(ctx, node.ID, "d1", nil, 57); err != nil { - t.Fatal(err) - } - if e, err := inv.SentEpoch(ctx, node.ID); err != nil || e != 57 { - t.Fatalf("sent under %d (%v)", e, err) - } - if err := inv.RecordSent(ctx, node.ID, "d2", nil); err != nil { - t.Fatal(err) - } - if e, _ := inv.SentEpoch(ctx, node.ID); e != 0 { - t.Fatalf("a declaration sent without an epoch left %d kept", e) - } - if reads, _ := inv.ReadsEpoch(ctx, node.ID); reads { - t.Fatal("a machine that never said so reads an epoch") - } -} diff --git a/internal/inventory/plan_backups_test.go b/internal/inventory/plan_backups_test.go deleted file mode 100644 index 40a9f301..00000000 --- a/internal/inventory/plan_backups_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package inventory - -import ( - "encoding/json" - "reflect" - "strings" - "testing" - "time" -) - -// The plan's backups are the contract mesh-delivery reads from `plan-moved` (issue 1065): their JSON field names -// are fixed here, and a plan keeps them through a round trip. -func TestAPlanKeepsItsBackupsThroughItsJSONUnderTheNamesTheDeliveryReads(t *testing.T) { - at := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC) - p := Plan{ID: "p", Backups: &PlanBackups{State: BackupFailed, NoHolder: []string{"bare"}, - Asked: []PlanBackup{{Machine: "ace", Module: "mail", State: BackupFailed, AskedAt: &at, Words: "no space left"}}}} - raw, err := json.Marshal(p) - if err != nil { - t.Fatal(err) - } - for _, name := range []string{`"backups":{"state":"failed","asked":[{"machine":"ace","module":"mail","state":"failed",` + - `"asked_at":"2026-10-17T12:00:00Z","words":"no space left"}],"no_holder":["bare"]}`} { - if !strings.Contains(string(raw), name) { - t.Fatalf("plan JSON %s lacks %s", raw, name) - } - } - var back Plan - if err := json.Unmarshal(raw, &back); err != nil || !reflect.DeepEqual(back.Backups, p.Backups) { - t.Fatalf("round trip: %v %+v", err, back.Backups) - } - raw, _ = json.Marshal(Plan{ID: "q"}) - if strings.Contains(string(raw), "backups") { - t.Fatalf("a plan with no backups says none: %s", raw) - } -} diff --git a/internal/inventory/plan_id_test.go b/internal/inventory/plan_id_test.go deleted file mode 100644 index 431643b2..00000000 --- a/internal/inventory/plan_id_test.go +++ /dev/null @@ -1,47 +0,0 @@ -//go:build integration - -package inventory - -import ( - "errors" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/chain" -) - -// **A plan's id is a value, never a query** (novox/hq issue 561): the id reaches PlanByID from a caller of the -// controller's plans verb, so an id holding a quote is read as a plain id — no error from the store, and no -// other plan answered for it. On the bus an id is a key, and one the bus takes no spelling of is refused to be kept. -func TestAPlanIdHoldingAQuoteIsReadAsAPlainId(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - p := Plan{ID: "plan-561", Repository: "novox/mesh-controller", Commit: "abc", Created: time.Now().UTC(), - State: PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*PlanModule{"a": {}}} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - for _, id := range []string{ - "x' or '1'='1", // every plan, when pasted into the query - "plan-561' or id like '%", // the plan above, by a pattern - "it's", // a quote alone: a syntax error, when pasted - } { - got, err := inv.PlanByID(ctx, id) - if err == nil { - t.Fatalf("PlanByID(%q) answered %s, a plan of another id", id, got.ID) - } - if want := "no plan " + id; err.Error() != want { - t.Fatalf("PlanByID(%q) failed with %q, not as a plain id no plan has (%q)", id, err, want) - } - } - // A plan whose id holds a quote cannot be a key of the bus: refused by name, never rewritten into another id. - quoted := p - quoted.ID, quoted.Revision = "plan-'561'", 0 - if err := inv.SavePlan(ctx, "ed); !errors.Is(err, chain.ErrBadKey) { - t.Fatalf("a plan with a quote in its id was kept, or refused for another reason: %v", err) - } - if recent, err := inv.RecentPlans(ctx, 1); err != nil || len(recent) != 1 || !strings.HasPrefix(recent[0].ID, "plan-") { - t.Fatalf("RecentPlans with a parameter: %v %+v", err, recent) - } -} diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go deleted file mode 100644 index 90af4a0c..00000000 --- a/internal/inventory/plans.go +++ /dev/null @@ -1,765 +0,0 @@ -package inventory - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "slices" - "strings" - "time" - - "github.com/novox/mesh-controller/internal/chain" -) - -// A Plan is what a merge produces (novox/hq ADR 0162): the modules it changed and everything -// standing on them, sorted into tiers, each module's state, and the tier the plan is at. Kept in -// the store so a controller replaced mid-plan resumes it, and so `status` can say what a merge -// still waits for. -type Plan struct { - ID string `json:"id"` - Repository string `json:"repository"` - // Branch is the branch the merge went into (novox/hq issue 254): a newer plan supersedes the open - // ones of the same repository and branch. Empty for a plan from before it was kept. - Branch string `json:"branch,omitempty"` - Commit string `json:"commit"` - // Merged is when the forge made the merge this plan answers (novox/hq issue 349): the order of a - // branch's merges, which is not the order their plans were made in when one was acted on late. Zero - // for a release, and for a plan kept before it was. - Merged time.Time `json:"merged,omitzero"` - Created time.Time `json:"created"` - Updated time.Time `json:"updated"` - State string `json:"state"` - Tier int `json:"tier"` - Tiers [][]string `json:"tiers"` - Modules map[string]*PlanModule `json:"modules"` - Note string `json:"note,omitempty"` - // Away is the machines the walk passed because they were out of touch as a tier was sent (novox/hq ADR 0435 §1, - // issue 1161): not sent to and not waited for. Their want of the walk's builds stays for the send they get when - // they are heard again (ADR 0360 §5). Sorted. - Away []string `json:"away,omitempty"` - // TierEntered is when the plan entered the tier it is at (novox/hq to-be 45 Phase 0), read and - // never written from here: a save measures the tier it leaves and stamps the next. What the - // watchdog of a plan's progress (S3) reads. - TierEntered time.Time `json:"tier_entered,omitempty"` - // Revision is the plan's as it was read, and the one a save must find (novox/hq to-be 45 §6): a - // plan is written by compare-and-set, so a write against a plan another writer moved since is - // refused rather than laid over it. Zero is a plan never saved. SavePlan moves it. - Revision int64 `json:"revision"` - // Epoch is the controller lease epoch that wrote it last; zero for a write that claimed none. - Epoch uint64 `json:"epoch,omitempty"` - // Release is set on a release plan (novox/hq ADR 0236): not a merge's, but the builds waiting for a - // gate, walked through the machines one at a time. - Release *PlanRelease `json:"release,omitempty"` - // Delivery is set on a walk that waits for its delivery's word (novox/hq ADR 0239), and on every walk and - // batch that names the merges it answers (ADR 0276): Awaits empty for a walk on the controller's own path, - // which starts when it is cut. - Delivery *PlanDelivery `json:"delivery,omitempty"` - // Commits is every repository's commit the walk carries, one per repository (novox/hq ADR 0276): a batch's - // walk can carry several. Repository and Commit above keep one of them, for a reader that knows one. Empty - // for a walk kept before it was: Repository and Commit are then the whole of it. - Commits []PlanCommit `json:"commits,omitempty"` - // Times are the moments of a walk no other field keeps (novox/hq ADR 0282 decision 6): when its batch's - // window closed, when it was cut, and its class. Nil for a walk kept before they were, whose phases before - // its build are said unknown. - Times *PlanTimes `json:"times,omitempty"` - // Phases is the walk's time from its merge, phase by phase (ADR 0282 decision 6): never kept, worked out - // from the walk's own moments by whoever says the walk (`delivery walks`, `plan-moved`). - Phases *WalkPhases `json:"phases,omitempty"` - // Backups is the backups the walk takes before it asks its first tier (novox/hq ADR 0404 decisions 2 and 3): - // per machine and module, what was asked and how it went, and the machines with no backup holder. Nil for a - // walk that moves no module with an item protected by a backup, and for a walk kept before it was. - Backups *PlanBackups `json:"backups,omitempty"` - // Links are what the rollout names of the chain (novox/hq ADR 0408 §1): the builds it carried, the machine steps it has - // taken and the assignments it changed, each with the build before and after. Kept in the rollout's record beside - // the fields above; empty in the controller's store. - chain.Links - // Held are the merges this plan or batch holds (novox/hq ADR 0276, ADR 0408): what the store kept as batched merges - // with this plan's id, and the merges no plan answers that it held last. **Written by the merge operations only**: a - // save of the plan keeps what the record holds, never what this copy read, so a copy read before a merge moved cannot - // lay it back (issue 1092). - Held []chain.HeldMerge `json:"held,omitempty"` - // WalkSaid is, on the save that started or ended the walk, which of the two it was (novox/hq ADR 0308 - // decision 1): set by the save, never kept, read by whoever says the save (`plan-moved`). Empty on every - // other save. - WalkSaid string `json:"-"` -} - -// The states of a plan's backups, as a whole and each. -const ( - BackupPending = "pending" - BackupDone = "done" - BackupFailed = "failed" -) - -// PlanBackups is a walk's record of the backups it takes before it asks anything (novox/hq ADR 0404): read by -// mesh-delivery from `plan-moved`, so its JSON field names are a contract between the two repositories. -type PlanBackups struct { - // State is the whole: pending while any backup is asked or unasked and none failed, failed as soon as one - // failed, done when every one is. - State string `json:"state"` - // Asked is one entry per machine and module, sorted by machine then module. - Asked []PlanBackup `json:"asked"` - // NoHolder are the machines that run a module with an item protected by a backup and hold no node-backup: - // nothing is asked of them, and the plan says so. - NoHolder []string `json:"no_holder,omitempty"` -} - -// PlanBackup is one backup asked of one machine's backup holder for one module. -type PlanBackup struct { - Machine string `json:"machine"` - Module string `json:"module"` - // State is pending, done or failed. - State string `json:"state"` - // AskedAt is when the holder was asked; nil while it is not asked yet. - AskedAt *time.Time `json:"asked_at,omitempty"` - // TriedAt is when the controller first tried to ask, kept while the holder cannot be reached: the bound of a - // backup that is never asked counts from it. Nil once asked. - TriedAt *time.Time `json:"tried_at,omitempty"` - // Words are the backup's own: the holder's error for a failed one, "did not answer in ten minutes" for a silent one. - Words string `json:"words,omitempty"` -} - -// PlanTimes are a walk's own moments beside its modules' (novox/hq ADR 0282 decision 6). -type PlanTimes struct { - // WindowClosed is when its batch's merge window closed: no merge for the window's length, or its maximum. - // Nil for a walk no window assembled (one merge walked alone). - WindowClosed *time.Time `json:"window_closed,omitempty"` - // Cut is when the batch became the walk. - Cut *time.Time `json:"cut,omitempty"` - // Class is the walk's module class, read at its cut: core when it moves a module on the controller's own - // path or the mesh's resolver, leaf otherwise (ADR 0282 decision 1). - Class string `json:"class,omitempty"` - // Started is when the walk was said started, and Ended when it was said ended (novox/hq ADR 0308 decision - // 1): each said once, on the save that marked it. - Started *time.Time `json:"started,omitempty"` - Ended *time.Time `json:"ended,omitempty"` -} - -// PlanCommit is one repository's commit a walk carries: the latest merge of its branch in the batch. -type PlanCommit struct { - Repository string `json:"repository"` - Branch string `json:"branch,omitempty"` - Commit string `json:"commit"` - Merged time.Time `json:"merged,omitzero"` -} - -// PlanMerge is one merge a walk or batch answers (novox/hq ADR 0276), as mesh-delivery reads it: Carried is -// the commit of its repository the walk builds and sends when that is a later merge containing it, and -// empty when it is the merge itself. -type PlanMerge struct { - Repository string `json:"repository"` - Commit string `json:"commit"` - Carried string `json:"carried,omitempty"` - // Number and Title are the merge's pull request as the forge announced it, and Moves the modules the merge - // moved when it was heard: what `plans` names a walk by. Empty where the announcement said none. - Number int `json:"number,omitempty"` - Title string `json:"title,omitempty"` - Moves []string `json:"moves,omitempty"` - // Head is the pull request's head commit the forge merged (novox/hq issue 703): the walk takes only a build a - // check of that head made. Empty where the announcement said none, and then it takes none. - Head string `json:"head,omitempty"` - // Merged is when the forge made the merge and Heard when the controller heard it (novox/hq ADR 0282): where - // its delivery time starts. Zero for a merge named before they were kept. - Merged time.Time `json:"merged,omitzero"` - Heard time.Time `json:"heard,omitzero"` -} - -// PlanBatch is a batch's window while it is one (novox/hq ADR 0276): when it closes unless another merge -// comes, when it closes at the latest, and the walk it waits behind once closed. -type PlanBatch struct { - ClosesAt time.Time `json:"closes_at"` - AtMost time.Time `json:"at_most"` - Behind string `json:"behind,omitempty"` - // BehindWhy says why the batch waits behind that walk: what the two share (issue 973). - BehindWhy string `json:"behind_why,omitempty"` - // Own says the batch holds merges on the controller's own path, whose walk waits for nobody's word: such - // a merge never shares a batch with one that waits for mesh-delivery's (ADR 0276, decided during the - // build, 2026-10-10). - Own bool `json:"own,omitempty"` - // Closed says the window was closed by a stop that walks a stopped walk's merges again in this batch: they - // were heard before, and an older commit never waits behind a newer one (novox/hq ADR 0299). Withheld are - // the modules a stopped merge moves: built at the branch, which holds that merge, they would deliver it, so - // the batch's walk does not build them. - Closed bool `json:"closed,omitempty"` - Withheld []string `json:"withheld,omitempty"` - // CutAtOnce says the batch's window closed when it was made: its merge shares nothing with a batch - // assembling and has no delivery group member still to merge, so it waits for no window (novox/hq ADR 0304 - // decision 1, ADR 0306 decisions 2 and 3). A merge that joins it later opens its window again. It is not - // Closed, which is a stop's. - CutAtOnce bool `json:"cut_at_once,omitempty"` - // Waits says why a batch made for one merge waits for the window: its delivery group has a member still to - // merge, or mesh-delivery could not say whether it has (novox/hq ADR 0306 decision 2). - Waits string `json:"waits,omitempty"` - // Holds are the groups whose last member is still to merge, by head branch name, each with why in words - // (novox/hq ADR 0418 clause 3): the batch does not close by the merge window while one is held, however long the - // member takes. Cleared when every member is merged or closed; the window then runs again from that moment. - Holds map[string]string `json:"holds,omitempty"` - // HeldAt is when the last hold was cleared: the maximum is counted from it, not from the batch's making. - HeldAt time.Time `json:"held_at,omitempty"` -} - -// OwnPath says the record is a batch of merges on the controller's own path. -func (p Plan) OwnPath() bool { - return p.Delivery != nil && p.Delivery.Batch != nil && p.Delivery.Batch.Own -} - -// CommitOf is the commit of a repository the walk carries; empty when it carries none of it. -func (p Plan) CommitOf(repository string) string { - for _, c := range p.Commits { - if strings.EqualFold(c.Repository, repository) { - return c.Commit - } - } - if strings.EqualFold(p.Repository, repository) { - return p.Commit - } - return "" -} - -// CommitOn is the commit of a repository's branch the walk carries; empty when it carries none of it. A -// commit kept without its branch matches any branch. -func (p Plan) CommitOn(repository, branch string) string { - for _, c := range p.Commits { - if strings.EqualFold(c.Repository, repository) && (branch == "" || c.Branch == "" || c.Branch == branch) { - return c.Commit - } - } - if len(p.Commits) == 0 && strings.EqualFold(p.Repository, repository) && - (branch == "" || p.Branch == "" || p.Branch == branch) { - return p.Commit - } - return "" -} - -// Carried is every repository's commit the walk carries: Commits, or its one repository and commit. -func (p Plan) Carried() []PlanCommit { - if len(p.Commits) > 0 { - return p.Commits - } - if p.Repository == "" && p.Commit == "" { - return nil - } - return []PlanCommit{{Repository: p.Repository, Branch: p.Branch, Commit: p.Commit, Merged: p.Merged}} -} - -// Named is the walk as a person reads it: each repository at its commit. -func (p Plan) Named() string { - var out []string - for _, c := range p.Carried() { - out = append(out, c.Repository+" "+short(c.Commit)) - } - return strings.Join(out, ", ") -} - -// Batch says the record is a batch still being assembled or waiting to be cut, not a walk (ADR 0276). -func (p Plan) Batch() bool { return p.State == PlanAssembling || p.State == PlanQueued } - -// PlanDelivery is what a walk waits for and what came of the wait (novox/hq ADR 0239). -type PlanDelivery struct { - // Order are the edges the plan's own order was drawn with (novox/hq ADR 0419): verb before caller, version skew, - // engine before controller. Kept on the plan because the catalogue's graph does not hold them, and the plan's gates - // (a later tier waits for the earlier to be applied) are read from them. - Order []Edge `json:"order,omitempty"` - // Awaits is who must say the walk may start: the seat whose holder owns the delivery. - Awaits string `json:"awaits"` - // Go is when it was let go, By by whom (the seat's holder, or a person's name) and Why. - Go *time.Time `json:"go,omitempty"` - By string `json:"by,omitempty"` - Why string `json:"why,omitempty"` - // Stopped is who ended the walk, through the delivery's owner or by hand (`plans stop|close`), and StoppedWhy - // why: the walk is failed, and the delivery reads it as stopped, not as a build that failed. - Stopped string `json:"stopped,omitempty"` - StoppedWhy string `json:"stopped_why,omitempty"` - // Merges is every merge the walk or batch answers (novox/hq ADR 0276): its own commits, and the earlier - // merges of a repository its later commit contains. - Merges []PlanMerge `json:"merges,omitempty"` - // TakenOverBy names the walk a walk folded before it started was taken over by: its merges are that - // walk's now. - TakenOverBy string `json:"taken_over_by,omitempty"` - // WalkedAgainBy names, on a walk stopped without some of its merges, the batch that walks its other merges - // again, and LeftOut the merges it left out (novox/hq issue 459, ADR 0299): a delivery of a merge left out - // ends stopped with the walk, one of a merge walked again follows that batch's walk. - WalkedAgainBy string `json:"walked_again_by,omitempty"` - LeftOut []PlanLeftOut `json:"left_out,omitempty"` - // Kept are the modules the stopped walk already sent and nothing walks again: the machines named keep running - // the stopped walk's build (novox/hq ADR 0299). - Kept []PlanKept `json:"kept,omitempty"` - // Withheld are, on a walk cut from a batch a stop walked merges again in, the modules it did not build: a - // stopped merge moves them, and built only as dependents at the branch they would deliver it (ADR 0299). - Withheld []string `json:"withheld,omitempty"` - // Unchanged are the modules a merge it answers changed files in the directory of, whose recorded build source - // holds none of them, each with why (novox/hq issue 611, ADR 0267 §5): not built, because the build the mesh - // holds is their build. A delivery plan that read them whole named them; this says they matched what runs, - // rather than that the walk left them unbuilt. Modules keeps meaning what the walk builds. - Unchanged map[string]string `json:"unchanged,omitempty"` - // Batch is the window of a batch; nil once it is cut into a walk. - Batch *PlanBatch `json:"batch,omitempty"` - // Alone says the walk walks one merge on its own commit, after a failed walk carried it in a later one: - // built at that commit, not at the branch. - Alone bool `json:"alone,omitempty"` -} - -// PlanLeftOut is a merge a stop left out of the batch that walks a stopped walk's merges again (novox/hq ADR -// 0299): a merge stopped, or a later merge of its repository's branch, which contains the stopped one and so -// cannot be walked without delivering it. -type PlanLeftOut struct { - Repository string `json:"repository"` - Commit string `json:"commit"` - // Contains is the stopped merge a later merge contains, or whose modules it shares; empty for a merge - // stopped itself. - Contains string `json:"contains,omitempty"` - // Shares are the modules it moves that a stopped merge moves too: built at the branch, which holds the stopped - // merge, they would deliver it. - Shares []string `json:"shares,omitempty"` -} - -// PlanKept is a module a stopped walk sent and nothing walks again, and the machines it reached. -type PlanKept struct { - Module string `json:"module"` - Machines []string `json:"machines"` -} - -// Waiting is whether the walk waits for its delivery's word. -func (p Plan) Waiting() bool { - return p.Delivery != nil && p.Delivery.Awaits != "" && p.Delivery.Go == nil -} - -// PlanSaved is told every plan this process kept, after it is kept, with the inventory that kept it (novox/hq -// ADR 0239): the serving controller says it on the bus as `plan-moved`, and so does a command (the controller's -// run installs it for every verb, novox/hq ADR 0308 decision 1), so the save that starts or ends a walk is said -// whichever process made it. Nil says nothing; whoever follows a walk also asks for it, and finds an unsaid save -// by comparison. -var PlanSaved func(*Inventory, Plan) - -// ErrPlanMoved is a save against a plan written by somebody else since it was read. -var ErrPlanMoved = errors.New("the plan was written by somebody else since it was read") - -// PlanModule is one module's state within a plan. -type PlanModule struct { - // State: asked, built, failed; empty for a module whose tier has not been asked yet. - State string `json:"state,omitempty"` - AskedAt *time.Time `json:"asked_at,omitempty"` - BuiltAt *time.Time `json:"built_at,omitempty"` - // BusWaitFrom is when the bus first did not answer an ask of this module (issue 1334): the module stays unasked and is - // asked again at the next look, until the bus has been silent for the bound, when the plan fails as for any refusal. - BusWaitFrom *time.Time `json:"bus_wait_from,omitempty"` - // SentAt is when the plan sent the machines running this module its new build, because a - // later tier is built by it (ADR 0163's gate): the reports that open the gate are the ones - // after this. - SentAt *time.Time `json:"sent_at,omitempty"` - // First is the machines the plan sent the new build to first, and FirstAt when (novox/hq issue - // 249, ADR 0218): unless the module's policy rolls it out together, one machine takes it before - // the rest, and the rest are sent once that one reports it applied. Kept so a controller - // replaced while the plan waits on that report resumes the wait rather than sending again. The - // machine holding the bus is among them when its user list had to go first. - First []string `json:"first,omitempty"` - FirstAt *time.Time `json:"first_at,omitempty"` - Commit string `json:"commit,omitempty"` - Why string `json:"why,omitempty"` - // Placed is the edge that put the module after another, and the rule that drew it (novox/hq ADR 0419 clause 5): - // "after (verb before caller: .)". Empty for a module placed first by nothing. - Placed string `json:"placed,omitempty"` - // Made is what the walk's build of it did (novox/hq ADR 0350 §4): "reused from ", naming the pull - // request's head, the trunk commit and the digests, or "built: ". Said in `plan-moved` and kept here. - Made string `json:"made,omitempty"` - // Build is the id of the build the plan asked for this module (novox/hq ADR 0219), so the plan - // matches its outcome by id — the one thing every outcome echoes, a failed one that never learnt - // its module's name included. Empty in a plan from before it was kept, which is matched by - // module, or by repository and path, as before. - Build string `json:"build,omitempty"` - // Previous is the build the first machine ran of this module before the plan sent it the new one — - // the commit its last send carried (ADR 0221) — kept at the first send: what a rollback puts back - // (novox/hq ADR 0236). Empty when the machine had never been sent the module, or what it was sent - // is not known. - Previous string `json:"previous,omitempty"` - // Gate is the new build's judging on its first machine (novox/hq ADR 0236, to-be 45 §8), kept so a - // controller replaced mid-judging resumes it, and read back through `plans` as the rollout's record. - Gate *PlanGate `json:"gate,omitempty"` - // GatedBy names the module of the same tier whose gate judges this one on its first machine: they - // went there in one send (novox/hq issue 281), and one gate judges what one send moved. Empty for - // the module the gate is kept on, and for a plan from before tiers were sent whole. - GatedBy string `json:"gated_by,omitempty"` - // Rest is, per machine of the rest, the declaration the send after the first-node gate carried there (novox/hq ADR - // 0282 decision 6): the machine's first report of it, applied, is when the build runs there. - Rest map[string]SentDeclaration `json:"rest,omitempty"` - // Recorded is, for a module whose policy records, per machine running it with a move, which way the walk took it - // (novox/hq ADR 0377 §6): "went out: …" when the move recreates none of its containers and moves none of its - // data, or "waits: …" with the recreate words, the data change named, the person's or the bus's hold, or why it - // could not be judged. - Recorded map[string]string `json:"recorded,omitempty"` - // Key says the walk takes this module from the builder's done key and asked no build (novox/hq ADR 0501, ADR 0453 - // §5; issue 1851): from R4e every module but the builder's own. Its outcome is the key's, never a build-agent outcome. - Key bool `json:"key,omitempty"` - // KeyAt is the commit of its repository whose indexer's trunk value names the fingerprint taken (ADR 0501 §2); empty - // for a module of a repository the walk carries no commit of (a dependent through a base), taken at the value now. - KeyAt string `json:"key_at,omitempty"` - // Fingerprint is the fingerprint of the new form (`src2:`) the walk takes the module at, read from that trunk value; - // empty until the indexer has written it. - Fingerprint string `json:"fingerprint,omitempty"` -} - -// PlanGate is one module's rollout record at its gate (to-be 45 §8): the component, the first machine, -// from and to which build, the verdict, how long it took to reach it, and whether it was rolled back. -type PlanGate struct { - // Component is the core component the module is — mesh-controller, mesh-host, node-tools — or empty - // for any other module, judged by its own health. - Component string `json:"component,omitempty"` - Machines []string `json:"machines"` - From string `json:"from,omitempty"` - To string `json:"to,omitempty"` - // Since is when the judging began: the first machine reported the new build applied. - Since *time.Time `json:"since,omitempty"` - // Sent is, per machine, the declaration the gate's send carried there (novox/hq issue 352): what a - // machine's report is held against. Absent on a gate kept before it was, which reads the report - // against the send made last, as before. - Sent map[string]SentDeclaration `json:"sent,omitempty"` - // Applied is, per machine, the time of the first report the gate read of its send applied: the - // node-engine started the build before it, so a witness window counted from it ends no earlier than the - // node-engine's own (novox/hq ADR 0351 decisions 1 and 4). - Applied map[string]time.Time `json:"applied,omitempty"` - // Passes counts the consecutive judgings that found it healthy, LastPass the newest; a judging that - // does not resets them. - Passes int `json:"passes,omitempty"` - // Wants is how many passes the send needs: three for the node-engine or the mesh's resolver, or fewer at two - // minutes when healthy throughout, one for any other module (novox/hq ADR 0351 and ADR 0352, changing ADR - // 0282 decision 4 and ADR 0304 decision 3). Zero in a gate - // judged before it was said: three. - Wants int `json:"wants,omitempty"` - LastPass *time.Time `json:"last_pass,omitempty"` - // Faulted says a judging found the send broken, or not healthy after a healthy reading: a module under the three-reading - // rule then passes on its readings alone, never at its two minutes (novox/hq ADR 0352). - Faulted bool `json:"faulted,omitempty"` - // HealthySeen says a judging found the send healthy, counted or not: one taken before a module's floor - // counts too, so a dip after it is a fault (novox/hq ADR 0352 decision 3). - HealthySeen bool `json:"healthy_seen,omitempty"` - // Last is what the newest judging found wanting, while it still may pass. - Last string `json:"last,omitempty"` - // Verdict is empty while judging, then passed or failed, with Why, at JudgedAt, Took after Since. - Verdict string `json:"verdict,omitempty"` - Why string `json:"why,omitempty"` - JudgedAt *time.Time `json:"judged_at,omitempty"` - Took string `json:"took,omitempty"` - // Rollback is how a failed build was put back: rolled-back, or not-rolled-back with why. - Rollback string `json:"rollback,omitempty"` - // Kept says a passing verdict was written to the gate's records. - Kept bool `json:"kept,omitempty"` - // Carried is every module whose build moved on the judged machines with the send — the plan's own - // module and whatever else was waiting there for a gate (novox/hq ADR 0236): each is judged here, a - // pass is its verdict too, and one that fails is put back. - Carried []CarriedMove `json:"carried,omitempty"` - // Failing names the modules the last judging found wanting. - Failing []string `json:"failing,omitempty"` - // Healthy counts, per module, the consecutive judgings that found it healthy on every machine judged, - // or waiting for a person (novox/hq ADR 0254): each module's passes, apart from the send's. - Healthy map[string]int `json:"healthy,omitempty"` - // Waits is, per module, the wait for a person the newest judging read (ADR 0254): carried along, never - // a reason to fail. - Waits map[string]string `json:"waits,omitempty"` - // Passing names the modules that passed on their own when the send failed (ADR 0254): each keeps its - // pass, and is not put back. - Passing []string `json:"passing,omitempty"` - // HealthyAt is, per module, when its newest healthy judging was counted: a pass is counted only gateEvery - // after the one before (issue 318 review). - HealthyAt map[string]time.Time `json:"healthy_at,omitempty"` - // Broken names the modules a judging found broken, kept for the rest of the judging, and BrokenWhy the - // first reason: the send fails, and the modules beside them are judged to their own verdict first. - Broken []string `json:"broken,omitempty"` - BrokenWhy string `json:"broken_why,omitempty"` - // Returned names the broken modules already put back, at once, while the rest of the send is judged. - Returned []string `json:"returned,omitempty"` - // Readings are the judging's readings with their times (novox/hq ADR 0282 decision 6): every reading that - // counted a pass, and the first that did not after one that did. At most maxReadings, the newest kept. - Readings []GateReading `json:"readings,omitempty"` - // Grants is the grants step taken before this gate's send (novox/hq issue 490): the bus's user list sent - // alone to the machine holding the bus, every build there kept, so what the send newly grants is on the - // bus before the gate judges it. Nil when the list did not change, or the gate's machine holds the bus. - Grants *GrantsStep `json:"grants,omitempty"` - // Busy is the wait for a busy container on a judged machine (novox/hq ADR 0335 §4): while the machine - // reports its module's container left running because it said busy, the judging waits and has not - // started; once it reports the container replaced, Since is moved to then. Nil when nothing waited. - Busy *GateBusy `json:"busy,omitempty"` - // Pending says the send this gate judges is kept and not yet made (novox/hq ADR 0353 §8): a controller that - // takes over reads the machine's last send before making it, and never sends it twice. - Pending bool `json:"pending,omitempty"` - // Silent says the machine reported nothing on its send within ApplySilentAfter (novox/hq ADR 0353 §4): left, - // neither failed nor put back. - Silent bool `json:"silent,omitempty"` - // TookOver names the gate whose readings this one took over (novox/hq ADR 0353 §7): another walk's judging of - // the same build on the same machine, which this gate goes on from instead of starting from zero. - TookOver string `json:"took_over,omitempty"` -} - -// TakeReadings makes this gate go on from another's readings of the same build on the same machine (novox/hq ADR -// 0353 §7): when the judging began, when each machine applied, the passes counted and when, and what each found. -// A module's own count is taken only for the modules this gate judges. -func (g *PlanGate) TakeReadings(from *PlanGate, modules []string, id string) { - if from == nil { - return - } - if from.Since != nil { - since := *from.Since - g.Since = &since - } - for n, s := range from.Sent { - if slices.Contains(g.Machines, n) { - if g.Sent == nil { - g.Sent = map[string]SentDeclaration{} - } - g.Sent[n] = s - } - } - for n, at := range from.Applied { - if slices.Contains(g.Machines, n) { - if g.Applied == nil { - g.Applied = map[string]time.Time{} - } - g.Applied[n] = at - } - } - g.Passes, g.Faulted, g.HealthySeen, g.Last = from.Passes, from.Faulted, from.HealthySeen, from.Last - if from.LastPass != nil { - at := *from.LastPass - g.LastPass = &at - } - for _, m := range modules { - if n, counted := from.Healthy[m]; counted { - if g.Healthy == nil { - g.Healthy = map[string]int{} - } - g.Healthy[m] = n - } - if at, counted := from.HealthyAt[m]; counted { - if g.HealthyAt == nil { - g.HealthyAt = map[string]time.Time{} - } - g.HealthyAt[m] = at - } - } - g.Readings = append([]GateReading(nil), from.Readings...) - g.TookOver = id -} - -// GateBusy is a first-node gate's wait for a busy container (novox/hq ADR 0335 §4). -type GateBusy struct { - Machine string `json:"machine"` - Module string `json:"module"` - Container string `json:"container"` - // Since is when the machine's first apply that would have replaced it ran, Until when its node-engine - // replaces it at the latest; Ended when the machine reported it replaced and the judging began — or when - // the controller stopped waiting past the engine's bound, which PastBound says. - Since time.Time `json:"since"` - Until time.Time `json:"until"` - // Seen is when this controller first read the wait, on its own clock: the gate's bound runs from it, never - // from the engine's word alone. - Seen time.Time `json:"seen,omitzero"` - Ended *time.Time `json:"ended,omitempty"` - PastBound bool `json:"past_bound,omitempty"` - // Said is the last output of its busy command, as the machine reported it. - Said string `json:"said,omitempty"` -} - -// Waiting says the gate waits for a busy container and has not started. -func (g *PlanGate) Waiting() bool { return g != nil && g.Busy != nil && g.Busy.Ended == nil } - -// Words is the wait in a line, as the walk says it. -func (b *GateBusy) Words() string { - said := "" - if b.Said != "" { - said = "; its busy command says: " + b.Said - } - return fmt.Sprintf("waits for a busy container on %s: %s holds work, and is replaced once it says idle, by %s "+ - "at the latest (novox/hq ADR 0335)%s", b.Machine, b.Container, b.Until.UTC().Format(time.RFC3339), said) -} - -// GrantsStep is the bus's user list sent to the machine holding the bus ahead of a gated send (novox/hq -// issue 490): to which machine, when, and, when it could not be sent, why — the send went on without it. -type GrantsStep struct { - Node string `json:"node"` - At *time.Time `json:"at,omitempty"` - Failed string `json:"failed,omitempty"` -} - -// GateReading is one reading of a first-node gate. -type GateReading struct { - At time.Time `json:"at"` - Healthy bool `json:"healthy"` - // Said is what a reading that did not pass found wanting. - Said string `json:"said,omitempty"` -} - -// maxReadings bounds a first-node gate's readings: a judging that never passes reads every few seconds for ten minutes. -const maxReadings = 24 - -// Read keeps one reading: a pass always, and a reading that did not pass only when the one before passed or -// there is none, so a judging waiting for a module to start keeps one line of it, not hundreds. -func (g *PlanGate) Read(at time.Time, healthy bool, said string) { - if !healthy && len(g.Readings) > 0 && !g.Readings[len(g.Readings)-1].Healthy { - return - } - if r := []rune(said); len(r) > 200 { - said = string(r[:200]) - } - g.Readings = append(g.Readings, GateReading{At: at, Healthy: healthy, Said: said}) - if len(g.Readings) > maxReadings { - g.Readings = g.Readings[len(g.Readings)-maxReadings:] - } -} - -// CarriedMove is one module's build moving on a machine with a gated send. -type CarriedMove struct { - Module string `json:"module"` - Node string `json:"node"` - From string `json:"from,omitempty"` - To string `json:"to"` - Build string `json:"build,omitempty"` - // Recreates is what the send does to the module's containers, in the mesh's words — how many it - // recreates, and whether with a new image or only their declaration (novox/hq ADR 0245); empty when - // it recreates none, or when the build the machine ran is not known. - Recreates string `json:"recreates,omitempty"` -} - -// PlanRelease is a release plan's walk through the machines (novox/hq ADR 0236): every module build -// that waits for a gate, sent one machine at a time, each judged before the next. -type PlanRelease struct { - Order []string `json:"order"` - Next int `json:"next"` - // Gate is the first machine being judged; nil between machines. - Gate *PlanGate `json:"gate,omitempty"` - // Rest says the first machine passed (novox/hq ADR 0353 §1): every other machine is sent at once, each judged by - // its own gate in Gates, by machine — a build no gate passed going to one machine of a step only (§2). - Rest bool `json:"rest,omitempty"` - Gates map[string]*PlanGate `json:"gates,omitempty"` - // Failed are the machines of the rest whose gate failed, and Silent those that said nothing on their send - // within ApplySilentAfter (ADR 0353 §3 and §4). - Failed []string `json:"failed,omitempty"` - Silent []string `json:"silent,omitempty"` - Done []string `json:"done,omitempty"` - // Skipped are the machines not heard from when their turn came, left as they were. - Skipped []string `json:"skipped,omitempty"` - // By is the person who released it, empty when the mesh did. - By string `json:"by,omitempty"` - // Settings is set on a settings walk (novox/hq ADR 0344): not the builds waiting for a gate, but a settings - // layer the operator approved, sent to the machines it reaches one at a time, its module judged on each. - Settings *PlanSettings `json:"settings,omitempty"` -} - -// PlanSettings is what a settings walk sends (novox/hq ADR 0344): the module whose layer the operator approved, the -// layer's scope, the ask that approved it and what changed, in words. -type PlanSettings struct { - Module string `json:"module"` - // Node is the machine the layer is for, empty for the whole mesh. - Node string `json:"node,omitempty"` - // Ask is the proposal the operator approved, and Change what its layer changed ("~ prefs"). - Ask string `json:"ask"` - Change string `json:"change,omitempty"` -} - -// The states a plan passes through. -const ( - PlanBuilding = "building" - PlanRolling = "rolling" - PlanDone = "done" - PlanFailed = "failed" - // PlanSuperseded is a plan a newer merge of the same repository and branch took over (novox/hq - // issue 254, ADR 0218): what it had not built is in the newer plan, and its note names it. - PlanSuperseded = "superseded" - // PlanAssembling is a batch whose merge window is open (novox/hq ADR 0276), and PlanQueued one whose - // window closed while a walk is open: neither is a walk yet. Cut, a batch keeps its id and is building. - PlanAssembling = "assembling" - PlanQueued = "queued" -) - -// Open says whether the plan is still being worked. -func (p Plan) Open() bool { return p.State == PlanBuilding || p.State == PlanRolling } - -// SavePlan writes a plan, new or changed, whole: the plan is small and read as one thing. -// -// **By compare-and-set on its revision, carrying the epoch** (novox/hq to-be 45 §6): written only if -// the plan is still at the revision it was read at — a new one only if it does not exist — and refused -// with ErrPlanMoved otherwise; and only by a process that may act (ActsUnder), whose epoch it records. -// On success p's revision and epoch are the ones written, so the caller may save it again. -func (i *Inventory) SavePlan(ctx context.Context, p *Plan) error { - return i.SavePlansMoving(ctx, []*Plan{p}, nil) -} - -// MergeMove is a merge given to the plan that answers it now (novox/hq ADR 0276). -type MergeMove struct { - Repository, Commit, Plan string - Alone bool - // Event replaces the merge's kept announcement when it is put back to waiting (Plan empty); nil leaves it. - Event json.RawMessage -} - -// SavePlansMoving writes plans, as SavePlan does each, and gives merges to the plans that answer them, all in one -// transaction: every write and move, or none (novox/hq ADR 0299 decision 7 — a stop that walks merges again never -// leaves a merge answered by two plans, nor a walk stopped whose merges were not moved). A plan written by -// somebody else since it was read refuses the whole act with ErrPlanMoved. Each plan is said (PlanSaved) once -// the transaction is committed, in order. -func (i *Inventory) SavePlansMoving(ctx context.Context, plans []*Plan, moves []MergeMove) error { - c, err := i.theChain(ctx) - if err != nil { - return err - } - return i.chainSavePlans(ctx, c, plans, moves) -} - -// short is a commit as a person reads it. -func short(commit string) string { - if len(commit) > 8 { - return commit[:8] - } - return commit -} - -// OpenPlans is every plan still being worked, oldest first. -func (i *Inventory) OpenPlans(ctx context.Context) ([]Plan, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.open() }, sortCreated) -} - -// PlansSince is every plan made after a moment, and every plan still being worked, oldest first. -func (i *Inventory) PlansSince(ctx context.Context, since time.Time) ([]Plan, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - return i.chainPlans(ctx, c, func(e viewEntry) bool { return e.sum.Created.After(since) || e.sum.open() }, sortCreated) -} - -// RecentPlans is the last few plans, newest first, open or not — what the overview shows. -func (i *Inventory) RecentPlans(ctx context.Context, limit int) ([]Plan, error) { - c, err := i.theChain(ctx) - if err != nil { - return nil, err - } - plans, err := i.chainPlans(ctx, c, func(viewEntry) bool { return true }, sortCreatedNewest) - if err == nil && limit >= 0 && len(plans) > limit { - plans = plans[:limit] - } - return plans, err -} - -// NewestMergeOf is the plan, in any state, of the newest merge into a repository's branch that the mesh -// planned: the branch's newest commit the mesh knows of (novox/hq issue 349). False when no plan of it -// recorded when its merge was made. -func (i *Inventory) NewestMergeOf(ctx context.Context, repository, branch string) (Plan, bool, error) { - c, err := i.theChain(ctx) - if err != nil { - return Plan{}, false, err - } - return i.chainNewestMergeOf(ctx, c, repository, branch) -} - -// PlanByID is one plan. The id is a parameter, never part of the query (novox/hq issue 561): it comes from a -// caller of the plans verb. -func (i *Inventory) PlanByID(ctx context.Context, id string) (Plan, error) { - c, err := i.theChain(ctx) - if err != nil { - return Plan{}, err - } - return i.chainPlanByID(ctx, c, id) -} diff --git a/internal/inventory/plans_leave_the_store_test.go b/internal/inventory/plans_leave_the_store_test.go new file mode 100644 index 00000000..ab30592b --- /dev/null +++ b/internal/inventory/plans_leave_the_store_test.go @@ -0,0 +1,47 @@ +//go:build integration + +package inventory + +import ( + "strings" + "testing" +) + +// **Migration 0108 drops the tables of the delivery plans' machinery** (novox/hq ADR 0539 §17, design 47 release 6, issue 1945): +// the record of each send, the assignment generation, the gate's verdicts and the measured durations. The store is built as +// the mesh holds it before the migration, with a row in each, and after it the four are gone and what the controller still +// keeps stays: the node and its columns, the module, the bus step, the core box and the refused registrations. +func TestMigration0108DropsTheTablesOfThePlansMachineryAndKeepsWhatTheControllerStillKeeps(t *testing.T) { + inv, migrate := ForTestBefore(t, 108) + tables := func() map[string]bool { + t.Helper() + out := map[string]bool{} + for _, row := range inv.QueryForTest(t, `select table_name::text from information_schema.tables where table_schema = 'public'`) { + out[strings.Trim(row[0], `"`)] = true + } + return out + } + gone := []string{"declaration_send", "assignment_generation", "build_gate", "duration"} + before := tables() + for _, name := range gone { + if !before[name] { + t.Fatalf("the table %s is not in the store before the migration", name) + } + } + migrate() + after := tables() + for _, name := range gone { + if after[name] { + t.Errorf("the table %s is still in the store", name) + } + } + for _, stays := range []string{"node", "module", "bus_step", "core_box", "build_refused", "controller_schema"} { + if !after[stays] { + t.Errorf("the table %s is gone", stays) + } + } + // The functions and triggers the tables carried go with them. + if rows := inv.QueryForTest(t, `select count(*) from pg_proc where proname = 'raise_assignment_generation'`); rows[0][0] != "0" { + t.Errorf("the function raise_assignment_generation is still in the store") + } +} diff --git a/internal/inventory/plans_test.go b/internal/inventory/plans_test.go deleted file mode 100644 index 819b393c..00000000 --- a/internal/inventory/plans_test.go +++ /dev/null @@ -1,225 +0,0 @@ -//go:build integration - -package inventory - -import ( - "reflect" - "testing" - "time" -) - -// A plan is a record the mesh keeps and resumes (novox/hq ADR 0162): written whole, read back open, -// advanced, and gone from the open ones when done. -func TestAPlanIsKeptAdvancedAndResumedFromTheStore(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - p := Plan{ID: "plan-1", Repository: "novox/mesh-tools", Commit: "abc", Created: time.Now().UTC(), - State: PlanBuilding, Tiers: [][]string{{"mesh-tools"}, {"builder"}, {"shop"}}, - Modules: map[string]*PlanModule{"mesh-tools": {}, "builder": {}, "shop": {}}} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - open, err := inv.OpenPlans(ctx) - if err != nil || len(open) != 1 || open[0].ID != "plan-1" || len(open[0].Tiers) != 3 { - t.Fatalf("the plan was not kept whole: %v %+v", err, open) - } - // Another controller picks it up where it was left: a tier advanced and a module built. - now := time.Now().UTC() - resumed := open[0] - resumed.Tier = 1 - resumed.Modules["mesh-tools"].State = "built" - resumed.Modules["mesh-tools"].BuiltAt = &now - resumed.State = PlanRolling - resumed.Note = "tier 0 built; waiting for builder on anchor to be applied" - if err := inv.SavePlan(ctx, &resumed); err != nil { - t.Fatal(err) - } - again, err := inv.PlanByID(ctx, "plan-1") - if err != nil || again.Tier != 1 || again.Modules["mesh-tools"].State != "built" || again.State != PlanRolling { - t.Fatalf("the advanced plan did not come back as left: %v %+v", err, again) - } - again.State = PlanDone - if err := inv.SavePlan(ctx, &again); err != nil { - t.Fatal(err) - } - if open, _ = inv.OpenPlans(ctx); len(open) != 0 { - t.Fatalf("a done plan is not open: %+v", open) - } - if recent, _ := inv.RecentPlans(ctx, 5); len(recent) != 1 || recent[0].State != PlanDone { - t.Fatalf("a done plan is still among the recent ones: %+v", recent) - } -} - -// novox/hq issue 254: a plan keeps the branch its merge went into, and a superseded plan is not open. -func TestASupersededPlanIsNotOpen(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - p := Plan{ID: "plan-1", Repository: "novox/mesh-catalog", Branch: "main", Commit: "abc", - Created: time.Now().UTC(), State: PlanBuilding, Tiers: [][]string{{"gitea"}}, - Modules: map[string]*PlanModule{"gitea": {}}} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - kept, err := inv.PlanByID(ctx, "plan-1") - if err != nil || kept.Branch != "main" { - t.Fatalf("the branch was not kept: %v %+v", err, kept) - } - kept.State = PlanSuperseded - kept.Note = "superseded at tier 0 by plan-2" - if err := inv.SavePlan(ctx, &kept); err != nil { - t.Fatal(err) - } - if open, err := inv.OpenPlans(ctx); err != nil || len(open) != 0 { - t.Fatalf("a superseded plan is still open: %v %+v", err, open) - } - if recent, _ := inv.RecentPlans(ctx, 5); len(recent) != 1 || recent[0].State != PlanSuperseded { - t.Fatalf("a superseded plan is not among the recent ones as superseded: %+v", recent) - } -} - -// novox/hq issue 349 (review of the follow-up): the newest merge of a branch is the one merged last, whatever -// order the plans were made in, in any state; a tie is broken by the plan made last; a release, another -// branch and another repository are never it; and its time is kept to the nanosecond. -func TestTheNewestMergeOfABranchIsTheOneMergedLast(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - t0 := time.Date(2026, 10, 9, 10, 0, 0, 0, time.UTC) - save := func(id, repo, branch string, merged, created time.Time, state string, release bool) { - t.Helper() - p := Plan{ID: id, Repository: repo, Branch: branch, Commit: id + "-commit", Merged: merged, Created: created, - State: state, Tiers: [][]string{}, Modules: map[string]*PlanModule{}} - if release { - p.Release = &PlanRelease{} - } - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - } - if _, found, err := inv.NewestMergeOf(ctx, "novox/mesh-catalog", "main"); err != nil || found { - t.Fatalf("a branch with no plan: %v %v", found, err) - } - // Made in the other order than merged: the later merge's plan made first, and done. - save("plan-later", "novox/mesh-catalog", "main", t0.Add(2*time.Minute+250*time.Millisecond), t0, PlanDone, false) - save("plan-earlier", "novox/mesh-catalog", "main", t0.Add(time.Minute), t0.Add(5*time.Minute), PlanRolling, false) - save("plan-other-branch", "novox/mesh-catalog", "release", t0.Add(time.Hour), t0, PlanRolling, false) - save("plan-other-repo", "novox/mesh-controller", "main", t0.Add(time.Hour), t0, PlanRolling, false) - save("release-1", "novox/mesh-catalog", "main", t0.Add(time.Hour), t0, PlanRolling, true) - save("plan-unknown", "novox/mesh-catalog", "main", time.Time{}, t0.Add(time.Hour), PlanRolling, false) - p, found, err := inv.NewestMergeOf(ctx, "Novox/Mesh-Catalog", "main") - if err != nil || !found || p.ID != "plan-later" { - t.Fatalf("the newest merge: %s %v %v", p.ID, found, err) - } - if !p.Merged.Equal(t0.Add(2*time.Minute + 250*time.Millisecond)) { - t.Fatalf("its merge time was not kept to the nanosecond: %s", p.Merged) - } - // The same merge time: the plan made last. - save("plan-again", "novox/mesh-catalog", "main", t0.Add(2*time.Minute+250*time.Millisecond), t0.Add(time.Minute), PlanBuilding, false) - if p, _, _ := inv.NewestMergeOf(ctx, "novox/mesh-catalog", "main"); p.ID != "plan-again" { - t.Fatalf("one merge time, two plans: %s", p.ID) - } -} - -// novox/hq issue 352: what a machine was last sent is read back by name with its sequence, a report keeps -// the declaration it was about and that declaration's sequence, and a gate's sends are kept with the plan. -func TestASendAndAReportAreKnownByTheirDeclaration(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - record, err := inv.AddNode(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - if _, found, err := inv.SentTo(ctx, "anchor"); err != nil || found { - t.Fatalf("a machine never sent anything: %v %v", found, err) - } - if _, _, err := inv.SentTo(ctx, "nobody"); err == nil { - t.Fatal("a machine that does not exist was answered") - } - seq, err := inv.NextSequence(ctx, record.ID) - if err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, record.ID, "d-1", map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - sent, found, err := inv.SentTo(ctx, "anchor") - if err != nil || !found || sent.Digest != "d-1" || sent.Sequence != seq { - t.Fatalf("sent %+v %v %v", sent, found, err) - } - if _, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Node: "anchor", Outcome: OutcomeApplied, Declared: "d-1", Applied: 1}, - ReportOrder{Sequence: seq, ReportSequence: 1}, func(ReportOrder) bool { return false }); err != nil { - t.Fatal(err) - } - reports, err := inv.LastReports(ctx) - if err != nil || len(reports) != 1 || reports[0].Declared != "d-1" || reports[0].ReportedSequence != seq || !reports[0].Current { - t.Fatalf("reports %+v %v", reports, err) - } - // Sent again, unreported: the report is no longer on the last send, and is still on the first. - if err := inv.RecordSent(ctx, record.ID, "d-2", map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - reports, _ = inv.LastReports(ctx) - if reports[0].Current || !sent.ReportsOn(reports[0]) { - t.Fatalf("after a newer send: %+v", reports[0]) - } - at := time.Now().UTC() - p := Plan{ID: "plan-352", Repository: "novox/x", Commit: "c", Created: at, State: PlanRolling, Tiers: [][]string{{"app"}}, - Modules: map[string]*PlanModule{"app": {Gate: &PlanGate{Machines: []string{"anchor"}, Since: &at, - Sent: map[string]SentDeclaration{"anchor": sent}}}}} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - kept, err := inv.PlanByID(ctx, "plan-352") - if err != nil || !reflect.DeepEqual(kept.Modules["app"].Gate.Sent["anchor"], sent) { - t.Fatalf("the gate's send was not kept with the plan: %+v %v", kept.Modules["app"].Gate, err) - } -} - -// A controller build's reach of the store's schema is kept by its version, and the store's own is read. -func TestASchemaReachIsKeptByBuild(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - applied, err := inv.SchemaApplied(ctx) - if err != nil || applied < 87 { - t.Fatalf("applied %d %v", applied, err) - } - if _, known, err := inv.SchemaReachOf(ctx, "ad62528c47c7"); err != nil || known { - t.Fatalf("an unrecorded build: %v %v", known, err) - } - if err := inv.RecordSchemaReach(ctx, "", 87); err == nil { - t.Fatal("a reach without a build was recorded") - } - if err := inv.RecordSchemaReach(ctx, "ad62528c47c7", 86); err != nil { - t.Fatal(err) - } - if err := inv.RecordSchemaReach(ctx, "ad62528c47c7", 87); err != nil { - t.Fatal(err) - } - if reach, known, err := inv.SchemaReachOf(ctx, "ad62528c47c7"); err != nil || !known || reach != 87 { - t.Fatalf("reach %d %v %v", reach, known, err) - } -} - -// A plan keeps its backups through the store, timestamps to the nanosecond (novox/hq ADR 0404, issue 1065). -func TestAPlanKeepsItsBackupsInTheStore(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - at := time.Now().UTC() - p := Plan{ID: "plan-backups", Repository: "novox/mesh-catalog", Commit: "abc", Created: at, State: PlanBuilding, - Tiers: [][]string{{"mail"}}, Modules: map[string]*PlanModule{"mail": {}}, - Backups: &PlanBackups{State: BackupPending, NoHolder: []string{"bare"}, - Asked: []PlanBackup{{Machine: "ace", Module: "mail", State: BackupPending, AskedAt: &at}}}} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - got, err := inv.PlanByID(ctx, p.ID) - if err != nil || got.Backups == nil || !reflect.DeepEqual(got.Backups, p.Backups) { - t.Fatalf("kept %+v (%v), wrote %+v", got.Backups, err, p.Backups) - } - got.Backups.State = BackupDone - if err := inv.SavePlan(ctx, &got); err != nil { - t.Fatal(err) - } - if again, _ := inv.PlanByID(ctx, p.ID); again.Backups == nil || again.Backups.State != BackupDone { - t.Fatalf("the change was not kept: %+v", again.Backups) - } -} diff --git a/internal/inventory/provisioned.go b/internal/inventory/provisioned.go index ac2779bd..ba2c8f0e 100644 --- a/internal/inventory/provisioned.go +++ b/internal/inventory/provisioned.go @@ -1,8 +1,6 @@ package inventory import ( - "fmt" - "sort" "time" ) @@ -37,84 +35,3 @@ const ( ProvFailed = "failed" ProvRemoved = "removed" ) - -// ProvLookup answers a client's newest report of a module on a machine; false when it has said nothing of it. -type ProvLookup func(node, module string) (ProvReport, bool) - -// OutcomeProvisioningFailed begins the outcome of a send whose module's files the provisioner failed to place. -const OutcomeProvisioningFailed = "provisioning failed" - -// ProvisionedSend is a machine's report of a send, with its provisioner client's reports of every module the send -// handed to it: what a walk counts as that machine's report. engine and reported are the machine's own report and -// whether it said one. A failed client report is the send's outcome, whatever the machine said; none, or one not -// yet applied on the build sent, leaves the send unreported. -func ProvisionedSend(node string, send SentDeclaration, engine AppliedReport, reported bool, prov ProvLookup) (AppliedReport, bool) { - if len(send.Provisioned) == 0 { - return engine, reported - } - // The machine's own failure, or its wait for a busy container, is its report whatever the client says. - if reported && (engine.Outcome != OutcomeApplied || engine.Waiting) { - return engine, true - } - modules := make([]string, 0, len(send.Provisioned)) - for m := range send.Provisioned { - modules = append(modules, m) - } - sort.Strings(modules) - at, placed := engine.At, true - for _, m := range modules { - r, said := ProvReport{}, false - if prov != nil { - r, said = prov(node, m) - } - switch { - case said && r.Status == ProvFailed && aboutTheSend(r, send, send.Provisioned[m]): - return AppliedReport{At: r.At, Outcome: fmt.Sprintf("%s (%s: %s)", OutcomeProvisioningFailed, m, - orNoReason(r.Reason))}, true - case said && r.Status == ProvApplied && r.Templates == send.Provisioned[m]: - if r.At.After(at) { - at = r.At - } - default: - placed = false - } - } - if !reported || !placed { - return AppliedReport{}, false - } - engine.At = at - return engine, true -} - -// Only is the send narrowed to what it handed to the provisioner for the named modules: a delivery waits on the -// modules it carries, never on another module the same machine's declaration hands (ADR 0362 §1-2, issue 840). -// A send handing nothing is returned as it is. -func (s SentDeclaration) Only(modules ...string) SentDeclaration { - if len(s.Provisioned) == 0 { - return s - } - kept := map[string]string{} - for _, m := range modules { - if d, ok := s.Provisioned[m]; ok { - kept[m] = d - } - } - s.Provisioned = kept - return s -} - -// aboutTheSend says a client's failure is about what the send handed: on the build's templates, or, said before it -// had any to name (its templates could not be fetched), no earlier than the send. -func aboutTheSend(r ProvReport, send SentDeclaration, digest string) bool { - if r.Templates != "" { - return r.Templates == digest - } - return send.At.IsZero() || !r.At.Before(send.At) -} - -func orNoReason(s string) string { - if s == "" { - return "no reason given" - } - return s -} diff --git a/internal/inventory/provisioned_test.go b/internal/inventory/provisioned_test.go deleted file mode 100644 index 9057168d..00000000 --- a/internal/inventory/provisioned_test.go +++ /dev/null @@ -1,124 +0,0 @@ -package inventory - -import ( - "strings" - "testing" - "time" -) - -// A walk counts a provisioned module applied on a machine only once its files are placed (novox/hq ADR 0362, design -// 57 P6, issue 672). The walk of issue 582 (one module, three machines of the rest), with g14's send having handed -// dunst to the provisioner: g14's own report comes 5 s after its send, shanks's, the last of the others, 15 s -// after its own. - -const dunstTemplates = "sha256:" + "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" - -func walkHandingDunst(t *testing.T) sendRest582 { - t.Helper() - c := walk582(t) - s := c.walk.Modules["build-agent"].Rest["g14"] - s.Provisioned = map[string]string{"build-agent": dunstTemplates} - c.walk.Modules["build-agent"].Rest["g14"] = s - return c -} - -// provisioning is a walk's lookup with the machines' clients saying reports. -func (w sendRest582) provisioning(reports map[string]ProvReport) AppliedLookup { - return func(node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool) { - r, ok := w.lookup(node, send, walkSent) - return ProvisionedSend(node, send, r, ok, func(node, module string) (ProvReport, bool) { - p, said := reports[node+"."+module] - return p, said - }) - } -} - -func restOf(w *WalkPhases, node string) RestReport { - for _, r := range w.Rest { - if r.Node == node { - return r - } - } - return RestReport{} -} - -// The node-engine's report alone does not end the walk: it waits for prov.applied; with it, the walk ends, and -// its delivery time runs to the later of the two reports. -func TestAWalkWaitsForTheProvisionersReportAndEndsAtTheLater(t *testing.T) { - c := walkHandingDunst(t) - g14Sent := c.walk.Modules["build-agent"].Rest["g14"].At - - w := c.walk.WalkPhases(g14Sent.Add(time.Minute), c.provisioning(nil)) - if w.End != nil || restOf(w, "g14").State != RestWaiting || - !strings.Contains(phase(w, PhaseApply, -1).Said, "waiting for g14") { - t.Fatalf("the walk did not wait for g14's prov.applied: end %v, g14 %+v, apply %+v", w.End, restOf(w, "g14"), - phase(w, PhaseApply, -1)) - } - - // A report on an older build of the templates is not this send's. - older := map[string]ProvReport{"g14.build-agent": {Status: ProvApplied, Templates: "sha256:" + strings.Repeat("0", 64), - At: g14Sent.Add(2 * time.Second)}} - if w := c.walk.WalkPhases(g14Sent.Add(time.Minute), c.provisioning(older)); w.End != nil { - t.Fatalf("a report on an older build ended the walk at %v", w.End) - } - - placed := g14Sent.Add(50 * time.Second) // after shanks's report, 35 s after the judgement - w = c.walk.WalkPhases(g14Sent.Add(time.Hour), c.provisioning(map[string]ProvReport{ - "g14.build-agent": {Status: ProvApplied, Templates: dunstTemplates, At: placed}})) - sumsUp(t, w) - if w.End == nil || !w.End.Equal(placed) { - t.Fatalf("the walk ends at %v, want the provisioner's report at %s: %+v", w.End, placed, w.Rest) - } - if g := restOf(w, "g14"); g.State != RestApplied || !g.Reported.Equal(placed) { - t.Fatalf("g14 is not applied at its later report: %+v", g) - } -} - -// A failed prov.applied is the machine's failed apply, named in the walk's end with its reason. -func TestAFailedProvisioningIsAFailedApply(t *testing.T) { - c := walkHandingDunst(t) - g14Sent := c.walk.Modules["build-agent"].Rest["g14"].At - w := c.walk.WalkPhases(g14Sent.Add(time.Hour), c.provisioning(map[string]ProvReport{ - "g14.build-agent": {Status: ProvFailed, Reason: "the templates cannot be fetched", At: g14Sent.Add(3 * time.Second)}})) - if w.End != nil { - t.Fatalf("a failed provisioning ended the walk: %v", w.End) - } - if g := restOf(w, "g14"); g.State != RestFailed || !strings.Contains(g.Outcome, "the templates cannot be fetched") { - t.Fatalf("g14 is not named failed with the reason: %+v", g) - } - if said := phase(w, PhaseApply, -1).Said; !strings.Contains(said, "not applied on g14 (provisioning failed (build-agent: "+ - "the templates cannot be fetched))") { - t.Fatalf("the walk's end does not name g14's failure: %q", said) - } - // A failure said before the send, of no build, is not this send's. - stale := map[string]ProvReport{"g14.build-agent": {Status: ProvFailed, Reason: "old", At: g14Sent.Add(-time.Hour)}} - if g := restOf(c.walk.WalkPhases(g14Sent.Add(time.Minute), c.provisioning(stale)), "g14"); g.State != RestWaiting { - t.Fatalf("a failure from before the send was read as this send's: %+v", g) - } -} - -// None within ApplySilentAfter names the machine silent, never counted applied; the others end the walk. -func TestNoProvisioningReportWithinTheBoundIsSilence(t *testing.T) { - c := walkHandingDunst(t) - g14Sent := c.walk.Modules["build-agent"].Rest["g14"].At - waiting := map[string]ProvReport{"g14.build-agent": {Status: ProvWaiting, WaitsFor: []string{"config:font-size"}, - At: g14Sent.Add(3 * time.Second)}} - w := c.walk.WalkPhases(g14Sent.Add(ApplySilentAfter+time.Minute), c.provisioning(waiting)) - sumsUp(t, w) - if g := restOf(w, "g14"); g.State != RestSilent { - t.Fatalf("g14 is not silent past ApplySilentAfter: %+v", g) - } - if len(w.Silent) != 1 || w.Silent[0] != "g14" || w.End == nil || !w.End.Equal(c.judged.Add(35*time.Second)) { - t.Fatalf("the walk does not end on the others, naming g14 silent: end %v, silent %v", w.End, w.Silent) - } -} - -// A send that handed nothing ends on the node-engine's report alone, as before. -func TestASendThatHandedNothingEndsOnTheEnginesReport(t *testing.T) { - c := walk582(t) - before := c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup) - now := c.walk.WalkPhases(c.judged.Add(time.Hour), c.provisioning(nil)) - if before.End == nil || now.End == nil || !now.End.Equal(*before.End) { - t.Fatalf("a walk handing nothing ends differently: before %v, now %v", before.End, now.End) - } -} diff --git a/internal/inventory/registered_held_integration_test.go b/internal/inventory/registered_held_integration_test.go deleted file mode 100644 index e690c21f..00000000 --- a/internal/inventory/registered_held_integration_test.go +++ /dev/null @@ -1,59 +0,0 @@ -//go:build integration - -package inventory - -import ( - "encoding/json" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -func TestWhatRunsIsTheRegisteredBuildNotANewerOnePutBack(t *testing.T) { - inv := fresh(t) - ctx := t.Context() - manifestOf := func(image string) catalogue.Manifest { - return catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{ - {"id": "app", "type": "container", "name": "one", "image": image}}} - } - record := func(id, image string, asked time.Time) Build { - t.Helper() - raw, _ := json.Marshal(manifestOf(image)) - b := Build{ID: id, Module: "one", Commit: id + "aaaaaaaa", Repository: "novox/one", Manifest: raw, - Asked: asked, At: asked, Made: []Artifact{{Name: "one", Kind: "image", Reference: image}}} - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, manifestOf(image), Source{Repository: "novox/one", BuiltFrom: b.Commit, - Asked: asked}); err != nil { - t.Fatal(err) - } - return b - } - x := "store/one@sha256:" + "1111111111111111111111111111111111111111111111111111111111111111" - y := "store/one@sha256:" + "2222222222222222222222222222222222222222222222222222222222222222" - before := record("x", x, time.Now().Add(-2*time.Hour)) - failed := record("y", y, time.Now().Add(-time.Minute)) - if err := inv.RecordGate(ctx, GateVerdict{Build: failed.ID, Module: "one", Commit: failed.Commit, - Machines: []string{"anchor"}, Verdict: GateFailed}); err != nil { - t.Fatal(err) - } - if err := inv.RestoreModule(ctx, before); err != nil { - t.Fatal(err) - } - held, err := inv.Held(ctx) - if err != nil { - t.Fatal(err) - } - running, err := inv.RegisteredHeld(ctx) - if err != nil { - t.Fatal(err) - } - if running["one/one"] == y { - t.Fatalf("what runs is read as the build whose gate failed (%s); Held says %s", y, held["one/one"]) - } - if got := running["one/one"]; got != "" && got != x { - t.Fatalf("what runs is read as %s, want the build put back %s or nothing", got, x) - } -} diff --git a/internal/inventory/sends.go b/internal/inventory/sends.go deleted file mode 100644 index 6ed98725..00000000 --- a/internal/inventory/sends.go +++ /dev/null @@ -1,364 +0,0 @@ -package inventory - -import ( - "context" - "fmt" - "time" - - "github.com/jackc/pgx/v5" -) - -// The assignment generation and the record of every send (novox/hq issue 234, migration 0094). -// -// A declaration carries the generation of the assignments it was composed from, and a machine refuses one -// composed from an older generation than it applied — the sequence orders arrival and cannot tell a later -// send carrying an older view. And every send is written down with who sent it, from which generation and -// naming which modules, so what a machine was last told, by whom, is read back rather than guessed at. - -// sendsKept is how many sends are kept per machine: the newest. Enough to read back a morning of pushes -// on a busy machine; the record is for finding who sent what, not an archive. -const sendsKept = 200 - -// AssignmentGeneration is the mesh's assignment generation now: raised by the store itself in the same -// transaction as every change to what is assigned where (a trigger on the assignment table). -func (i *Inventory) AssignmentGeneration(ctx context.Context) (int64, error) { - var g int64 - if err := i.store.Pool().QueryRow(ctx, `select generation from assignment_generation`).Scan(&g); err != nil { - return 0, fmt.Errorf("reading the assignment generation: %w", err) - } - return g, nil -} - -// RaiseAssignmentGeneration raises the generation past one a machine applied, and answers it: one more -// than that, or what it already was when it is past it. It never lowers it. -// -// For one case only: a machine refused a send composed from the generation the mesh holds now, so the -// machine applied a higher one than the mesh's counter — a store put back from a backup. Every send after -// would be refused for ever; raised past it, the next send carries a generation the machine takes, composed -// from the assignments the store holds now. A stale sender never meets this: its generation is below the -// counter, and the counter is not touched for it. -func (i *Inventory) RaiseAssignmentGeneration(ctx context.Context, past int64) (int64, error) { - var g int64 - err := i.store.Pool().QueryRow(ctx, - `update assignment_generation set generation = greatest(generation, $1 + 1) returning generation`, past).Scan(&g) - if err != nil { - return 0, fmt.Errorf("raising the assignment generation past %d: %w", past, err) - } - return g, nil -} - -// SentGeneration is the generation a machine was last sent, by its id; zero for one sent without. What the -// mesh WOULD send is stamped with it, so it reads as byte for byte what it DID send when nothing else changed. -func (i *Inventory) SentGeneration(ctx context.Context, id string) (int64, error) { - var g *int64 - if err := i.store.Pool().QueryRow(ctx, `select sent_generation from node where id = $1`, id).Scan(&g); err != nil { - return 0, fmt.Errorf("reading the generation %s was last sent: %w", id, err) - } - if g == nil { - return 0, nil - } - return *g, nil -} - -// ReadsGeneration says a machine's node-engine said it reads a generation in a declaration, by its id. -func (i *Inventory) ReadsGeneration(ctx context.Context, id string) (bool, error) { - var reads bool - if err := i.store.Pool().QueryRow(ctx, `select reads_generation from node where id = $1`, id).Scan(&reads); err != nil { - return false, fmt.Errorf("reading whether %s reads a generation: %w", id, err) - } - return reads, nil -} - -// RecordReadsGeneration keeps what a machine's latest report said of reading a generation. -func (i *Inventory) RecordReadsGeneration(ctx context.Context, id string, reads bool) error { - _, err := i.store.Pool().Exec(ctx, `update node set reads_generation = $2 where id = $1`, id, reads) - return err -} - -// ReadsModules says a machine's node-engine said it reads a module per resource and a build per module (issue 1145). -func (i *Inventory) ReadsModules(ctx context.Context, id string) (bool, error) { - var reads bool - if err := i.store.Pool().QueryRow(ctx, `select reads_modules from node where id = $1`, id).Scan(&reads); err != nil { - return false, fmt.Errorf("reading whether %s reads modules: %w", id, err) - } - return reads, nil -} - -// RecordReadsModules keeps what a machine's latest report said of reading modules. -func (i *Inventory) RecordReadsModules(ctx context.Context, id string, reads bool) error { - _, err := i.store.Pool().Exec(ctx, `update node set reads_modules = $2 where id = $1`, id, reads) - return err -} - -// Send is one declaration sent to a machine, as the mesh records it. -type Send struct { - // Node is the machine's id; NodeName its name, filled where it is read back. - Node string - NodeName string - Sequence int64 - // Epoch is the lease epoch its sender acted under, zero for one that acted under none. - Epoch int64 - // Sender is who sent it, in words: the caller, and the process and build that composed it. - Sender string - // Generation is the assignment generation it was composed from, zero when not known. - Generation int64 - Digest string - // Modules are the modules it named. - Modules []string - // Provisioning is the provisioner's plan of each module the send handed to it, by module, as JSON (novox/hq - // ADR 0349 §4, design 57 P6): kept on the machine's record with the send, nil when it handed none. Never in - // the record of the send itself. - Provisioning []byte - SentAt time.Time - // RefusedAt is when the machine refused it for its generation, nil when it did not; RefusedApplied the - // generation the machine said it had applied then; CounterRaisedTo what the mesh's counter was raised to - // because the refusal showed it behind the machine, zero when it was not. - RefusedAt *time.Time - RefusedApplied int64 - CounterRaisedTo int64 - // Recorded is false for a refusal of a sequence the mesh has no send for: its sender is unknown. - Recorded bool - // Answer is what the machine last reported of this send; filled only where a last send is read back - // (LastSend, LastSends), the zero answer elsewhere. - Answer SendAnswer -} - -// SendAnswer is what a machine reported of one send, read from its last report when that report is about the -// send's declaration (novox/hq issue 485): whether it applied it, failed part of it or refused it, and why. -// The zero answer is a machine that has not reported on this send — yet, or since a later report replaced it. -type SendAnswer struct { - // Outcome is applied, failed or refused, as the machine reported it; empty when it has not reported on it. - Outcome string - // Refused is the machine's own words when it refused the declaration whole. - Refused string - // Failed is how many of its resources failed, when some did. - Failed int - // At is when the machine reported it; nil when it has not. - At *time.Time -} - -// unknownSender is the sender of a refused sequence the mesh has no record of sending. -const unknownSender = "a sender the mesh has no record of (no send of this sequence was recorded: sent by " + - "hand, before sends were recorded, or by a controller whose record was not written)" - -// SendNotRecordedError is a send whose machine's record was written — its digest and the generation it -// carried — and whose record of who sent it was not. The send is away and the machine's record stands; the -// caller says this loudly and does not take the send for failed. -type SendNotRecordedError struct{ Err error } - -func (e *SendNotRecordedError) Error() string { - return "the record of the send was not written: " + e.Err.Error() -} -func (e *SendNotRecordedError) Unwrap() error { return e.Err } - -// RecordSentWith writes down what a machine was just sent — its digest, the builds it carried, the epoch and -// the generation on the wire — and the send itself, who sent it and from which generation (novox/hq issue -// 234), in one transaction. The digest and the generation are written together, so the would-send is never -// stamped with a generation the machine was not sent. The send's own record is written under a savepoint: if -// it cannot be, the machine's record is still committed and a *SendNotRecordedError says so. The oldest -// sends beyond the newest 200 for the machine are let go. -func (i *Inventory) RecordSentWith(ctx context.Context, node, digest string, builds map[string]string, epoch uint64, - generation int64, s Send) error { - var sentEpoch *int64 - if epoch > 0 { - e := int64(epoch) - sentEpoch = &e - } - tx, err := i.store.Pool().Begin(ctx) - if err != nil { - return err - } - defer func() { _ = tx.Rollback(ctx) }() - if _, err := tx.Exec(ctx, - `update node set sent = $2, sent_at = now(), sent_builds = null, sent_epoch = $3, sent_generation = $4, - sent_provisioning = $5::jsonb - where id = $1`, node, digest, sentEpoch, nullIfZero(generation), nullIfEmpty(s.Provisioning)); err != nil { - return err - } - recordErr := recordSend(ctx, tx, node, s) - if err := tx.Commit(ctx); err != nil { - return err - } - // And what it put on the machine, in the assignments (issues 1095, 1109): a write of it that fails fails this record, - // for what a machine was sent is read from there. - if err := i.recordRuns(ctx, node, builds, time.Now()); err != nil { - return fmt.Errorf("%s was sent its declaration, and what each module of it runs there could not be written: %w", node, err) - } - if recordErr != nil { - return &SendNotRecordedError{Err: recordErr} - } - return nil -} - -// recordSend writes one send under a savepoint of tx, which it rolls back when the send cannot be written. -func recordSend(ctx context.Context, tx pgx.Tx, node string, s Send) error { - sp, err := tx.Begin(ctx) - if err != nil { - return err - } - defer func() { _ = sp.Rollback(ctx) }() - modules := s.Modules - if modules == nil { - modules = []string{} - } - if _, err := sp.Exec(ctx, - `insert into declaration_send (node, sequence, epoch, sender, generation, digest, modules) - values ($1, $2, $3, $4, $5, $6, $7)`, - node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), s.Sender, nullIfZero(s.Generation), s.Digest, - modules); err != nil { - return err - } - if err := pruneSends(ctx, sp, node); err != nil { - return err - } - return sp.Commit(ctx) -} - -// pruneSends lets go of a machine's sends beyond the newest 200. -func pruneSends(ctx context.Context, q queries, node string) error { - _, err := q.Exec(ctx, - `delete from declaration_send where node = $1 and id not in - (select id from declaration_send where node = $1 order by id desc limit $2)`, node, sendsKept) - return err -} - -// sendColumns are a send's columns as scanSends reads them. -const sendColumns = `s.node, n.name, coalesce(s.sequence, 0), coalesce(s.epoch, 0), s.sender, coalesce(s.generation, 0), - s.digest, s.modules, s.sent_at, s.refused_at, coalesce(s.refused_applied, 0), coalesce(s.counter_raised_to, 0), - s.recorded` - -func scanSends(rows pgx.Rows) ([]Send, error) { - defer rows.Close() - var out []Send - for rows.Next() { - var s Send - if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.Digest, - &s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied, &s.CounterRaisedTo, &s.Recorded); err != nil { - return nil, err - } - out = append(out, s) - } - return out, rows.Err() -} - -// LastSend is the last declaration a machine was sent, by its name, with what the machine answered of it; false -// when none was recorded. A refusal of a sequence the mesh has no send for is not a send, and is not it. -func (i *Inventory) LastSend(ctx context.Context, name string) (Send, bool, error) { - sends, err := i.lastSends(ctx, `n.name = $1`, name) - if err != nil { - return Send{}, false, err - } - s, found := sends[name] - return s, found, nil -} - -// LastSends is every machine's last send, by machine name, each with what the machine answered of it: what -// `status` says per machine (novox/hq issue 485). A machine never sent anything has no entry. -func (i *Inventory) LastSends(ctx context.Context) (map[string]Send, error) { - return i.lastSends(ctx, `true`) -} - -// lastSends reads the newest recorded send of each machine matching where, beside the machine's last report -// when that report is about the send (novox/hq issue 485). It names the send's declaration (its digest), and: -// - from an ordered node-engine, one that reports the epoch and sequence it acted on, those are the send's -// own. A periodic report about an identical earlier send names the same digest and may arrive after a later -// send the machine never received; only the sequence tells them apart. It also holds for a fast machine -// whose report lands before the send's row is stamped. -// - from an engine that reports no order, the report came after the send — both times the store's own — so -// a machine sent again a declaration it once applied does not read as having applied the new send. -func (i *Inventory) lastSends(ctx context.Context, where string, args ...any) (map[string]Send, error) { - rows, err := i.store.Pool().Query(ctx, `select distinct on (n.name) `+sendColumns+`, - coalesce(r.outcome, ''), coalesce(r.refused, ''), case when jsonb_typeof(r.failed) = 'array' then jsonb_array_length(r.failed) else 0 end, r.at - from declaration_send s join node n on n.id = s.node - left join node_report r on r.node = s.node and r.declared <> '' and r.declared = s.digest - and case when r.reported_sequence is not null - then r.reported_sequence = s.sequence and coalesce(r.reported_epoch, 0) = coalesce(s.epoch, 0) - else r.at >= s.sent_at end - where s.recorded and `+where+` - order by n.name, s.id desc`, args...) - if err != nil { - return nil, err - } - defer rows.Close() - out := map[string]Send{} - for rows.Next() { - var s Send - if err := rows.Scan(&s.Node, &s.NodeName, &s.Sequence, &s.Epoch, &s.Sender, &s.Generation, &s.Digest, - &s.Modules, &s.SentAt, &s.RefusedAt, &s.RefusedApplied, &s.CounterRaisedTo, &s.Recorded, - &s.Answer.Outcome, &s.Answer.Refused, &s.Answer.Failed, &s.Answer.At); err != nil { - return nil, err - } - out[s.NodeName] = s - } - return out, rows.Err() -} - -// RefusedSend keeps a machine's refusal of the send of a sequence for the generation it came from, and -// answers that send, sender and all; false when no send of that sequence was recorded (RecordUnrecordedRefusal -// keeps that one). The newest of that sequence when there are several: a declaration a person sent by hand may -// repeat one. raisedTo is what the mesh's counter was raised to for it, zero for none. -func (i *Inventory) RefusedSend(ctx context.Context, node string, sequence, applied, raisedTo int64) (Send, bool, error) { - rows, err := i.store.Pool().Query(ctx, `with refused as ( - update declaration_send set refused_at = now(), refused_applied = $3, counter_raised_to = $4 - where id = (select id from declaration_send where node = $1 and sequence = $2 and recorded - order by id desc limit 1) - returning *) - select `+sendColumns+` from refused s join node n on n.id = s.node`, node, sequence, applied, - nullIfZero(raisedTo)) - if err != nil { - return Send{}, false, err - } - sends, err := scanSends(rows) - if err != nil || len(sends) == 0 { - return Send{}, false, err - } - return sends[0], true, nil -} - -// RecordUnrecordedRefusal keeps a machine's refusal of a sequence the mesh has no send for, as a row of its -// own whose sender is unknown, and answers it: S20 raises it like any other, because a refusal nobody can -// attribute is no quieter for that. s carries the machine, the sequence, epoch, generation and digest the -// refused declaration carried, and the refusal's applied generation and counter raise. -func (i *Inventory) RecordUnrecordedRefusal(ctx context.Context, s Send) (Send, error) { - rows, err := i.store.Pool().Query(ctx, `with refused as ( - insert into declaration_send (node, sequence, epoch, sender, generation, digest, refused_at, refused_applied, - counter_raised_to, recorded) - values ($1, $2, $3, $4, $5, $6, now(), $7, $8, false) returning *) - select `+sendColumns+` from refused s join node n on n.id = s.node`, - s.Node, nullIfZero(s.Sequence), nullIfZero(s.Epoch), unknownSender, nullIfZero(s.Generation), s.Digest, - s.RefusedApplied, nullIfZero(s.CounterRaisedTo)) - if err != nil { - return Send{}, err - } - sends, err := scanSends(rows) - if err != nil { - return Send{}, err - } - if len(sends) == 0 { - return Send{}, fmt.Errorf("the refusal of sequence %d was not kept", s.Sequence) - } - if err := pruneSends(ctx, i.store.Pool(), s.Node); err != nil { - return Send{}, err - } - return sends[0], nil -} - -// RefusedSendsSince is every send refused for its generation since a moment, newest first. -func (i *Inventory) RefusedSendsSince(ctx context.Context, since time.Time) ([]Send, error) { - rows, err := i.store.Pool().Query(ctx, `select `+sendColumns+` - from declaration_send s join node n on n.id = s.node - where s.refused_at >= $1 order by s.refused_at desc`, since) - if err != nil { - return nil, err - } - return scanSends(rows) -} - -// nullIfEmpty is raw JSON, or null for none. -func nullIfEmpty(raw []byte) *string { - if len(raw) == 0 { - return nil - } - text := string(raw) - return &text -} diff --git a/internal/inventory/sends_test.go b/internal/inventory/sends_test.go deleted file mode 100644 index f36cf94a..00000000 --- a/internal/inventory/sends_test.go +++ /dev/null @@ -1,342 +0,0 @@ -//go:build integration - -package inventory - -import ( - "errors" - "slices" - "testing" - "time" -) - -// The assignment generation and the record of every send (novox/hq issue 234). On 2026-10-04 a declaration -// newer in sequence than every other named four fewer modules than the assignments held, a machine applied -// it, and nothing the mesh kept could say who sent it or from what view. - -func generationNow(t *testing.T, inv *Inventory) int64 { - t.Helper() - g, err := inv.AssignmentGeneration(t.Context()) - if err != nil { - t.Fatal(err) - } - return g -} - -// **Every change to what is assigned where raises the generation** — an assignment and an unassignment — and a repeated -// assignment, which changes nothing, does not. -func TestEveryAssignmentChangeRaisesTheGeneration(t *testing.T) { - inv, node := aNodeWithModules(t, "postgres", "web") - ctx := t.Context() - start := generationNow(t, inv) - if start < 1 { - t.Fatalf("the generation starts at %d; zero is \"none claimed\" on the wire", start) - } - if _, err := inv.Assign(ctx, node, "postgres"); err != nil { - t.Fatal(err) - } - assigned := generationNow(t, inv) - if assigned <= start { - t.Fatalf("an assignment left the generation at %d", assigned) - } - if _, err := inv.Assign(ctx, node, "postgres"); err != nil { - t.Fatal(err) - } - if again := generationNow(t, inv); again != assigned { - t.Errorf("an assignment repeated, which changed nothing, moved the generation from %d to %d", assigned, again) - } - if err := inv.Unassign(ctx, node, "postgres"); err != nil { - t.Fatal(err) - } - unassigned := generationNow(t, inv) - if unassigned <= assigned { - t.Fatalf("an unassignment left the generation at %d", unassigned) - } - // A machine's removal, which no verb makes, is read from the machines the store knows: an assignment on a machine it - // does not know is none (assignments in the bucket have no key of the store to cascade through). -} - -// **The generation never goes down, and is raised past what a machine applied when the mesh's own counter -// is behind it** — a store put back from a backup — so the next send is not refused for ever. -func TestTheGenerationIsRaisedPastWhatAMachineApplied(t *testing.T) { - inv, _ := aNodeWithModules(t) - now := generationNow(t, inv) - if raised, err := inv.RaiseAssignmentGeneration(t.Context(), now+40); err != nil || raised != now+41 { - t.Fatalf("raised past %d to %d (%v)", now+40, raised, err) - } - if raised, err := inv.RaiseAssignmentGeneration(t.Context(), 2); err != nil || raised != now+41 { - t.Fatalf("a raise below the counter moved it to %d (%v)", raised, err) - } -} - -// **Every send is recorded: its sequence, its sender, the generation it came from and the modules it -// named** — and the machine's last send is what `plan` reads, the would-send is stamped with the generation -// written with the digest, and a refusal is kept on the send it refused, naming its sender. -func TestASendIsRecordedWithItsSenderGenerationAndModules(t *testing.T) { - inv, node := aNodeWithModules(t) - ctx := t.Context() - record, err := inv.NodeByName(ctx, node) - if err != nil { - t.Fatal(err) - } - first := Send{Sequence: 11, Epoch: 57, Sender: "the controller's daemon (pid 7 on anchor)", - Generation: 40, Digest: "d11", Modules: []string{"docker", "pacman", "sudo"}} - if err := inv.RecordSentWith(ctx, record.ID, "d11", nil, 57, 40, first); err != nil { - t.Fatal(err) - } - stale := Send{Sequence: 12, Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", - Generation: 38, Digest: "d12", Modules: []string{"docker"}} - if err := inv.RecordSentWith(ctx, record.ID, "d12", nil, 57, 38, stale); err != nil { - t.Fatal(err) - } - last, found, err := inv.LastSend(ctx, node) - if err != nil || !found { - t.Fatalf("the last send is not kept: %v", err) - } - if last.Sequence != 12 || last.Sender != stale.Sender || last.Generation != 38 || !last.Recorded || - !slices.Equal(last.Modules, []string{"docker"}) || last.SentAt.IsZero() { - t.Fatalf("the last send reads %+v", last) - } - if generation, err := inv.SentGeneration(ctx, record.ID); err != nil || generation != 38 { - t.Fatalf("what the machine was last sent of it reads %d (%v)", generation, err) - } - - refused, found, err := inv.RefusedSend(ctx, record.ID, 12, 40, 0) - if err != nil || !found || refused.Sender != stale.Sender || refused.RefusedApplied != 40 || - refused.CounterRaisedTo != 0 { - t.Fatalf("the refusal is not kept on the send it refused: %+v, %v, %v", refused, found, err) - } - since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour)) - if err != nil || len(since) != 1 || since[0].NodeName != node || since[0].Sequence != 12 { - t.Fatalf("the refused sends within the hour read %+v (%v)", since, err) - } - if _, found, err := inv.RefusedSend(ctx, record.ID, 99, 40, 0); err != nil || found { - t.Errorf("a refusal of a send never recorded was found: %v, %v", found, err) - } -} - -// **A refusal of a sequence the mesh has no send for is kept too, its sender unknown**, so S20 raises it; -// it is not the machine's last send, and it says what the counter was raised to. -func TestARefusalOfASendNeverRecordedIsKeptWithAnUnknownSender(t *testing.T) { - inv, node := aNodeWithModules(t) - ctx := t.Context() - record, err := inv.NodeByName(ctx, node) - if err != nil { - t.Fatal(err) - } - kept, err := inv.RecordUnrecordedRefusal(ctx, Send{Node: record.ID, Sequence: 99, Epoch: 57, Generation: 41, - Digest: "d99", RefusedApplied: 44, CounterRaisedTo: 45}) - if err != nil { - t.Fatal(err) - } - if kept.Recorded || kept.Sender != unknownSender || kept.RefusedAt == nil || kept.CounterRaisedTo != 45 || - kept.NodeName != node { - t.Fatalf("the refusal reads %+v", kept) - } - since, err := inv.RefusedSendsSince(ctx, time.Now().Add(-time.Hour)) - if err != nil || len(since) != 1 || since[0].Sequence != 99 { - t.Fatalf("the refused sends within the hour read %+v (%v)", since, err) - } - if _, found, err := inv.LastSend(ctx, node); err != nil || found { - t.Errorf("a refusal of a send never recorded reads as the machine's last send: %v, %v", found, err) - } -} - -// **The machine's record and the send's are written together, and a send record that cannot be written -// leaves the machine's record standing and says so**: the digest and generation are the machine's, and a -// send that is away must not read as failed, nor the machine as behind. -func TestASendRecordThatCannotBeWrittenLeavesTheMachinesRecord(t *testing.T) { - inv, node := aNodeWithModules(t) - ctx := t.Context() - record, err := inv.NodeByName(ctx, node) - if err != nil { - t.Fatal(err) - } - // A NUL byte is text PostgreSQL refuses: the send's own row cannot be written. - broken := Send{Sequence: 13, Sender: "a test", Generation: 42, Digest: "d\x0013"} - err = inv.RecordSentWith(ctx, record.ID, "d13", nil, 0, 42, broken) - var unrecorded *SendNotRecordedError - if !errors.As(err, &unrecorded) { - t.Fatalf("a send record that could not be written read as %v", err) - } - if generation, err := inv.SentGeneration(ctx, record.ID); err != nil || generation != 42 { - t.Fatalf("the machine's generation was not written with its digest: %d (%v)", generation, err) - } - if sent, err := inv.Outstanding(ctx, node); err != nil || sent != "d13" { - t.Fatalf("the machine's digest was not written: %q (%v)", sent, err) - } -} - -// **A machine that reads a generation is recorded from its reports**, and one rolled back says so no longer. -func TestWhetherAMachineReadsAGenerationIsItsLatestWord(t *testing.T) { - inv, node := aNodeWithModules(t) - record, err := inv.NodeByName(t.Context(), node) - if err != nil { - t.Fatal(err) - } - for _, reads := range []bool{false, true, false} { - if err := inv.RecordReadsGeneration(t.Context(), record.ID, reads); err != nil { - t.Fatal(err) - } - if got, err := inv.ReadsGeneration(t.Context(), record.ID); err != nil || got != reads { - t.Fatalf("recorded %v, read %v (%v)", reads, got, err) - } - } -} - -// **The last send says what the machine answered of it** (novox/hq issue 485): applied, refused with its -// words, or not answered yet — read from the machine's report only when that report is about the send's -// declaration and came after it, so a report about an earlier send is never this one's answer. And every -// machine's last send is read at once, for status, by name; a machine never sent anything has none. -func TestTheLastSendSaysWhatTheMachineAnswered(t *testing.T) { - inv, node := aNodeWithModules(t) - ctx := t.Context() - record, err := inv.NodeByName(ctx, node) - if err != nil { - t.Fatal(err) - } - if _, err := inv.AddNode(ctx, "laptop"); err != nil { - t.Fatal(err) - } - send := func(sequence int64, digest string) { - t.Helper() - if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57, - Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest, - Modules: []string{"docker", "sudo"}}); err != nil { - t.Fatal(err) - } - } - report := func(d Doing) { - t.Helper() - if _, err := inv.RecordDoing(ctx, record.ID, d); err != nil { - t.Fatal(err) - } - } - answer := func() SendAnswer { - t.Helper() - last, found, err := inv.LastSend(ctx, node) - if err != nil || !found { - t.Fatalf("the last send is not read back: %v, %v", found, err) - } - return last.Answer - } - - send(11, "d11") - if a := answer(); a.Outcome != "" || a.At != nil { - t.Fatalf("a send not reported on reads as answered: %+v", a) - } - report(Doing{Outcome: OutcomeApplied, Declared: "d11", Applied: 3}) - if a := answer(); a.Outcome != OutcomeApplied || a.At == nil { - t.Fatalf("a send the machine reported applying reads %+v", a) - } - - send(12, "d12") - if a := answer(); a.Outcome != "" { - t.Fatalf("the report about the send before is taken for this one's answer: %+v", a) - } - report(Doing{Outcome: OutcomeRefused, Declared: "d12", Refused: "unknown field \"generation\"\nand more"}) - if a := answer(); a.Outcome != OutcomeRefused || a.Refused != "unknown field \"generation\"\nand more" { - t.Fatalf("a send the machine refused reads %+v", a) - } - - // Sent again a declaration it reported applying before the send: not answered until it reports again. - report(Doing{Outcome: OutcomeApplied, Declared: "d12", Applied: 3}) - send(13, "d12") - if a := answer(); a.Outcome != "" { - t.Fatalf("a report from before the send is taken for its answer: %+v", a) - } - - all, err := inv.LastSends(ctx) - if err != nil { - t.Fatal(err) - } - if len(all) != 1 || all[node].Sequence != 13 || all[node].Epoch != 57 || all[node].Generation != 40 || - !slices.Equal(all[node].Modules, []string{"docker", "sudo"}) { - t.Fatalf("every machine's last send reads %+v", all) - } - if _, found := all["laptop"]; found { - t.Error("a machine never sent anything has a last send") - } -} - -// **A report about another declaration, or another send of the same one, is not the last send's answer** -// (novox/hq issue 485, review of mesh-controller #234). A report naming an earlier send's digest that arrives -// after a later send is not the later send's answer. And from an ordered node-engine, a periodic report about -// an identical earlier send — the same digest, arriving after the later send — is not its answer either: only -// the report's epoch and sequence say which send it is about. -func TestAReportIsTheLastSendsAnswerOnlyWhenItIsAboutThatSend(t *testing.T) { - inv, node := aNodeWithModules(t) - ctx := t.Context() - record, err := inv.NodeByName(ctx, node) - if err != nil { - t.Fatal(err) - } - send := func(sequence int64, digest string) { - t.Helper() - if err := inv.RecordSentWith(ctx, record.ID, digest, nil, 57, 40, Send{Sequence: sequence, Epoch: 57, - Sender: "the controller's daemon (pid 7 on anchor)", Generation: 40, Digest: digest}); err != nil { - t.Fatal(err) - } - } - ordered := func(epoch, sequence int64, digest string) { - t.Helper() - if _, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: digest, Applied: 1}, - ReportOrder{Epoch: epoch, Sequence: sequence}, func(ReportOrder) bool { return false }); err != nil { - t.Fatal(err) - } - } - answer := func() string { - t.Helper() - last, found, err := inv.LastSend(ctx, node) - if err != nil || !found { - t.Fatalf("the last send is not read back: %v, %v", found, err) - } - return last.Answer.Outcome - } - - // An engine that reports no order: the digest decides. - send(11, "d11") - send(12, "d12") - if _, err := inv.RecordDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d11"}); err != nil { - t.Fatal(err) - } - if a := answer(); a != "" { - t.Fatalf("a report about d11, after d12 was sent, reads as d12's answer: %q", a) - } - - // An ordered engine: the epoch and sequence decide, whatever the digest and the time. - send(20, "d20") - ordered(57, 20, "d20") - if a := answer(); a != OutcomeApplied { - t.Fatalf("an ordered report about sequence 20 is not its answer: %q", a) - } - send(21, "d20") - ordered(57, 20, "d20") - if a := answer(); a != "" { - t.Fatalf("a periodic report about sequence 20, after 21 was sent with the same digest, reads as 21's answer: %q", a) - } - ordered(56, 21, "d20") - if a := answer(); a != "" { - t.Fatalf("a report about sequence 21 of another epoch reads as this send's answer: %q", a) - } - ordered(57, 21, "d20") - if a := answer(); a != OutcomeApplied { - t.Fatalf("an ordered report about sequence 21 is not its answer: %q", a) - } -} - -// 1145/AC3: whether a machine reads modules is its latest word, as whether it reads a generation is. -func TestWhetherAMachineReadsModulesIsItsLatestWord(t *testing.T) { - inv, node := aNodeWithModules(t) - record, err := inv.NodeByName(t.Context(), node) - if err != nil { - t.Fatal(err) - } - for _, reads := range []bool{false, true, false} { - if err := inv.RecordReadsModules(t.Context(), record.ID, reads); err != nil { - t.Fatal(err) - } - if got, err := inv.ReadsModules(t.Context(), record.ID); err != nil || got != reads { - t.Fatalf("1145/AC3: recorded %v, read %v (%v)", reads, got, err) - } - } -} diff --git a/internal/inventory/sent_builds_test.go b/internal/inventory/sent_builds_test.go deleted file mode 100644 index eeea1567..00000000 --- a/internal/inventory/sent_builds_test.go +++ /dev/null @@ -1,86 +0,0 @@ -//go:build integration - -package inventory - -import ( - "reflect" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" -) - -// novox/hq issue 259: a send keeps which build of each module it carried. A machine never sent -// anything, or sent with nothing recorded — before this was kept, or by hand — is not known, which -// is not the same as having been sent nothing. -func TestASendKeepsTheBuildsItCarried(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - node, err := inv.AddNode(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || known || builds != nil { - t.Fatalf("a machine never sent anything has known builds %v (%v): %v", builds, known, err) - } - - carried := map[string]string{"resolver": "abc123", "network": ""} - if err := inv.RecordSent(ctx, node.ID, "d1", carried); err != nil { - t.Fatal(err) - } - // What it was sent is read from the keys of the machine's assignments that name a commit (issue 1109): a module carried - // with no commit known names none. - builds, known, err := inv.SentBuilds(ctx, "anchor") - if err != nil || !known || !reflect.DeepEqual(builds, map[string]string{"resolver": "abc123"}) { - t.Fatalf("the builds sent were not kept: %v %v %v", builds, known, err) - } - - // Sent with nothing carried: known, and empty. - if err := inv.RecordSent(ctx, node.ID, "d2", map[string]string{}); err != nil { - t.Fatal(err) - } - if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || !known || len(builds) != 0 { - t.Fatalf("an empty send: %v %v %v", builds, known, err) - } - - // Sent by hand: not known, and the digest still recorded. - if err := inv.RecordSent(ctx, node.ID, "d3", nil); err != nil { - t.Fatal(err) - } - if builds, known, err := inv.SentBuilds(ctx, "anchor"); err != nil || known || builds != nil { - t.Fatalf("a send whose builds are not known read as %v %v: %v", builds, known, err) - } - if sent, err := inv.Outstanding(ctx, "anchor"); err != nil || sent != "d3" { - t.Fatalf("the digest was not recorded with it: %q %v", sent, err) - } - - if _, known, err := inv.SentBuilds(ctx, "nobody"); err != nil || known { - t.Fatalf("a machine the mesh does not know: %v %v", known, err) - } -} - -// A module's current build is the commit its manifest was read at, with its upgrade policy. -func TestTheCurrentBuildsAreTheCatalogues(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "resolver", Version: "1"}, - Source{Repository: "novox/mesh-catalog", BuiltFrom: "c1"}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "by-hand", Version: "1"}, Source{}); err != nil { - t.Fatal(err) - } - if err := inv.SetUpgradeOf(ctx, "by-hand", Upgrade{RollOut: true}); err != nil { - t.Fatal(err) - } - current, err := inv.CurrentBuilds(ctx) - if err != nil { - t.Fatal(err) - } - // Rolled out by default, one machine first and gated (novox/hq ADR 0236). - if got := current["resolver"]; got != (CurrentBuild{Commit: "c1", RollOut: true, From: catalogue.FromDefault}) { - t.Errorf("resolver is at %+v", got) - } - if got := current["by-hand"]; got != (CurrentBuild{RollOut: true, From: catalogue.FromPerson}) { - t.Errorf("a module with no source is at %+v", got) - } -} diff --git a/internal/inventory/sent_bus_users_test.go b/internal/inventory/sent_bus_users_test.go deleted file mode 100644 index 33439454..00000000 --- a/internal/inventory/sent_bus_users_test.go +++ /dev/null @@ -1,27 +0,0 @@ -//go:build integration - -package inventory - -import "testing" - -// novox/hq issue 249: the digest of the user list a machine was last sent is kept, by its name, and -// is empty for a machine never sent one. -func TestTheUserListAMachineWasSentIsKept(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - if _, err := inv.AddNode(ctx, "anchor"); err != nil { - t.Fatal(err) - } - if sent, err := inv.SentBusUsers(ctx, "anchor"); err != nil || sent != "" { - t.Fatalf("a machine never sent a list has %q: %v", sent, err) - } - if err := inv.RecordSentBusUsers(ctx, "anchor", "abc"); err != nil { - t.Fatal(err) - } - if sent, err := inv.SentBusUsers(ctx, "anchor"); err != nil || sent != "abc" { - t.Fatalf("the list sent was not kept: %q %v", sent, err) - } - if sent, err := inv.SentBusUsers(ctx, "nobody"); err != nil || sent != "" { - t.Fatalf("a machine the mesh does not know: %q %v", sent, err) - } -} diff --git a/internal/inventory/sent_plans_test.go b/internal/inventory/sent_plans_test.go deleted file mode 100644 index fb1a081c..00000000 --- a/internal/inventory/sent_plans_test.go +++ /dev/null @@ -1,95 +0,0 @@ -//go:build integration - -package inventory - -import ( - "encoding/json" - "reflect" - "strings" - "testing" -) - -// A send keeps the provisioner's plans it handed (design 57 P6, issue 672, migration 0100): the assignments mirror -// carries the plan of the build that machine was sent, in the module's own assignment and no other, and a walk -// reads which templates the send handed. -func TestASendKeepsThePlansItHandedAndTheMirrorCarriesThem(t *testing.T) { - inv, node := aNodeWithModules(t, "dunst", "web") - ctx := t.Context() - for _, m := range []string{"dunst", "web"} { - if _, err := inv.Assign(ctx, node, m); err != nil { - t.Fatal(err) - } - } - record, err := inv.NodeByName(ctx, node) - if err != nil { - t.Fatal(err) - } - digest := "sha256:" + strings.Repeat("d", 64) - plan := Provisioning{Templates: TemplatesArchive{Source: "http://store:5000/v2/dunst/files/blobs/" + digest, - Digest: digest}, Files: []ProvisionedFile{{ID: "configuration", Path: "/home/op/.config/dunst/dunstrc", - Template: "dunstrc", OwnerAtRuntime: "owned", Takes: "reload-on"}}} - // A plan for a module the machine no longer has assigned is kept with the send and mirrored nowhere; one for a - // module only pulled in by another is mirrored under its own key, saying what requires it (issue 765). - raw := []byte(`{"dunst":` + mustJSON(t, plan) + `,"gone":` + mustJSON(t, plan) + `,"notify":` + - strings.Replace(mustJSON(t, plan), `{`, `{"required-by":"desk",`, 1) + `}`) - if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, Send{Digest: "first", Provisioning: raw}); err != nil { - t.Fatal(err) - } - - sent, _, err := inv.SentTo(ctx, node) - if err != nil || !reflect.DeepEqual(sent.Provisioned, map[string]string{"dunst": digest, "gone": digest, - "notify": digest}) { - t.Fatalf("the send says it handed %v (%v)", sent.Provisioned, err) - } - plans, err := inv.SentProvisioningPlans(ctx) - required := plan - required.RequiredBy = "desk" - if err != nil || !reflect.DeepEqual(plans, map[string]Provisioning{node + ".dunst": plan, node + ".notify": required}) { - t.Fatalf("the plans mirrored are %+v (%v)", plans, err) - } - inv.SetProvisioningPlans(inv.SentProvisioningPlans) - mirror := inv.PlanValuesForTest(t) - var dunst, web AssignmentMirrored - if err := json.Unmarshal(mirror[node+".dunst"], &dunst); err != nil { - t.Fatal(err) - } - if err := json.Unmarshal(mirror[node+".web"], &web); err != nil { - t.Fatal(err) - } - if dunst.Provisioning == nil || !reflect.DeepEqual(*dunst.Provisioning, plan) || web.Provisioning != nil { - t.Fatalf("the mirror carries %+v for dunst and %+v for web", dunst.Provisioning, web.Provisioning) - } - var notify AssignmentMirrored - if err := json.Unmarshal(mirror[node+".notify"], ¬ify); err != nil { - t.Fatalf("no value for the pulled-in module: %s (%v)", mirror[node+".notify"], err) - } - if notify.RequiredBy != "desk" || !notify.Assigned.IsZero() || notify.Provisioning == nil || - !reflect.DeepEqual(*notify.Provisioning, plan) { - t.Fatalf("the mirror carries %s for the pulled-in module", mirror[node+".notify"]) - } - if _, has := mirror[node+".gone"]; has { - t.Fatalf("a module no longer assigned is mirrored: %s", mirror[node+".gone"]) - } - - // The next send hands nothing: nothing is kept, and the assignment's value is the bytes it was. - if err := inv.RecordSentWith(ctx, record.ID, "second", nil, 0, 0, Send{Digest: "second"}); err != nil { - t.Fatal(err) - } - if sent, _, err := inv.SentTo(ctx, node); err != nil || sent.Provisioned != nil { - t.Fatalf("a send handing nothing says it handed %v (%v)", sent.Provisioned, err) - } - if mirror := inv.PlanValuesForTest(t); strings.Contains(string(mirror[node+".dunst"]), "provisioning") { - t.Fatalf("dunst's assignment still carries a plan: %s", mirror[node+".dunst"]) - } else if _, has := mirror[node+".notify"]; has { - t.Fatalf("a module the machine no longer runs is mirrored: %s", mirror[node+".notify"]) - } -} - -func mustJSON(t *testing.T, v any) string { - t.Helper() - raw, err := json.Marshal(v) - if err != nil { - t.Fatal(err) - } - return string(raw) -} diff --git a/internal/inventory/source.go b/internal/inventory/source.go index a498b07b..1b2ca3ae 100644 --- a/internal/inventory/source.go +++ b/internal/inventory/source.go @@ -30,7 +30,6 @@ type sourceRow struct { made []byte manifest []byte manifestDigest string - failedGate bool } // sourceRows is every successful build of each module (of one module, when named), newest request @@ -53,13 +52,10 @@ func (i *Inventory) sourceRows(ctx context.Context, module string) (map[string][ func standing(builds []sourceRow, from int) int { at := from fp := builds[from].fingerprint - if fp == "" || builds[from].failedGate { + if fp == "" { return at } for j := from + 1; j < len(builds); j++ { - if builds[j].failedGate { - break - } if builds[j].fingerprint != fp { if !newForm(fp) || newForm(builds[j].fingerprint) || !sameDigests(builds[at].made, builds[j].made) { break diff --git a/internal/inventory/stored_silent_file_test.go b/internal/inventory/stored_silent_file_test.go index 5840e6f3..715bcf1b 100644 --- a/internal/inventory/stored_silent_file_test.go +++ b/internal/inventory/stored_silent_file_test.go @@ -39,10 +39,14 @@ func TestAStoredBuildWithASilentMergeableFileIsStillReadAndComposed(t *testing.T t.Fatal(err) } - m, found, err := inv.ManifestAt(ctx, "mail-client", b.Commit) + kept, found, err := inv.BuildByID(ctx, b.ID) if err != nil || !found { t.Fatalf("the stored build is not read again: found %v, %v", found, err) } + m, err := catalogue.ParseManifest(kept.Manifest) + if err != nil { + t.Fatalf("the stored manifest is not read again: %v", err) + } if written := catalogue.WrittenSettingsProblems(m); len(written) != 1 { t.Fatalf("the rule this build predates should name its accounts file where it is written: %v", written) } diff --git a/internal/inventory/tier_entered_resume_1886_test.go b/internal/inventory/tier_entered_resume_1886_test.go deleted file mode 100644 index f2263f09..00000000 --- a/internal/inventory/tier_entered_resume_1886_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package inventory - -import ( - "testing" - "time" -) - -// Issue 1886: R4e entered tier 1 at 02:01, was stopped, and was resumed at 07:43 at the same tier. Its save kept the -// tier's entry of 02:01, so at 07:50 the plan read "waiting for mesh-build-image on shanks to be applied for 5h48m53s" -// and its progress watchdog measured the tier from before the stop. A walk set open again at the tier it ended at -// enters that tier anew: its time in the tier runs from the resume. -func TestAWalkResumedAtItsTierEntersItAnew(t *testing.T) { - entered := time.Date(2026, 10, 23, 0, 1, 7, 0, time.UTC) - resumed := time.Date(2026, 10, 23, 5, 43, 30, 0, time.UTC) - old := &Plan{ID: "plan-r4e", State: PlanFailed, Tier: 1, Tiers: [][]string{{"a"}, {"b"}, {"c"}}, - Created: entered.Add(-2 * time.Hour), TierEntered: entered} - p := *old - p.State = PlanBuilding - got, d := tierLeftOf(old, p, resumed) - if !got.Equal(resumed) || d != nil { - t.Fatalf("1886: a walk resumed at tier 1 entered it at %s (duration %v); want %s, no tier left", got, d, resumed) - } - // A walk still in its tier keeps its entry; one that ended keeps it too. - open := *old - open.State = PlanRolling - if got, _ := tierLeftOf(&open, open, resumed); !got.Equal(entered) { - t.Fatalf("an open walk still in its tier was stamped anew: %s", got) - } - if got, _ := tierLeftOf(old, *old, resumed); !got.Equal(entered) { - t.Fatalf("a failed walk saved again was stamped anew: %s", got) - } -} diff --git a/internal/inventory/waiting_merges_test.go b/internal/inventory/waiting_merges_test.go deleted file mode 100644 index 4601e937..00000000 --- a/internal/inventory/waiting_merges_test.go +++ /dev/null @@ -1,33 +0,0 @@ -package inventory - -import ( - "context" - "testing" - "time" -) - -// A merge no plan answers waits, and is listed as waiting until it is answered where it is kept (novox/hq issue 1146): -// what a first merge into a new repository does while it waits for mesh-delivery's word. -func TestAMergeNoPlanAnswersIsListedWaitingUntilItIsAnswered(t *testing.T) { - inv, c := anInventoryOnTheChain(t) - ctx := context.Background() - now := time.Now().UTC() - if added, err := inv.chainAddMerge(ctx, c, BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: "c1", - Merged: now, Heard: now}); err != nil || !added { - t.Fatalf("added %v, %v", added, err) - } - waiting, err := inv.chainWaitingMerges(ctx, c) - if err != nil || len(waiting) != 1 || waiting[0].Commit != "c1" { - t.Fatalf("waiting %+v, %v", waiting, err) - } - move := []MergeMove{{Repository: "novox/mesh-delivery", Commit: "c1", Plan: WaitingRecord}} - if err := inv.chainSavePlans(ctx, c, nil, move); err != nil { - t.Fatal(err) - } - if waiting, err = inv.chainWaitingMerges(ctx, c); err != nil || len(waiting) != 0 { - t.Fatalf("answered, it still waits: %+v, %v", waiting, err) - } - if _, known, err := inv.chainMergeOf(ctx, c, "novox/mesh-delivery", "c1"); err != nil || !known { - t.Fatalf("an answered merge is no longer kept: %v %v", known, err) - } -} diff --git a/internal/inventory/walkphases.go b/internal/inventory/walkphases.go deleted file mode 100644 index e7d42a45..00000000 --- a/internal/inventory/walkphases.go +++ /dev/null @@ -1,676 +0,0 @@ -package inventory - -import ( - "context" - "fmt" - "sort" - "strings" - "time" -) - -// Where a walk's time went (novox/hq ADR 0282 decision 6, issue 382): from its merge to every machine of its -// rest running its builds, phase by phase, worked out from the walk's own moments. Measurement only: nothing -// here decides anything about the walk. -// -// **A phase that cannot be measured is said unknown, never zero.** Where a moment is missing (a walk kept -// before it was recorded, a machine not yet reported), the phases around it are unknown, and the span between -// the moments on either side is counted as unknown time: the measured phases and the unknown time always add -// up to the total. A phase that did not happen (no window for a merge walked alone, no first machine for a -// module nobody runs) is said none, with no time. - -// The phases, in the order a walk passes them (ADR 0282's table). -const ( - PhaseWindow = "window" // the merge to its batch's window closing - PhaseQueued = "queued" // the window closed to the walk being cut: waiting behind another walk - PhaseWord = "word" // the cut to the delivery's word, for a walk that waits for one - PhaseBetween = "between-tiers" // one tier's end to the next tier's ask - PhaseBuild = "build" // a tier asked to its last module built - PhaseSend = "send-first" // built to sent to the first machines - PhaseBusy = "busy-wait" // sent first to a busy container there replaced, when one waited (ADR 0335) - PhaseJudge = "judgement" // sent first to the first-node gate's last verdict: its readings - PhaseRest = "send-rest" // judged to sent to the rest - PhaseApply = "apply" // the last send to every machine of the rest reporting the build applied - PhaseOpen = "open" // a walk still running: since its last moment - PhaseMeasured = "measured" - PhaseUnknown = "unknown" - PhaseNone = "none" -) - -// The classes of a walk (ADR 0282 decision 1) and their delivery budgets. -const ( - ClassCore = "core" - ClassLeaf = "leaf" -) - -// ApplySilentAfter is how long after a send to the rest a machine that has said nothing is left out of the -// walk's end, as a machine not heard from (ADR 0282 decision 1: a sleeping laptop is listed, not waited for). -const ApplySilentAfter = 15 * time.Minute - -// WalkPhases is a walk's time, phase by phase. -type WalkPhases struct { - Class string `json:"class,omitempty"` - // From is the walk's earliest merge; End when its last phase ended: every machine of its rest reported the - // build applied. Nil End while that is not known. - From *time.Time `json:"from,omitempty"` - End *time.Time `json:"end,omitempty"` - // TotalMS is End − From; zero while either is unknown, Total its words. - TotalMS int64 `json:"total_ms,omitempty"` - Total string `json:"total,omitempty"` - // UnknownMS is the time within the walk no phase could be measured over. - UnknownMS int64 `json:"unknown_ms,omitempty"` - Phases []WalkPhase `json:"phases"` - Silent []string `json:"silent,omitempty"` - // Rest is each machine of the rest's own report of the send that carried the walk's build there (novox/hq ADR - // 0282 decision 6, issue 582): when it was sent, when it said, and what. - Rest []RestReport `json:"rest,omitempty"` - Said string `json:"said"` - Merges []MergeStart `json:"merges,omitempty"` -} - -// MergeStart is one merge the walk answers and when it was made: where that merge's delivery time starts. -type MergeStart struct { - Repository string `json:"repository"` - Commit string `json:"commit"` - Merged time.Time `json:"merged"` -} - -// WalkPhase is one phase of a walk. -type WalkPhase struct { - Name string `json:"name"` - // Tier is the tier a tier's phase belongs to; -1 for a phase of the walk. - Tier int `json:"tier"` - State string `json:"state"` - Start *time.Time `json:"start,omitempty"` - End *time.Time `json:"end,omitempty"` - TookMS int64 `json:"took_ms,omitempty"` - Took string `json:"took,omitempty"` - Said string `json:"said,omitempty"` -} - -// AppliedReport is a machine's first report, after a send, that it applied what it was sent. -type AppliedReport struct { - At time.Time - Outcome string - // Answered is, for a machine that waited for a busy container (novox/hq ADR 0335), its first report of the - // send; Busy how long it then waited; Waiting that it still waits, At being its first report then. - Answered time.Time - Busy time.Duration - Waiting bool -} - -// AppliedLookup answers a machine's report of the send the walk made to it: send is what the walk kept of it, and -// walkSent the walk's own record of its send to the rest; false when it has not reported. -type AppliedLookup func(node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool) - -// RestReport is one machine of the rest and its own report of a send of the walk (novox/hq issue 582). -type RestReport struct { - Node string `json:"node"` - Modules []string `json:"modules"` - // Sent is when that machine was sent, by its own send time where the walk kept it; else the walk's record of - // its send to the rest, and SentSaid says so. - Sent time.Time `json:"sent"` - SentSaid string `json:"sent_said,omitempty"` - Sequence int64 `json:"sequence,omitempty"` - Reported *time.Time `json:"reported,omitempty"` - Took string `json:"took,omitempty"` - Outcome string `json:"outcome,omitempty"` - // Busy is how long it waited for a busy container before it ran the build (novox/hq ADR 0335): inside Took, - // never read as the machine being silent. - Busy string `json:"busy,omitempty"` - // State is applied, failed, waiting (not yet heard, within ApplySilentAfter), busy (it answered and waits for - // a busy container: never silent) or silent (not heard within it: listed, never moving the walk's end). - State string `json:"state"` - busy time.Duration -} - -// The states of a machine of the rest. -const ( - RestApplied = "applied" - RestFailed = "failed" - RestWaiting = "waiting" - RestBusy = "busy" - RestSilent = "silent" -) - -// point is one moment of the walk, ending the phase named. -type point struct { - phase string - tier int - at *time.Time - none bool // the phase did not happen - said string // why it is none, or unknown -} - -// Phases is the walk's time phase by phase, at now; applied answers the rest's reports (nil: none read). -func (p Plan) WalkPhases(now time.Time, applied AppliedLookup) *WalkPhases { - if p.Release != nil || p.Batch() { - return nil - } - w := &WalkPhases{} - if p.Times != nil { - w.Class = p.Times.Class - } - from := p.firstMerge(w) - if from != nil { - f := from.Truncate(time.Millisecond) - from = &f - } - if from == nil { - w.Said = "when its merge was made is not kept: its delivery time is unknown" - } else { - w.From = from - } - points := p.points(now, applied, w) - // Walk the moments: each known moment after a known one is a measured phase; a missing moment makes the - // phases up to the next known one unknown, and their span unknown time. - last := from - var pending []int - for _, pt := range points { - if pt.none { - w.Phases = append(w.Phases, WalkPhase{Name: pt.phase, Tier: pt.tier, State: PhaseNone, Said: pt.said}) - continue - } - ph := WalkPhase{Name: pt.phase, Tier: pt.tier, Said: pt.said} - if pt.at == nil { - ph.State = PhaseUnknown - w.Phases = append(w.Phases, ph) - pending = append(pending, len(w.Phases)-1) - continue - } - at := pt.at.Truncate(time.Millisecond) - ph.End = &at - if last != nil && at.Before(*last) { - // **Out of order** (issue 382): this moment was recorded before the one before it — a module's first - // send kept before its build was, two clocks, a late record. Neither phase can be measured honestly: - // the one before it is said unknown and its time joins the unknown time, and this one is said unknown - // with no time. The walk goes on from the later moment, so nothing is negative and the sum holds. - for i := len(w.Phases) - 1; i >= 0; i-- { - prev := &w.Phases[i] - if prev.End == nil { - continue // a phase that did not happen, or is unknown with no moment: the one before carries last - } - if prev.State == PhaseMeasured { - w.UnknownMS += prev.TookMS - } - prev.State, prev.Start, prev.TookMS, prev.Took = PhaseUnknown, nil, 0, "" - prev.Said = "out of order: the phase after it ended first" - break - } - ph.State, ph.Said = PhaseUnknown, "out of order: it ended before the phase before it" - w.Phases = append(w.Phases, ph) - pending = nil - continue - } - if last != nil && len(pending) == 0 { - start := *last - ph.State, ph.Start = PhaseMeasured, &start - ph.TookMS = at.Sub(start).Milliseconds() - ph.Took = words(at.Sub(start)) - } else { - ph.State = PhaseUnknown - if last != nil { - w.UnknownMS += at.Sub(*last).Milliseconds() - } - } - w.Phases = append(w.Phases, ph) - pending = nil - last = &at - } - if len(pending) > 0 { - // The walk's last moments are unknown: it has no end. - if w.Said == "" { - w.Said = "its end is unknown: " + w.Phases[pending[0]].Name + " " + orNot(w.Phases[pending[0]].Said) - } - return w - } - if last == nil || from == nil { - return w - } - if p.Open() { - since := *last - w.Phases = append(w.Phases, WalkPhase{Name: PhaseOpen, Tier: -1, State: PhaseOpen, Start: &since, - TookMS: now.Sub(since).Milliseconds(), Took: words(now.Sub(since)), Said: "the walk is " + p.State}) - w.Said = "open: " + p.State + ", " + words(now.Sub(*from)) + " since its merge" - return w - } - if p.State != PlanDone { - w.Said = "ended " + p.State + ": no delivery time" - return w - } - end := *last - w.End = &end - w.TotalMS = end.Sub(*from).Milliseconds() - w.Total = words(end.Sub(*from)) - if w.Said == "" { - if w.UnknownMS > 0 { - w.Said = fmt.Sprintf("%s from its merge to running everywhere, %s of it unknown", w.Total, - words(time.Duration(w.UnknownMS)*time.Millisecond)) - } else { - w.Said = w.Total + " from its merge to running everywhere" - } - } - return w -} - -// firstMerge is when the walk's earliest merge was made, and every merge's start kept on w. -func (p Plan) firstMerge(w *WalkPhases) *time.Time { - var first *time.Time - if p.Delivery != nil { - for _, m := range p.Delivery.Merges { - if m.Merged.IsZero() { - continue - } - w.Merges = append(w.Merges, MergeStart{Repository: m.Repository, Commit: m.Commit, Merged: m.Merged}) - if first == nil || m.Merged.Before(*first) { - at := m.Merged - first = &at - } - } - } - if first == nil { - for _, c := range p.Carried() { - if c.Merged.IsZero() { - continue - } - w.Merges = append(w.Merges, MergeStart{Repository: c.Repository, Commit: c.Commit, Merged: c.Merged}) - if first == nil || c.Merged.Before(*first) { - at := c.Merged - first = &at - } - } - } - return first -} - -// points are the walk's moments in order. -func (p Plan) points(now time.Time, applied AppliedLookup, w *WalkPhases) []point { - var out []point - // The window and the wait behind another walk. - cut := p.Created - if p.Times != nil && p.Times.Cut != nil { - cut = *p.Times.Cut - } - switch { - case p.Times == nil: - out = append(out, point{phase: PhaseWindow, tier: -1, said: "not kept for a walk made before ADR 0282"}, - point{phase: PhaseQueued, tier: -1, at: &cut, said: "the window and the wait behind another walk together"}) - case p.Times.WindowClosed == nil: - out = append(out, point{phase: PhaseWindow, tier: -1, none: true, said: "no window: walked on its own"}, - point{phase: PhaseQueued, tier: -1, at: &cut}) - default: - closed := *p.Times.WindowClosed - out = append(out, point{phase: PhaseWindow, tier: -1, at: &closed}, point{phase: PhaseQueued, tier: -1, at: &cut}) - } - if p.Delivery != nil && p.Delivery.Awaits != "" { - out = append(out, point{phase: PhaseWord, tier: -1, at: p.Delivery.Go, said: "waiting for " + p.Delivery.Awaits + "'s word"}) - } else { - out = append(out, point{phase: PhaseWord, tier: -1, none: true, said: "waits for no word"}) - } - var lastSends []restSend - for t, tier := range p.Tiers { - if t > p.Tier || (t == p.Tier && p.Tier < len(p.Tiers) && !askedAnyOf(p, tier)) { - break - } - var asked, built, first, judged, rest, busyEnded *time.Time - allBuilt, anyFirst, allJudged, allRest := true, false, true, true - // A busy container on a first machine holds the judging until it is replaced (novox/hq ADR 0335 §4): a - // phase of its own, only in a tier where one waited, counted in the delivery time and never exempted. - anyBusy, allBusyEnded, busyOn := false, true, "" - for _, m := range tier { - s := p.Modules[m] - if s == nil { - allBuilt, allRest = false, false - continue - } - asked = earliest(asked, s.AskedAt) - if s.BuiltAt == nil { - if s.State != "deleted" { - allBuilt = false - } - } else { - built = latest(built, s.BuiltAt) - } - if s.FirstAt != nil { - anyFirst = true - first = earliest(first, s.FirstAt) - if s.Gate != nil && s.Gate.Busy != nil { - anyBusy = true - if s.Gate.Busy.Ended == nil { - allBusyEnded, busyOn = false, s.Gate.Busy.Machine - } else { - busyEnded = latest(busyEnded, s.Gate.Busy.Ended) - } - } - if s.Gate != nil { - if s.Gate.JudgedAt == nil { - allJudged = false - } else { - judged = latest(judged, s.Gate.JudgedAt) - } - } - } - if s.SentAt == nil { - if s.State != "deleted" { - allRest = false - } - } else { - rest = latest(rest, s.SentAt) - for node, what := range s.Rest { - lastSends = append(lastSends, restSend{module: m, node: node, walkSent: *s.SentAt, what: what}) - } - } - } - if t > 0 { - out = append(out, point{phase: PhaseBetween, tier: t, at: asked}) - } - if !allBuilt { - built = nil - } - out = append(out, point{phase: PhaseBuild, tier: t, at: built}) - if anyFirst { - if !allJudged { - judged = nil - } - out = append(out, point{phase: PhaseSend, tier: t, at: first}) - if anyBusy { - said := "" - if !allBusyEnded { - busyEnded, said = nil, "waits for a busy container on "+busyOn - } - out = append(out, point{phase: PhaseBusy, tier: t, at: busyEnded, said: said}) - } - out = append(out, point{phase: PhaseJudge, tier: t, at: judged}) - } else { - out = append(out, point{phase: PhaseSend, tier: t, none: true, said: "no first machine: nothing to judge"}, - point{phase: PhaseJudge, tier: t, none: true, said: "no first machine: nothing to judge"}) - } - if !allRest { - rest = nil - } - out = append(out, point{phase: PhaseRest, tier: t, at: rest}) - } - if p.State != PlanDone { - return out - } - // Every machine of the rest running the build: its first report after the send, applied. - if p.Times == nil { - return append(out, point{phase: PhaseApply, tier: -1, said: "the rest's sends are not kept for a walk made before ADR 0282"}) - } - if len(lastSends) == 0 { - // novox/hq issue 720: since ADR 0360 no gate reads a walk; its one machine's report is its run. - return append(out, point{phase: PhaseApply, tier: -1, none: true, - said: "no machine beyond the first: its one machine's report was its run"}) - } - if applied == nil { - return append(out, point{phase: PhaseApply, tier: -1, said: "the rest's reports were not read"}) - } - // Each machine of the rest, by its own report of the send that carried the build there (novox/hq issue 582): - // never a report of a send made after it, which another walk's send may be. - reports := map[string]*RestReport{} - var order []string - for _, s := range lastSends { - at, said := s.walkSent, "" - if !s.what.At.IsZero() { - at = s.what.At - } else { - said = "the walk's record of its send to the rest: this machine's own send time was not kept" - } - key := s.node + "\x00" + s.what.Digest + "\x00" + at.Format(time.RFC3339Nano) - if r := reports[key]; r != nil { - r.Modules = appendOnce(r.Modules, s.module) - continue - } - rr := &RestReport{Node: s.node, Modules: []string{s.module}, Sent: at, SentSaid: said, Sequence: s.what.Sequence} - // Judged on the modules this walk sent there, not on every module the machine's declaration hands (ADR 0362). - var sentThere []string - for _, o := range lastSends { - if o.node == s.node && o.what.Digest == s.what.Digest && o.what.At.Equal(s.what.At) { - sentThere = append(sentThere, o.module) - } - } - if r, ok := applied(s.node, s.what.Only(sentThere...), s.walkSent); ok { - got := r.At - rr.Reported, rr.Outcome, rr.Took = &got, r.Outcome, words(r.At.Sub(at)) - if r.Busy > 0 { - rr.Busy = words(r.Busy) - rr.busy = r.Busy - } - if r.Waiting { - rr.Reported, rr.Took, rr.State = nil, "", RestBusy - } - } - reports[key] = rr - order = append(order, key) - } - // A machine sent twice by the walk (one tier's rest, then the next's) that said nothing of the first send has - // said it of the second, which carries the first's builds too: both are this walk's. - for _, k := range order { - rr := reports[k] - if rr.Reported != nil { - continue - } - for _, l := range order { - later := reports[l] - if later.Node == rr.Node && later.Sent.After(rr.Sent) && later.Reported != nil && - (rr.Reported == nil || later.Reported.Before(*rr.Reported)) { - got := *later.Reported - rr.Reported, rr.Outcome, rr.Took, rr.State = &got, later.Outcome, words(got.Sub(rr.Sent)), "" - } - } - } - var end *time.Time - var waiting, failed, busy []string - for _, k := range order { - rr := reports[k] - switch { - case rr.State == RestBusy: - // It answered and waits for a busy container: never silent, and the walk's end waits for it. - busy = appendOnce(busy, rr.Node) - case rr.Reported == nil && now.Sub(rr.Sent) > ApplySilentAfter, - rr.Reported != nil && rr.Reported.Sub(rr.Sent)-rr.busy > ApplySilentAfter: - rr.State = RestSilent - w.Silent = appendOnce(w.Silent, rr.Node) - case rr.Reported == nil: - rr.State = RestWaiting - waiting = appendOnce(waiting, rr.Node) - case rr.Outcome != OutcomeApplied: - rr.State = RestFailed - failed = appendOnce(failed, rr.Node+" ("+rr.Outcome+")") - default: - rr.State = RestApplied - end = latest(end, rr.Reported) - } - sort.Strings(rr.Modules) - w.Rest = append(w.Rest, *rr) - } - sort.SliceStable(w.Rest, func(a, b int) bool { - if w.Rest[a].Node != w.Rest[b].Node { - return w.Rest[a].Node < w.Rest[b].Node - } - return w.Rest[a].Sent.Before(w.Rest[b].Sent) - }) - switch { - case len(failed) > 0: - return append(out, point{phase: PhaseApply, tier: -1, said: "not applied on " + strings.Join(failed, ", ")}) - case len(waiting) > 0 || len(busy) > 0: - var said []string - if len(waiting) > 0 { - said = append(said, "waiting for "+strings.Join(waiting, ", ")+" to report it applied") - } - if len(busy) > 0 { - said = append(said, "waits for a busy container on "+strings.Join(busy, ", ")) - } - return append(out, point{phase: PhaseApply, tier: -1, said: strings.Join(said, "; ")}) - case end == nil: - return append(out, point{phase: PhaseApply, tier: -1, said: "no machine of the rest heard from: " + - strings.Join(w.Silent, ", ")}) - } - // A machine may have reported before the last tier ended: the walk ends at whichever is later. - for i := len(out) - 1; i >= 0; i-- { - if out[i].at != nil { - if out[i].at.After(*end) { - e := *out[i].at - end = &e - } - break - } - } - said := "" - if len(w.Silent) > 0 { - sort.Strings(w.Silent) - said = "not waited for, not heard from: " + strings.Join(w.Silent, ", ") - } - return append(out, point{phase: PhaseApply, tier: -1, at: end, said: said}) -} - -type restSend struct { - module, node string - walkSent time.Time - what SentDeclaration -} - -func askedAnyOf(p Plan, tier []string) bool { - for _, m := range tier { - if s := p.Modules[m]; s != nil && s.AskedAt != nil { - return true - } - } - return false -} - -func earliest(a, b *time.Time) *time.Time { - if b == nil { - return a - } - if a == nil || b.Before(*a) { - t := *b - return &t - } - return a -} - -func latest(a, b *time.Time) *time.Time { - if b == nil { - return a - } - if a == nil || b.After(*a) { - t := *b - return &t - } - return a -} - -func appendOnce(to []string, s string) []string { - for _, x := range to { - if x == s { - return to - } - } - return append(to, s) -} - -func orNot(s string) string { - if s == "" { - return "is not known" - } - return "(" + s + ")" -} - -// words is a duration as a person reads it. -func words(d time.Duration) string { - if d < 0 { - return "-" + words(-d) - } - return d.Round(100 * time.Millisecond).String() -} - -// FirstAppliedAfter is a machine's report of the send a walk made to it: the controller's `apply` durations, which -// measure every send to its first report (to-be 45 Phase 0), each row starting at the machine's own send time and -// naming the declaration it was about. send is what the walk kept of that send; walkSent its own record of the -// send to the rest, made after every machine of the rest was sent (novox/hq issue 582). FirstReportOf chooses. -func (i *Inventory) FirstAppliedAfter(ctx context.Context, node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool, error) { - from, to := applyRowsBetween(send, walkSent) - rows, err := i.store.Pool().Query(ctx, - `select kind, ref, started, took_ms, detail from duration - where subject = $2 and ((kind = $1 and started >= $3 and started < $4) or (kind = $5 and started >= $3)) - order by started`, - DurationApply, node, from, to, DurationBusy) - if err != nil { - return AppliedReport{}, false, err - } - defer rows.Close() - var ds []Duration - for rows.Next() { - d := Duration{Subject: node, Node: node} - var ms int64 - if err := rows.Scan(&d.Kind, &d.Ref, &d.Started, &ms, &d.Detail); err != nil { - return AppliedReport{}, false, err - } - d.Took = time.Duration(ms) * time.Millisecond - ds = append(ds, d) - } - if err := rows.Err(); err != nil { - return AppliedReport{}, false, err - } - r, ok := FirstReportOf(ds, send, walkSent) - return r, ok, nil -} - -// applyRowsBetween is the span of a machine's `apply` durations that may hold its report of a walk's send. -func applyRowsBetween(send SentDeclaration, walkSent time.Time) (time.Time, time.Time) { - switch { - case !send.At.IsZero(): - return send.At.Add(-time.Millisecond), send.At.Add(time.Millisecond) - case send.Digest != "": - return walkSent.Add(-restSendBound), walkSent.Add(ApplySilentAfter) - } - return walkSent.Add(-appliedSlack), walkSent.Add(ApplySilentAfter) -} - -// FirstReportOf picks, from a machine's `apply` durations ordered by their start, its report of the send a walk made -// to it (novox/hq issue 582): -// - with the machine's own send time kept (every walk since issue 582): the duration of that very send, and of the -// declaration sent where the walk kept its digest. A report of a later send is never this walk's: a send made -// later, by another walk, would end this walk at that walk's send. -// - with only the declaration's digest kept (a walk kept before): the first duration about that declaration -// within restSendBound before the walk's record of its send, the machine sent first in the rest included. -// - with neither (a walk kept before ADR 0282's rest): the first duration started no earlier than appliedSlack -// before the walk's record of its send, as it was read before. -// -// A send past ApplySilentAfter is never read: its machine slept, and is listed silent (ADR 0282 decision 1). -// -// **A first report that said a container waits because it is busy** (novox/hq ADR 0335) is the machine's answer, -// and not yet the build running: its report is the end of its busy wait, the `busy-wait` duration of the same -// send, with the wait said apart (Busy). While none is kept the machine is still waiting (Waiting), never silent. -func FirstReportOf(ds []Duration, send SentDeclaration, walkSent time.Time) (AppliedReport, bool) { - from, to := applyRowsBetween(send, walkSent) - for _, d := range ds { - if d.Kind == DurationBusy || d.Started.Before(from) || !d.Started.Before(to) { - continue - } - if send.Digest != "" && !strings.HasPrefix(d.Ref, send.Digest+"@") { - continue - } - answered := d.Started.Add(d.Took).UTC() - if d.Detail != ApplyBusy { - return AppliedReport{At: answered, Outcome: d.Detail}, true - } - for _, b := range ds { - if b.Kind == DurationBusy && b.Ref == d.Ref { - return AppliedReport{At: b.Started.Add(b.Took).UTC(), Outcome: OutcomeApplied, Answered: answered, - Busy: b.Took}, true - } - } - return AppliedReport{At: answered, Outcome: OutcomeApplied, Answered: answered, Waiting: true}, true - } - return AppliedReport{}, false -} - -// appliedSlack is how much earlier than a walk's record of its send the machine's own record of it may be, for a -// walk that kept neither the machine's send time nor what it sent. -const appliedSlack = 2 * time.Second - -// restSendBound is how long before a walk's record of its send to the rest a machine of the rest may have been -// sent, for a walk that kept what it sent and not when: the send to the rest goes machine by machine, and took -// 19 to 28 seconds on 2026-10-12 (novox/hq issue 582). -const restSendBound = 5 * time.Minute diff --git a/internal/inventory/walkphases_582_integration_test.go b/internal/inventory/walkphases_582_integration_test.go deleted file mode 100644 index 643d2510..00000000 --- a/internal/inventory/walkphases_582_integration_test.go +++ /dev/null @@ -1,49 +0,0 @@ -//go:build integration - -package inventory - -import ( - "testing" - "time" -) - -func TestAWalksRestIsReadFromEachMachinesOwnSend(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - record, err := inv.AddNode(ctx, "g14") - if err != nil { - t.Fatal(err) - } - if _, err := inv.NextSequence(ctx, record.ID); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, record.ID, "d-582", map[string]string{"build-agent": "c1"}); err != nil { - t.Fatal(err) - } - sent, found, err := inv.SentTo(ctx, "g14") - if err != nil || !found || sent.At.IsZero() || sent.Digest != "d-582" { - t.Fatalf("the machine's own send time: %+v %v %v", sent, found, err) - } - if _, ok, err := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)); err != nil || ok { - t.Fatalf("no report yet: %v %v", ok, err) - } - if err := inv.RecordApplyDuration(ctx, "g14", "d-582", OutcomeApplied); err != nil { - t.Fatal(err) - } - // The walk's record of its send, 20 s after the machine's own: still found, by the machine's own send. - r, ok, err := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)) - if err != nil || !ok || r.Outcome != OutcomeApplied || r.At.Before(sent.At) { - t.Fatalf("the machine's own report of its send: %+v %v %v", r, ok, err) - } - // A later send, reported: never this walk's. - if err := inv.RecordSent(ctx, record.ID, "d-next", map[string]string{"build-agent": "c2"}); err != nil { - t.Fatal(err) - } - if err := inv.RecordApplyDuration(ctx, "g14", "d-next", OutcomeApplied); err != nil { - t.Fatal(err) - } - again, _, _ := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)) - if !again.At.Equal(r.At) { - t.Fatalf("a later send's report read as this walk's: %+v, was %+v", again, r) - } -} diff --git a/internal/inventory/walkphases_582_test.go b/internal/inventory/walkphases_582_test.go deleted file mode 100644 index 0b363f87..00000000 --- a/internal/inventory/walkphases_582_test.go +++ /dev/null @@ -1,170 +0,0 @@ -package inventory - -import ( - "strings" - "testing" - "time" -) - -// novox/hq issue 582: the walk's own record of its send to the rest is made after every machine of the rest was -// sent, and that send took 19 to 28 s on 2026-10-12. The machine sent first reported 5 s after its own send, long -// before the walk's record; read from the walk's record, its report was missed and the walk ended at another walk's -// send to it, minutes later. A walk of one tier and three machines of the rest, the send to the rest taking 20 s. -type sendRest582 struct { - judged, walkSent time.Time - walk Plan - rows map[string][]Duration -} - -func walk582(t *testing.T) sendRest582 { - t.Helper() - judged := time.Date(2026, 10, 12, 17, 45, 35, 137000000, time.UTC) - ownSend := map[string]time.Duration{"g14": 500 * time.Millisecond, "novox": 10 * time.Second, "shanks": 20 * time.Second} - walkSent := judged.Add(20200 * time.Millisecond) - merged := judged.Add(-10 * time.Minute) - built, first := judged.Add(-3*time.Minute), judged.Add(-2*time.Minute) - rest := map[string]SentDeclaration{} - rows := map[string][]Duration{} - took := map[string]time.Duration{"g14": 5 * time.Second, "novox": 13 * time.Second, "shanks": 15 * time.Second} - for i, n := range []string{"g14", "novox", "shanks"} { - at := judged.Add(ownSend[n]) - digest := "walk-" + n - rest[n] = SentDeclaration{Digest: digest, Sequence: int64(647 + i), At: at} - rows[n] = append(rows[n], - // This walk's send, and its report. - Duration{Kind: DurationApply, Subject: n, Node: n, Ref: digest + "@" + at.Format("2006-01-02T15:04:05.000000"), - Started: at, Took: took[n], Detail: OutcomeApplied}, - // The next walk's send to the same machine, nine minutes later, and its report. - Duration{Kind: DurationApply, Subject: n, Node: n, Ref: "next-" + n + "@x", - Started: judged.Add(9*time.Minute + ownSend[n]), Took: 4 * time.Second, Detail: OutcomeApplied}) - } - p := Plan{ID: "plan-582", State: PlanDone, Tier: 1, Tiers: [][]string{{"build-agent"}}, Created: merged, - Delivery: &PlanDelivery{Merges: []PlanMerge{{Repository: "novox/mesh-controller", Commit: "c", Merged: merged}}}, - Times: &PlanTimes{Cut: &merged, Class: ClassCore}, - Modules: map[string]*PlanModule{"build-agent": {State: "built", AskedAt: &merged, BuiltAt: &built, - FirstAt: &first, SentAt: &walkSent, Gate: &PlanGate{JudgedAt: &judged}, Rest: rest}}} - return sendRest582{judged: judged, walkSent: walkSent, walk: p, rows: rows} -} - -// lookup answers from the rows as the controller's durations would. -func (w sendRest582) lookup(node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool) { - return FirstReportOf(w.rows[node], send, walkSent) -} - -// The apply phase ends at the last machine's own report of this walk's send — shanks, sent at the end of the 20 s -// and reporting 15 s later — and never at the next walk's send; every machine's report is shown. -func TestTheApplyPhaseEndsAtEachMachinesOwnReport(t *testing.T) { - c := walk582(t) - w := c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup) - sumsUp(t, w) - want := c.judged.Add(35 * time.Second) // shanks: sent 20 s after the judgement, reported 15 s later - if !w.End.Equal(want) { - t.Fatalf("the walk ends at %s, want %s: its rest %+v", w.End, want, w.Rest) - } - if got := phase(w, PhaseApply, -1); got.State != PhaseMeasured || got.TookMS != want.Sub(c.walkSent).Milliseconds() { - t.Fatalf("apply: %+v", got) - } - if len(w.Rest) != 3 || len(w.Silent) != 0 { - t.Fatalf("every machine of the rest is shown with its report: %+v, silent %v", w.Rest, w.Silent) - } - g14 := w.Rest[0] - if g14.Node != "g14" || g14.State != RestApplied || g14.Reported == nil || - !g14.Reported.Equal(c.judged.Add(5500*time.Millisecond)) || g14.Took != "5s" || g14.Sequence != 647 || - !g14.Sent.Equal(c.judged.Add(500*time.Millisecond)) || g14.SentSaid != "" { - t.Fatalf("g14, sent first and reporting 5 s after its own send: %+v", g14) - } -} - -// A walk kept before its machines' own send times were: matched by the declaration it sent, within restSendBound -// of its record of the send, so the machine sent first is still found; a walk that kept neither is read as before. -func TestAnOlderWalkIsReadByWhatItSent(t *testing.T) { - c := walk582(t) - for n, s := range c.walk.Modules["build-agent"].Rest { - s.At = time.Time{} - c.walk.Modules["build-agent"].Rest[n] = s - } - w := c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup) - sumsUp(t, w) - if !w.End.Equal(c.judged.Add(35 * time.Second)) { - t.Fatalf("by the declaration sent: end %s, rest %+v", w.End, w.Rest) - } - if !strings.Contains(w.Rest[0].SentSaid, "walk's record") || !w.Rest[0].Sent.Equal(c.walkSent) { - t.Fatalf("a send time not kept is said: %+v", w.Rest[0]) - } - // Nothing of the send kept: the first report no earlier than appliedSlack before the walk's record, as before. - if r, ok := FirstReportOf(c.rows["shanks"], SentDeclaration{}, c.walkSent); !ok || !r.At.Equal(c.judged.Add(35*time.Second)) { - t.Fatalf("a walk that kept nothing of its send: %+v %v", r, ok) - } -} - -// A machine that said nothing of this walk's send is waiting, then silent past ApplySilentAfter: the report of a -// later send to it is never taken as this walk's, and never moves the walk's end. -func TestALaterSendsReportIsNeverThisWalks(t *testing.T) { - c := walk582(t) - c.rows["g14"] = c.rows["g14"][1:] // g14 slept through this walk's send; the next walk's send it reported - w := c.walk.WalkPhases(c.judged.Add(5*time.Minute), c.lookup) - if w.End != nil || !strings.Contains(phase(w, PhaseApply, -1).Said, "waiting for g14") { - t.Fatalf("g14 not heard from yet: end %v, apply %+v", w.End, phase(w, PhaseApply, -1)) - } - w = c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup) - sumsUp(t, w) - if len(w.Silent) != 1 || w.Silent[0] != "g14" || !w.End.Equal(c.judged.Add(35*time.Second)) { - t.Fatalf("g14 silent, the end at shanks' report: silent %v, end %s", w.Silent, w.End) - } - if w.Rest[0].State != RestSilent || w.Rest[0].Reported != nil { - t.Fatalf("g14's later report taken as this walk's: %+v", w.Rest[0]) - } -} - -// A machine sent by two tiers of one walk that said nothing of the first send has said it of the second, which -// carries the first tier's build too. -func TestAMachineSentTwiceByOneWalkIsHeardByItsLaterSend(t *testing.T) { - c := walk582(t) - second := c.judged.Add(4 * time.Minute) - secondSent := second.Add(time.Second) - c.walk.Tiers = append(c.walk.Tiers, []string{"route-proxy"}) - c.walk.Tier = 2 - asked, built := c.judged.Add(time.Minute), c.judged.Add(2*time.Minute) - first := built.Add(time.Second) - c.walk.Modules["route-proxy"] = &PlanModule{State: "built", AskedAt: &asked, BuiltAt: &built, FirstAt: &first, - SentAt: &secondSent, Gate: &PlanGate{JudgedAt: &second}, - Rest: map[string]SentDeclaration{"novox": {Digest: "walk-novox-2", Sequence: 1554, At: second.Add(500 * time.Millisecond)}}} - c.rows["novox"] = []Duration{{Kind: DurationApply, Subject: "novox", Ref: "walk-novox-2@x", - Started: second.Add(500 * time.Millisecond), Took: 14 * time.Second, Detail: OutcomeApplied}} - w := c.walk.WalkPhases(c.judged.Add(time.Hour), c.lookup) - sumsUp(t, w) - want := second.Add(14500 * time.Millisecond) - if !w.End.Equal(want) || len(w.Silent) != 0 { - t.Fatalf("end %s (want %s), silent %v, rest %+v", w.End, want, w.Silent, w.Rest) - } - for _, r := range w.Rest { - if r.Node == "novox" && (r.State != RestApplied || !r.Reported.Equal(want)) { - t.Fatalf("novox, heard by its later send of this walk: %+v", r) - } - } -} - -// The store: a machine's own send time is read with what it was sent, and its `apply` duration is found by it — -// the send and the duration start at one moment, the node's sent_at. - -// Issue 840 (ADR 0362): the walk's report of a machine is asked for with only the provisioned modules the walk sent -// there, never every module the machine's declaration hands to the provisioner. -func TestTheRestIsAskedAboutTheModulesTheWalkSent(t *testing.T) { - c := walk582(t) - rest := c.walk.Modules["build-agent"].Rest - for n, s := range rest { - s.Provisioned = map[string]string{"build-agent": "t1", "other": "t2"} - rest[n] = s - } - asked := 0 - c.walk.WalkPhases(c.judged.Add(time.Hour), func(node string, send SentDeclaration, walkSent time.Time) (AppliedReport, bool) { - asked++ - if len(send.Provisioned) != 1 || send.Provisioned["build-agent"] != "t1" { - t.Fatalf("%s is asked about %v", node, send.Provisioned) - } - return c.lookup(node, send, walkSent) - }) - if asked == 0 { - t.Fatal("no machine was asked") - } -} diff --git a/internal/inventory/walkphases_integration_test.go b/internal/inventory/walkphases_integration_test.go deleted file mode 100644 index f7cbb108..00000000 --- a/internal/inventory/walkphases_integration_test.go +++ /dev/null @@ -1,77 +0,0 @@ -//go:build integration - -package inventory - -import ( - "testing" - "time" -) - -func TestAWalksMomentsAreKeptAndItsRestsReportRead(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - p := walkOf(t, recordedWalk) - p.Revision, p.Epoch = 0, 0 - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - back, err := inv.PlanByID(ctx, p.ID) - if err != nil || back.Times == nil || back.Times.Class != ClassCore || back.Times.WindowClosed == nil || - !back.Times.WindowClosed.Equal(*p.Times.WindowClosed) { - t.Fatalf("the walk's moments were not kept: %v %+v", err, back.Times) - } - if back.Delivery.Merges[0].Merged.IsZero() { - t.Fatal("a merge's time was not kept") - } - sent := time.Now().UTC().Add(-time.Minute).Truncate(time.Millisecond) - if _, ok, err := inv.FirstAppliedAfter(ctx, "ace", SentDeclaration{}, sent); err != nil || ok { - t.Fatalf("no report yet: %v %v", ok, err) - } - for _, d := range []Duration{ - {Kind: DurationApply, Subject: "ace", Node: "ace", Ref: "old@1", Started: sent.Add(-time.Hour), Took: time.Second, Detail: OutcomeApplied}, - {Kind: DurationApply, Subject: "ace", Node: "ace", Ref: "d1@2", Started: sent.Add(-time.Second), Took: 12 * time.Second, Detail: OutcomeApplied}, - } { - if err := inv.RecordDuration(ctx, d); err != nil { - t.Fatal(err) - } - } - r, ok, err := inv.FirstAppliedAfter(ctx, "ace", SentDeclaration{}, sent) - if err != nil || !ok || r.Outcome != OutcomeApplied || !r.At.Equal(sent.Add(11*time.Second)) { - t.Fatalf("the report after the send: %+v %v %v", r, ok, err) - } - if anyKept, total, err := inv.WalkPhasesKept(ctx, p.ID); err != nil || anyKept || total { - t.Fatalf("nothing kept yet: %v %v %v", anyKept, total, err) - } - if err := inv.RecordDuration(ctx, Duration{Kind: DurationWalkPhase, Subject: "core/total", Ref: p.ID + "/total", - Started: sent, Took: time.Minute}); err != nil { - t.Fatal(err) - } - if anyKept, total, err := inv.WalkPhasesKept(ctx, p.ID); err != nil || !anyKept || !total { - t.Fatalf("the total kept: %v %v %v", anyKept, total, err) - } -} - -func TestALateReportIsSilentNotTheEnd(t *testing.T) { - sent := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC) - p := walkOf(t, recordedWalk) - p.Modules["mesh-delivery"].SentAt = &sent - p.Modules["mesh-delivery"].Rest = map[string]SentDeclaration{"laptop": {Digest: "d"}, "server": {Digest: "e"}} - w := p.WalkPhases(sent.Add(3*time.Hour), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) { - if node == "server" { - return AppliedReport{At: sent.Add(20 * time.Second), Outcome: OutcomeApplied}, true - } - return AppliedReport{At: sent.Add(2 * time.Hour), Outcome: OutcomeApplied}, true - }) - if len(w.Silent) != 1 || w.Silent[0] != "laptop" || w.End == nil || !w.End.Equal(sent.Add(20*time.Second)) { - t.Fatalf("a late report: silent %v, end %v", w.Silent, w.End) - } - inv := ForTest(t) - ctx := t.Context() - if err := inv.RecordDuration(ctx, Duration{Kind: DurationApply, Subject: "laptop", Node: "laptop", Ref: "late@1", - Started: sent.Add(time.Hour), Took: time.Second, Detail: OutcomeApplied}); err != nil { - t.Fatal(err) - } - if _, ok, err := inv.FirstAppliedAfter(ctx, "laptop", SentDeclaration{}, sent); err != nil || ok { - t.Fatalf("a send past the bound read as the walk's: %v %v", ok, err) - } -} diff --git a/internal/inventory/walkphases_test.go b/internal/inventory/walkphases_test.go deleted file mode 100644 index 216d6d5a..00000000 --- a/internal/inventory/walkphases_test.go +++ /dev/null @@ -1,263 +0,0 @@ -package inventory - -import ( - "encoding/json" - "strings" - "testing" - "time" -) - -// A walk recorded on the live mesh on 2026-10-10 (mesh-delivery, one tier, one machine), as `delivery walks` -// gave it, with the moments ADR 0282 adds: its window closed ninety seconds after its merge was heard, and it was -// cut eleven seconds later. -const recordedWalk = `{ - "id": "plan-1791654663505629616", "repository": "novox/mesh-catalog", "branch": "main", - "commit": "a14fa306f262d88bdcca83432a70df353d2eceb0", "merged": "2026-10-10T17:50:53Z", - "created": "2026-10-10T19:52:34.037755+02:00", "updated": "2026-10-10T19:55:37.974069+02:00", - "state": "done", "tier": 1, "tiers": [["mesh-delivery"]], - "modules": {"mesh-delivery": {"state": "built", - "asked_at": "2026-10-10T17:52:34.209423145Z", "built_at": "2026-10-10T17:52:50.818011314Z", - "sent_at": "2026-10-10T17:55:35.894985053Z", "first": ["novox"], "first_at": "2026-10-10T17:53:07.287136827Z", - "gate": {"machines": ["novox"], "since": "2026-10-10T17:53:07.287136827Z", "passes": 3, - "verdict": "passed", "judged_at": "2026-10-10T17:55:35.894985053Z"}}}, - "delivery": {"awaits": "", "merges": [{"repository": "novox/mesh-catalog", - "commit": "a14fa306f262d88bdcca83432a70df353d2eceb0", "number": 187, "merged": "2026-10-10T17:50:53Z"}]}, - "times": {"window_closed": "2026-10-10T17:52:23Z", "cut": "2026-10-10T17:52:34.037755Z", "class": "core"} -}` - -func walkOf(t *testing.T, raw string) Plan { - t.Helper() - var p Plan - if err := json.Unmarshal([]byte(raw), &p); err != nil { - t.Fatal(err) - } - return p -} - -// sumsUp checks the phases' measured time and the unknown time add up to the total, to the millisecond. -func sumsUp(t *testing.T, w *WalkPhases) { - t.Helper() - if w.End == nil || w.From == nil { - t.Fatalf("the walk has no end: %+v", w) - } - var sum int64 - for _, ph := range w.Phases { - if ph.State == PhaseMeasured { - if ph.Start == nil || ph.End == nil || ph.End.Sub(*ph.Start).Milliseconds() != ph.TookMS { - t.Errorf("phase %s %d says %dms between %v and %v", ph.Name, ph.Tier, ph.TookMS, ph.Start, ph.End) - } - sum += ph.TookMS - } - if ph.State == PhaseUnknown && ph.TookMS != 0 { - t.Errorf("an unknown phase %s says a time: %dms", ph.Name, ph.TookMS) - } - } - if sum+w.UnknownMS != w.TotalMS || w.End.Sub(*w.From).Milliseconds() != w.TotalMS { - t.Fatalf("the phases add up to %dms and %dms unknown, the total is %dms (%s): %+v", sum, w.UnknownMS, - w.TotalMS, w.End.Sub(*w.From), w.Phases) - } -} - -func phase(w *WalkPhases, name string, tier int) WalkPhase { - for _, ph := range w.Phases { - if ph.Name == name && ph.Tier == tier { - return ph - } - } - return WalkPhase{} -} - -// The phases of a recorded walk add up to its measured total: its merge at 17:50:53 to its verdict on its only -// machine at 17:55:35.894, 4m42.894s. -func TestARecordedWalksPhasesAddUpToItsTotal(t *testing.T) { - w := walkOf(t, recordedWalk).WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil) - sumsUp(t, w) - if w.TotalMS != 282894 || w.Class != ClassCore || w.UnknownMS != 0 { - t.Fatalf("total %dms (want 282894), class %q, unknown %d", w.TotalMS, w.Class, w.UnknownMS) - } - for name, want := range map[string]int64{PhaseWindow: 90000, PhaseQueued: 11037, PhaseBuild: 16781, - PhaseSend: 16469, PhaseJudge: 148607, PhaseRest: 0} { - tier := 0 - if name == PhaseWindow || name == PhaseQueued { - tier = -1 - } - if got := phase(w, name, tier); got.State != PhaseMeasured || got.TookMS != want { - t.Errorf("%s: %s %dms, want measured %dms", name, got.State, got.TookMS, want) - } - } - // The cut to the first ask (171ms) is a time no phase of ADR 0282's table names: it is counted in the build. - if got := phase(w, PhaseWord, -1); got.State != PhaseNone { - t.Errorf("a walk that waits for no word says its word phase %s", got.State) - } - if got := phase(w, PhaseApply, -1); got.State != PhaseNone || !strings.Contains(got.Said, "no machine beyond the first: its one machine's report was its run") { - t.Errorf("one machine only: apply %s (%s)", got.State, got.Said) - } -} - -// Two tiers, a machine of the rest each, both reports read: every phase measured, the walk ends at the last -// report applied, and the phases add up. -func TestAWalkOfTwoTiersEndsAtItsRestsLastReport(t *testing.T) { - at := func(s string) *time.Time { - v, err := time.Parse(time.RFC3339Nano, "2026-10-10T18:"+s+"Z") - if err != nil { - t.Fatal(err) - } - return &v - } - p := Plan{ID: "plan-2", State: PlanDone, Tier: 2, Tiers: [][]string{{"a"}, {"b"}}, Created: *at("01:40"), - Delivery: &PlanDelivery{Merges: []PlanMerge{{Repository: "novox/x", Commit: "c1", Merged: *at("00:00")}, - {Repository: "novox/x", Commit: "c0", Merged: *at("00:30")}}}, - Times: &PlanTimes{WindowClosed: at("01:30"), Cut: at("01:40"), Class: ClassLeaf}, - Modules: map[string]*PlanModule{ - "a": {State: "built", AskedAt: at("01:41"), BuiltAt: at("02:05"), FirstAt: at("02:20"), SentAt: at("05:00"), - Gate: &PlanGate{JudgedAt: at("04:50")}, Rest: map[string]SentDeclaration{"ace": {Digest: "d1"}}}, - "b": {State: "built", AskedAt: at("05:10"), BuiltAt: at("05:40"), FirstAt: at("05:50"), SentAt: at("08:00.5"), - Gate: &PlanGate{JudgedAt: at("07:59")}, Rest: map[string]SentDeclaration{"shanks": {Digest: "d2"}}}, - }} - reports := map[string]AppliedReport{"ace": {At: *at("05:12"), Outcome: OutcomeApplied}, - "shanks": {At: *at("08:15.25"), Outcome: OutcomeApplied}} - w := p.WalkPhases(*at("30:00"), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) { - r, ok := reports[node] - return r, ok - }) - sumsUp(t, w) - if w.TotalMS != (8*time.Minute + 15250*time.Millisecond).Milliseconds() { - t.Fatalf("the walk's delivery time runs from its earliest merge to the last report: %s", w.Total) - } - if got := phase(w, PhaseBetween, 1); got.TookMS != 10000 { - t.Errorf("between the tiers: %dms", got.TookMS) - } - if got := phase(w, PhaseApply, -1); got.State != PhaseMeasured || got.TookMS != 14750 { - t.Errorf("apply: %s %dms", got.State, got.TookMS) - } - if len(w.Merges) != 2 { - t.Errorf("each merge's start is said, for its own delivery time: %+v", w.Merges) - } - // A report not yet in: the walk has no end, and its apply is unknown — never zero. - delete(reports, "shanks") - w = p.WalkPhases(*at("10:00"), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) { - r, ok := reports[node] - return r, ok - }) - if w.End != nil || w.TotalMS != 0 { - t.Fatalf("a walk whose rest has not reported has no end: %+v", w) - } - if got := phase(w, PhaseApply, -1); got.State != PhaseUnknown || got.TookMS != 0 || !strings.Contains(got.Said, "shanks") { - t.Errorf("apply while shanks has not reported: %+v", got) - } - // Silent past the bound: listed, not waited for. - w = p.WalkPhases(at("08:00.5").Add(ApplySilentAfter+time.Second), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) { - r, ok := reports[node] - return r, ok - }) - sumsUp(t, w) - if len(w.Silent) != 1 || w.Silent[0] != "shanks" { - t.Errorf("a machine silent past the bound is said, not waited for: %+v", w.Silent) - } - // A failed report: the walk has no end, said. - reports["shanks"] = AppliedReport{At: *at("08:10"), Outcome: OutcomeFailed} - w = p.WalkPhases(*at("30:00"), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) { - r, ok := reports[node] - return r, ok - }) - if w.End != nil || !strings.Contains(phase(w, PhaseApply, -1).Said, "shanks (failed)") { - t.Errorf("a failed apply ends nothing: %+v", phase(w, PhaseApply, -1)) - } -} - -// A moment that is missing makes the phases around it unknown, and their span unknown time: never a zero. -func TestAMissingMomentIsUnknownNeverZero(t *testing.T) { - p := walkOf(t, recordedWalk) - p.Modules["mesh-delivery"].BuiltAt = nil - w := p.WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil) - sumsUp(t, w) - if got := phase(w, PhaseBuild, 0); got.State != PhaseUnknown || got.TookMS != 0 { - t.Errorf("a build without its moment: %+v", got) - } - if got := phase(w, PhaseSend, 0); got.State != PhaseUnknown { - t.Errorf("the send after a build without its moment: %+v", got) - } - if w.UnknownMS != 16781+16469 { - t.Errorf("the unknown time is the span between the moments around it: %dms", w.UnknownMS) - } - // A walk kept before ADR 0282: its window and its wait together, and its apply unknown. - p = walkOf(t, recordedWalk) - p.Times = nil - w = p.WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil) - if got := phase(w, PhaseWindow, -1); got.State != PhaseUnknown { - t.Errorf("a window not kept: %+v", got) - } - if got := phase(w, PhaseApply, -1); got.State != PhaseUnknown || w.End != nil { - t.Errorf("a rest not kept: %+v, end %v", got, w.End) - } -} - -// An open walk is said open since its last moment, with no end. -func TestAnOpenWalkHasNoEnd(t *testing.T) { - p := walkOf(t, recordedWalk) - p.State, p.Tier = PlanRolling, 0 - p.Modules["mesh-delivery"].SentAt = nil - p.Modules["mesh-delivery"].Gate.JudgedAt = nil - w := p.WalkPhases(time.Date(2026, 10, 10, 17, 54, 0, 0, time.UTC), nil) - if w.End != nil || !strings.HasPrefix(w.Said, "its end is unknown") && !strings.HasPrefix(w.Said, "open") { - t.Fatalf("an open walk: %+v", w) - } - if got := phase(w, PhaseBuild, 0); got.State != PhaseMeasured { - t.Errorf("an open walk's phases so far are measured: %+v", got) - } -} - -// A gate keeps every pass and the first reading after one that did not pass, at most maxReadings. -func TestAGateKeepsItsReadings(t *testing.T) { - var g PlanGate - t0 := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC) - g.Read(t0, false, "starting") - g.Read(t0.Add(time.Second), false, "starting") - g.Read(t0.Add(40*time.Second), true, "") - g.Read(t0.Add(80*time.Second), true, "") - g.Read(t0.Add(81*time.Second), false, "gone") - g.Read(t0.Add(82*time.Second), false, "gone") - if len(g.Readings) != 4 || g.Readings[0].Said != "starting" || !g.Readings[2].Healthy || g.Readings[3].Said != "gone" { - t.Fatalf("readings: %+v", g.Readings) - } - for i := 0; i < 50; i++ { - g.Read(t0.Add(time.Duration(100+i)*time.Second), true, "") - } - if len(g.Readings) != maxReadings { - t.Fatalf("readings are bounded: %d", len(g.Readings)) - } -} - -// A walk's moments are kept and read back with it; a machine's first report after a send is read from the -// controller's apply durations. - -// A machine that slept past the bound and reported later is listed silent: its late report never moves the -// walk's end; and a report sent past the bound is not read as the walk's. - -// Moments out of order — #206's own walk on 2026-10-10 kept its first send 783 ms before its build — are said -// unknown, never a negative phase: the phase before is unknown and its time joins the unknown time, the phase -// out of order has no time, and the phases still add up to the total. -func TestMomentsOutOfOrderAreUnknownNeverNegative(t *testing.T) { - p := walkOf(t, recordedWalk) - built := time.Date(2026, 10, 10, 17, 53, 8, 70000000, time.UTC) // 783 ms after its first send - p.Modules["mesh-delivery"].BuiltAt = &built - w := p.WalkPhases(time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC), nil) - sumsUp(t, w) - for _, ph := range w.Phases { - if ph.TookMS < 0 { - t.Fatalf("a negative phase: %+v", ph) - } - } - build, send := phase(w, PhaseBuild, 0), phase(w, PhaseSend, 0) - if build.State != PhaseUnknown || send.State != PhaseUnknown || send.TookMS != 0 || - !strings.Contains(send.Said, "out of order") || !strings.Contains(build.Said, "out of order") { - t.Fatalf("build %+v, send-first %+v", build, send) - } - // The build's span (cut to its later moment) is unknown time; the judgement is measured from the later moment. - if w.UnknownMS != built.Sub(time.Date(2026, 10, 10, 17, 52, 34, 37000000, time.UTC)).Milliseconds() { - t.Fatalf("unknown %dms", w.UnknownMS) - } - if j := phase(w, PhaseJudge, 0); j.State != PhaseMeasured || j.Start == nil || !j.Start.Equal(built) { - t.Fatalf("the judgement is measured from the later moment: %+v", j) - } -} diff --git a/internal/inventory/walksaid.go b/internal/inventory/walksaid.go deleted file mode 100644 index 95c5749a..00000000 --- a/internal/inventory/walksaid.go +++ /dev/null @@ -1,81 +0,0 @@ -package inventory - -import "time" - -// A walk says when it starts and when it ends (novox/hq ADR 0308 decision 1): the save that starts it and the -// save that ends it are marked, so `plan-moved` names what the walk touches, its bound and its outcome on those -// two saves, and on no other. The marks are kept in the walk's times, so a controller replaced mid-walk (a core -// walk replaces it) neither says a start again nor misses the end. - -// The moments of a walk `plan-moved` names. -const ( - WalkStarted = "started" - WalkEnded = "ended" -) - -// The outcomes of an ended walk (ADR 0308 decision 1). -const ( - WalkDelivered = "delivered" - WalkFailed = "failed" - WalkStopped = "stopped" -) - -// Live says the walk is walking: cut, not waiting for its delivery's word, and not ended. -func (p Plan) Live() bool { return p.Open() && !p.Batch() && !p.Waiting() } - -// Ended says the walk is in a state it does not leave by itself: done, failed or superseded. -func (p Plan) Ended() bool { - return p.State == PlanDone || p.State == PlanFailed || p.State == PlanSuperseded -} - -// Outcome is how an ended walk ended: delivered when done; stopped when a person or the delivery's owner stopped -// it, or a newer walk took it over; failed otherwise. Empty for a walk not ended. -func (p Plan) Outcome() string { - switch { - case p.State == PlanDone: - return WalkDelivered - case p.State == PlanSuperseded: - return WalkStopped - case p.State == PlanFailed && p.Delivery != nil && p.Delivery.Stopped != "": - return WalkStopped - case p.State == PlanFailed: - return WalkFailed - } - return "" -} - -// markWalk marks the save that starts or ends the walk, in its times and in WalkSaid, and returns what undoes -// the mark should the save not be written. A walk starts on the first save that finds it live — when it is cut -// and needs no word, or when it is let go — which is before its first tier is asked, and so before anything is -// sent: what it restarts is held from its first send on. It starts again when a failed walk is retried. It ends -// on the first save that finds it ended after it started; one that ends without ever starting (a batch that -// moved nothing, a walk stopped while it waited for its word) touched nothing and says neither. A walk kept -// before its times were (ADR 0282 decision 6) is not marked; a release plan is, its times holding only these. -func (p *Plan) markWalk(now time.Time) (undo func()) { - p.WalkSaid = "" - if p.Batch() || (p.Times == nil && p.Release == nil) { - return func() {} - } - var was *PlanTimes - if p.Times != nil { - kept := *p.Times - was = &kept - } - times := PlanTimes{} - if p.Times != nil { - times = *p.Times - } - at := now.UTC() - switch { - case p.Live() && (times.Started == nil || times.Ended != nil): - times.Started, times.Ended = &at, nil - p.WalkSaid = WalkStarted - case p.Ended() && times.Started != nil && times.Ended == nil: - times.Ended = &at - p.WalkSaid = WalkEnded - default: - return func() {} - } - p.Times = × - return func() { p.Times, p.WalkSaid = was, "" } -} diff --git a/internal/inventory/walksaid_integration_test.go b/internal/inventory/walksaid_integration_test.go deleted file mode 100644 index 6bd21f1f..00000000 --- a/internal/inventory/walksaid_integration_test.go +++ /dev/null @@ -1,50 +0,0 @@ -//go:build integration - -package inventory - -import ( - "testing" - "time" -) - -func TestAWalksStartIsKeptAndARefusedSaveLeavesItUnsaid(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - cut := time.Now().UTC().Add(-time.Minute) - p := Plan{ID: "plan-308", Repository: "novox/mesh-catalog", Commit: "abc", Created: cut, State: PlanBuilding, - Tiers: [][]string{{"messenger"}}, Modules: map[string]*PlanModule{"messenger": {}}, - Times: &PlanTimes{Cut: &cut, Class: ClassLeaf}} - var said []string - was := PlanSaved - PlanSaved = func(_ *Inventory, p Plan) { said = append(said, p.WalkSaid) } - t.Cleanup(func() { PlanSaved = was }) - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - back, err := inv.PlanByID(ctx, p.ID) - if err != nil || back.Times == nil || back.Times.Started == nil || back.Times.Class != ClassLeaf { - t.Fatalf("the start is kept with the walk: %v %+v", err, back.Times) - } - // Read back by another controller and saved: not said again. - back.State = PlanRolling - if err := inv.SavePlan(ctx, &back); err != nil { - t.Fatal(err) - } - // A stale copy ending it is refused, and unmarked. - stale := p - stale.Revision = 1 - stale.State = PlanDone - if err := inv.SavePlan(ctx, &stale); err == nil { - t.Fatal("a stale save is refused") - } - if stale.WalkSaid != "" || stale.Times.Ended != nil { - t.Fatalf("a refused save is unmarked: %q %+v", stale.WalkSaid, stale.Times) - } - back.State = PlanDone - if err := inv.SavePlan(ctx, &back); err != nil { - t.Fatal(err) - } - if len(said) != 3 || said[0] != WalkStarted || said[1] != "" || said[2] != WalkEnded { - t.Fatalf("said: %q, want started, nothing, ended", said) - } -} diff --git a/internal/inventory/walksaid_test.go b/internal/inventory/walksaid_test.go deleted file mode 100644 index 6c0a5275..00000000 --- a/internal/inventory/walksaid_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package inventory - -import ( - "testing" - "time" -) - -// A walk is said started on the save that finds it live, ended on the save that finds it ended after it started, -// and neither on any other save (novox/hq ADR 0308 decision 1). -func TestAWalkIsSaidStartedOnceAndEndedOnce(t *testing.T) { - now := time.Date(2026, 10, 12, 11, 0, 0, 0, time.UTC) - cut := now.Add(-time.Second) - p := Plan{ID: "plan-1", State: PlanBuilding, Tiers: [][]string{{"messenger"}}, - Modules: map[string]*PlanModule{"messenger": {}}, Times: &PlanTimes{Cut: &cut, Class: ClassLeaf}} - p.markWalk(now) - if p.WalkSaid != WalkStarted || p.Times.Started == nil || !p.Times.Started.Equal(now) { - t.Fatalf("a live walk's first save starts it: %q %+v", p.WalkSaid, p.Times) - } - p.State = PlanRolling - p.markWalk(now.Add(time.Minute)) - if p.WalkSaid != "" || !p.Times.Started.Equal(now) { - t.Fatalf("a later save says nothing and keeps the start: %q %+v", p.WalkSaid, p.Times) - } - p.State = PlanDone - p.markWalk(now.Add(2 * time.Minute)) - if p.WalkSaid != WalkEnded || p.Times.Ended == nil || p.Outcome() != WalkDelivered { - t.Fatalf("the save that finds it done ends it, delivered: %q %+v %q", p.WalkSaid, p.Times, p.Outcome()) - } - p.markWalk(now.Add(3 * time.Minute)) - if p.WalkSaid != "" { - t.Fatalf("an ended walk saved again says nothing: %q", p.WalkSaid) - } - // Retried, it walks again: a new start. - p.State = PlanBuilding - p.markWalk(now.Add(4 * time.Minute)) - if p.WalkSaid != WalkStarted || p.Times.Ended != nil || !p.Times.Started.Equal(now.Add(4*time.Minute)) { - t.Fatalf("a retried walk starts again: %q %+v", p.WalkSaid, p.Times) - } -} - -// What is never said: a batch, a walk waiting for its word, a walk that ends without having started, and a walk -// kept before its times were. A release plan is said. -func TestWhatIsNotAWalkStartIsNotSaid(t *testing.T) { - now := time.Now() - for name, p := range map[string]Plan{ - "a batch": {State: PlanAssembling, Times: &PlanTimes{}}, - "waiting for a word": {State: PlanBuilding, Times: &PlanTimes{}, Delivery: &PlanDelivery{Awaits: "mesh-delivery"}}, - "ended, never begun": {State: PlanDone, Times: &PlanTimes{}}, - "kept before times": {State: PlanBuilding}, - "stopped while waits": {State: PlanFailed, Times: &PlanTimes{}, Delivery: &PlanDelivery{Awaits: "mesh-delivery", Stopped: "x"}}, - } { - p.markWalk(now) - if p.WalkSaid != "" { - t.Errorf("%s: said %q", name, p.WalkSaid) - } - } - r := Plan{State: PlanRolling, Release: &PlanRelease{Order: []string{"ace"}}} - r.markWalk(now) - if r.WalkSaid != WalkStarted || r.Times == nil || r.Times.Started == nil || r.Times.Class != "" { - t.Fatalf("a release plan is said started, its times holding only that: %q %+v", r.WalkSaid, r.Times) - } - // Let go, the walk starts. - go1 := now - w := Plan{State: PlanBuilding, Times: &PlanTimes{}, Delivery: &PlanDelivery{Awaits: "mesh-delivery", Go: &go1}} - w.markWalk(now) - if w.WalkSaid != WalkStarted { - t.Fatalf("a walk let go starts: %q", w.WalkSaid) - } -} - -// An ended walk's outcome: delivered, failed, or stopped by a person or the delivery's owner, or by a newer walk. -func TestAWalksOutcome(t *testing.T) { - for want, p := range map[string]Plan{ - WalkDelivered: {State: PlanDone}, - WalkFailed: {State: PlanFailed}, - WalkStopped: {State: PlanFailed, Delivery: &PlanDelivery{Stopped: "mesh-delivery"}}, - "": {State: PlanRolling}, - } { - if got := p.Outcome(); got != want { - t.Errorf("%s: outcome %q, want %q", p.State, got, want) - } - } - if (Plan{State: PlanSuperseded}).Outcome() != WalkStopped { - t.Error("a superseded walk is stopped") - } -} - -// The marks are kept with the walk, so a controller replaced mid-walk neither says its start again nor misses -// its end; and a save refused leaves the walk unmarked, so the save that is made says it. diff --git a/internal/link/enrolment.go b/internal/link/enrolment.go index 845d9c01..9ee84314 100644 --- a/internal/link/enrolment.go +++ b/internal/link/enrolment.go @@ -449,17 +449,6 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err if err := e.Inventory.RecordReadsEpoch(ctx, node.ID, report.ReadsEpoch()); err != nil { return false, err } - // And whether it reads a generation (novox/hq issue 234), the same way: only from a node-engine that - // orders its reports, so a one-shot report (a rekey) does not say an engine stopped reading one. - if report.Ordered() { - if err := e.Inventory.RecordReadsGeneration(ctx, node.ID, report.ReadsGeneration); err != nil { - return false, err - } - // And whether it reads a module per resource and a build per module (novox/hq issue 1145), the same way. - if err := e.Inventory.RecordReadsModules(ctx, node.ID, report.ReadsModules); err != nil { - return false, err - } - } if report.Ordered() { account := AccountOf(report) news, err = e.Inventory.RecordOrderedDoing(ctx, node.ID, doing, @@ -496,27 +485,6 @@ func (e Enrolment) Heard(ctx context.Context, report Report) (news bool, err err return false, err } } - // And how long the machine took, from the send to this first account of it (novox/hq to-be 45 - // Phase 0): what the sent-not-reported bound will be set from. Said if lost, never a failure. - // - // **A wait for a busy container is its own measure** (novox/hq ADR 0335): the first report that says one - // waits is the machine's answer, kept as its apply with the detail ApplyBusy; the report that says the - // container replaced ends the wait, kept as a `busy-wait` of the same send — so the apply's time holds no - // busy time, and the walk reads the machine as answered and waiting, never as silent. - outcome := doing.Outcome - if doing.Waiting() { - outcome = inventory.ApplyBusy - log.Printf("%s applied declaration %s and leaves a busy container running until it says idle: %s", - report.Node, short(report.Declared), busyWords(doing.Busy)) - } else if replacedAfterWaiting(doing.Busy) { - if err := e.Inventory.RecordBusyDuration(ctx, report.Node, report.Declared); err != nil { - log.Printf("how long %s waited for a busy container could not be recorded: %v", report.Node, err) - } - } - if err := e.Inventory.RecordApplyDuration(ctx, report.Node, report.Declared, outcome); err != nil { - log.Printf("how long %s took to apply could not be recorded: %v", report.Node, err) - } - // A refusal, a failure, or a bare word that the node is there — none of them is an account of // what the machine holds, so each moves last_seen and nothing else. Recording a partial list // as though it were the whole would tell a rebuilding node to remove what it still has.