From c531c78eb514bf3b6406c58346d7c7fdf53c05a5 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 17 Oct 2026 22:48:28 +0200 Subject: [PATCH] Hold a plan whose backup failed or stayed silent, say it, and ask again by a verb (issue 1066) --- .../backups_before_send_integration_test.go | 218 ++++++++++++++++++ cmd/mesh-controller/delivery.go | 15 +- cmd/mesh-controller/doctor.go | 4 + cmd/mesh-controller/plain_words.go | 7 + cmd/mesh-controller/plan_backups.go | 129 +++++++++++ cmd/mesh-controller/plan_backups_hold_test.go | 117 ++++++++++ cmd/mesh-controller/release_plan.go | 3 + cmd/mesh-controller/seatverbs.go | 9 + cmd/mesh-controller/watchdogs.go | 5 + internal/catalogue/verbs.go | 6 + module.json | 1 + 11 files changed, 513 insertions(+), 1 deletion(-) create mode 100644 cmd/mesh-controller/backups_before_send_integration_test.go create mode 100644 cmd/mesh-controller/plan_backups_hold_test.go diff --git a/cmd/mesh-controller/backups_before_send_integration_test.go b/cmd/mesh-controller/backups_before_send_integration_test.go new file mode 100644 index 00000000..b8342eb4 --- /dev/null +++ b/cmd/mesh-controller/backups_before_send_integration_test.go @@ -0,0 +1,218 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "strings" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +// aBackupSeatOnTheBus is one machine's node-backup holder as a test makes it: it counts the `now` asks it hears and +// answers `backed-up` with the last night it is told (novox/hq ADR 0404). +type aBackupSeatOnTheBus struct { + mu sync.Mutex + nows []string + night map[string]any // module → lastNight +} + +func (f *aBackupSeatOnTheBus) serve(t *testing.T, conn *nats.Conn, node string) { + t.Helper() + reply := func(m *nats.Msg, result any) { + body, _ := json.Marshal(map[string]any{"result": result, "error": "", "node": node}) + _ = m.Respond(body) + } + now, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "now", node), func(m *nats.Msg) { + var a struct { + Module string `json:"module"` + } + _ = json.Unmarshal(m.Data, &a) + f.mu.Lock() + f.nows = append(f.nows, a.Module) + f.mu.Unlock() + reply(m, map[string]any{"started": a.Module}) + }) + if err != nil { + t.Fatal(err) + } + read, err := conn.Subscribe(link.NodeSeatToolSubject(catalogue.BackupSeat, "backed-up", node), func(m *nats.Msg) { + f.mu.Lock() + defer f.mu.Unlock() + var out []map[string]any + for module, n := range f.night { + out = append(out, map[string]any{"module": module, "lastNight": n}) + } + reply(m, out) + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = now.Unsubscribe(); _ = read.Unsubscribe() }) + if err := conn.Flush(); err != nil { + t.Fatal(err) + } +} + +func (f *aBackupSeatOnTheBus) answers(module string, ok bool, words string) { + f.mu.Lock() + defer f.mu.Unlock() + f.night = map[string]any{module: map[string]any{"ok": ok, "at": time.Now().UTC().Add(time.Second), "error": words}} +} + +func (f *aBackupSeatOnTheBus) asks() int { + f.mu.Lock() + defer f.mu.Unlock() + return len(f.nows) +} + +// Acceptance test 3 of design 47, on the controller's side: a delivery plan that moves a module keeping data waits +// for its backups on both machines before it asks a build; a backup that fails holds it, with the machine, the +// module and the backup's words, and a condition says the same; the verb the delivery's release calls asks the +// backups again, and the plan goes on. One bus and one store for the scenarios. +func TestADeliveryThatMovesABackedUpModuleWaitsForItsBackupAndAFailedOneHoldsItAndIsSaid(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + asked := asksRecorded(t) + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + before := handActConn + handActConn = js.Conn() + t.Cleanup(func() { handActConn = before }) + + if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { + t.Fatal(err) + } + register(t, open, catalogue.Manifest{Module: "keeper", Version: "1", + Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}) + notes, err := catalogue.ParseManifest([]byte(`{"module":"notes","version":"1","data":{"own":[` + + `{"id":"files","path":"/srv/notes","class":"valuable","why":"people's notes"}]}}`)) + if err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, notes, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", + Path: "modules/notes", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + for _, node := range []string{"anchor", "laptop"} { + if _, err := assign(ctx, open, node, "keeper"); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, node, "notes"); err != nil { + t.Fatal(err) + } + } + seats := map[string]*aBackupSeatOnTheBus{"anchor": {}, "laptop": {}} + for node, s := range seats { + s.serve(t, js.Conn(), node) + } + // Short waits: the bound and the look are variables, so nothing here waits for the minutes of a real backup. + wasWithin, wasEvery := backupsWithin, backupsPollEvery + backupsWithin, backupsPollEvery = time.Minute, 10*time.Millisecond + t.Cleanup(func() { backupsWithin, backupsPollEvery = wasWithin, wasEvery }) + + newPlan := func(id string) { + t.Helper() + p := inventory.Plan{ID: id, Repository: "novox/mesh-catalog", Branch: "main", Commit: "c1", Created: time.Now().UTC(), + State: inventory.PlanBuilding, Tiers: [][]string{{"notes"}}, Modules: map[string]*inventory.PlanModule{"notes": {}}} + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + } + read := func(id string) inventory.Plan { + t.Helper() + p, err := inv.PlanByID(ctx, id) + if err != nil { + t.Fatal(err) + } + return p + } + + // 1. Both machines are asked; while one has not answered, no build is asked. + newPlan("plan-one") + advancePlans(ctx, open) + if seats["anchor"].asks() != 1 || seats["laptop"].asks() != 1 || len(*asked) != 0 { + t.Fatalf("asks anchor=%d laptop=%d builds=%v", seats["anchor"].asks(), seats["laptop"].asks(), *asked) + } + seats["anchor"].answers("notes", true, "") + advancePlans(ctx, open) + if len(*asked) != 0 || read("plan-one").Backups.State != inventory.BackupPending { + t.Fatalf("a tier was asked while a backup was pending: %v %+v", *asked, read("plan-one").Backups) + } + seats["laptop"].answers("notes", true, "") + advancePlans(ctx, open) + if len(*asked) != 1 || read("plan-one").Backups.State != inventory.BackupDone { + t.Fatalf("the tier was not asked once every backup was done: %v %+v", *asked, read("plan-one").Backups) + } + finish := read("plan-one") + finish.State = inventory.PlanDone + if err := inv.SavePlan(ctx, &finish); err != nil { + t.Fatal(err) + } + + // 2. A backup that fails holds the plan: not failed, nothing asked, the words kept, a condition raised. + seats["anchor"].night, seats["laptop"].night = nil, nil + newPlan("plan-two") + advancePlans(ctx, open) + seats["anchor"].answers("notes", true, "") + seats["laptop"].answers("notes", false, "no space left on device") + advancePlans(ctx, open) + held := read("plan-two") + if held.State != inventory.PlanBuilding || held.Backups.State != inventory.BackupFailed || len(*asked) != 1 { + t.Fatalf("the plan was not held: %s %+v builds=%v", held.State, held.Backups, *asked) + } + var failed *inventory.PlanBackup + for i := range held.Backups.Asked { + if held.Backups.Asked[i].State == inventory.BackupFailed { + failed = &held.Backups.Asked[i] + } + } + if failed == nil || failed.Machine != "laptop" || failed.Module != "notes" || failed.Words != "no space left on device" || + strings.Contains(held.Note, "tried again") { + t.Fatalf("the failed backup: %+v note=%q", failed, held.Note) + } + advancePlans(ctx, open) // not retried on the next look + if again := read("plan-two"); again.Backups.State != inventory.BackupFailed || len(*asked) != 1 || + seats["laptop"].asks() != 2 { + t.Fatalf("a held plan was asked again by itself: %+v builds=%v asks=%d", again.Backups, *asked, seats["laptop"].asks()) + } + said, err := probeBackupHeld(ctx, &doctor{open: open}) + if err != nil || len(said) != 1 || said[0].Kind != kindBackupHeld || !strings.Contains(said[0].Said, "no space left on device") { + t.Fatalf("the condition: %+v %v", said, err) + } + + // 3. The verb the release calls: the failed backup is asked again, and only it, and the plan goes on. + seats["laptop"].answers("notes", false, "no space left on device") + if err := deliveryCommand(ctx, []string{"backups-again", "plan-two", "--why", "a person released it", "--by", + "mesh-delivery for jochen"}); err != nil { + t.Fatal(err) + } + if err := deliveryCommand(ctx, []string{"backups-again", "plan-two", "--why", "again"}); err == nil { + t.Fatal("a plan with no failed backup was asked again") + } + seats["laptop"].night = nil + advancePlans(ctx, open) + if seats["laptop"].asks() != 3 || seats["anchor"].asks() != 2 { + t.Fatalf("asked again: laptop=%d anchor=%d", seats["laptop"].asks(), seats["anchor"].asks()) + } + seats["laptop"].answers("notes", true, "") + advancePlans(ctx, open) + if got := read("plan-two"); got.Backups.State != inventory.BackupDone || len(*asked) != 2 { + t.Fatalf("the plan did not go on: %+v builds=%v", got.Backups, *asked) + } + if said, _ := probeBackupHeld(ctx, &doctor{open: open}); len(said) != 0 { + t.Fatalf("the condition stays: %+v", said) + } +} diff --git a/cmd/mesh-controller/delivery.go b/cmd/mesh-controller/delivery.go index e8a8da77..64c5805b 100644 --- a/cmd/mesh-controller/delivery.go +++ b/cmd/mesh-controller/delivery.go @@ -884,6 +884,19 @@ func deliveryCommand(ctx context.Context, args []string) error { fmt.Printf("%s stopped by %s at tier %d of %d; what was asked still builds and registers, nothing further "+ "is asked or sent\n", p.ID, *by, p.Tier, len(p.Tiers)) return nil + case "backups-again": + if len(positionals) != 1 || strings.TrimSpace(*why) == "" { + return errors.New("delivery backups-again --why [--by ]") + } + var p inventory.Plan + var again []inventory.PlanBackup + if err := sayingOnTheBus(ctx, inv, func() (err error) { + p, again, err = backupsAgain(ctx, inv, positionals[0], *by, strings.TrimSpace(*why)) + return err + }); err != nil { + return err + } + return answer(map[string]any{"plan": p.ID, "asked_again": again}) case "walks": var walks []inventory.Plan if *planID != "" { @@ -924,7 +937,7 @@ func deliveryCommand(ctx context.Context, args []string) error { return answer(map[string]any{"held": deliverySeatHeld(entries), "walks": walks, "own-path": sortedKeysOf(ownPathWords())}) } - return fmt.Errorf("delivery %s: plan, order, check, go, stop or walks", sub) + return fmt.Errorf("delivery %s: plan, order, check, go, stop, backups-again or walks", sub) } // appliedFrom answers a machine's own report of the send a walk made to it from the controller's `apply` durations diff --git a/cmd/mesh-controller/doctor.go b/cmd/mesh-controller/doctor.go index 93d9799e..4358b135 100644 --- a/cmd/mesh-controller/doctor.go +++ b/cmd/mesh-controller/doctor.go @@ -174,6 +174,10 @@ var probeRegistry = []probe{ {ID: busStepProbe, Asserts: "a bus upgrade a person started is said while it runs, and is followed by the bus's " + "health within its bound — or is said failed, with its snapshot as the way back", From: "ADR 0236, to-be 45 §8", Kind: kindBusMaintenance, Raises: []string{kindBusUpgradeFailed}, Phase: 4, run: probeBusStep}, + // A delivery plan held for a backup that failed or stayed silent (ADR 0404 decision 4): one condition per plan. + {ID: backupHeldProbe, Asserts: "no delivery plan is held for a backup before its send that failed or did not " + + "answer in ten minutes", From: "ADR 0404 decision 4, issue 1066", Kind: kindBackupHeld, Phase: 1, + run: probeBackupHeld}, } // probeVerdict is one probe's outcome in a run. diff --git a/cmd/mesh-controller/plain_words.go b/cmd/mesh-controller/plain_words.go index 801a2549..e3d488b4 100644 --- a/cmd/mesh-controller/plain_words.go +++ b/cmd/mesh-controller/plain_words.go @@ -603,6 +603,13 @@ var plainWordings = map[string]func(conditions.Observation) words{ Explanation: "The upgrade of the mesh's message system did not end healthy in time.", Resolved: "Resolved: the bus is healthy after its upgrade"} }), + kindBackupHeld: worded(func(o conditions.Observation) words { + return words{Headline: "A delivery waits for a backup that failed", + Needs: "fix what stopped the backup, then release the delivery.", + Explanation: "Before a delivery moves a module that keeps data, that data is backed up. A backup failed or " + + "did not answer in ten minutes, so nothing of the delivery is built or sent.", + Resolved: "Resolved: the backup is done and the delivery goes on"} + }), kindConsumerBehind: worded(func(o conditions.Observation) words { return words{Headline: "Messages pile up for a listener", Explanation: "One of the mesh's listeners on the bus is far behind, so what it handles happens late. A " + diff --git a/cmd/mesh-controller/plan_backups.go b/cmd/mesh-controller/plan_backups.go index 08111c26..81a8fbfa 100644 --- a/cmd/mesh-controller/plan_backups.go +++ b/cmd/mesh-controller/plan_backups.go @@ -7,11 +7,13 @@ import ( "fmt" "slices" "sort" + "strings" "sync" "time" "github.com/nats-io/nats.go" "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -132,6 +134,14 @@ func loadBackupFacts(ctx context.Context, inv *inventory.Inventory) (backupFacts // moved modules have no backed-up item keeps no record and goes on at once. An error is an ask that could not be // made or a read the store refused: the step is tried again, the asks already made kept. func stepBackups(ctx context.Context, p *inventory.Plan, facts backupFacts, now time.Time) (bool, error) { + proceed, err := stepBackupsOnce(ctx, p, facts, now) + if err == nil { + noteBackups(p) + } + return proceed, err +} + +func stepBackupsOnce(ctx context.Context, p *inventory.Plan, facts backupFacts, now time.Time) (bool, error) { if p.Backups == nil { b := wantedBackups(*p, facts) if b == nil { @@ -292,3 +302,122 @@ func backupsFirst(ctx context.Context, inv *inventory.Inventory, p *inventory.Pl } return stepBackups(ctx, p, facts, time.Now().UTC()) } + +// backupsNote begins the note of a plan that waits on or is held by its backups. +const backupsNote = "backups: " + +// noteBackups says in the plan's note what its backups wait on, or, held, which backup failed and in what words and +// the verb that asks again; a note of its own that the backups ended is cleared. Not the "tried again" note of a +// failed step: a held plan is not retried (novox/hq ADR 0404 decision 4). +func noteBackups(p *inventory.Plan) { + if p.Backups == nil { + return + } + var pending, failed []string + for _, a := range p.Backups.Asked { + switch a.State { + case inventory.BackupPending: + pending = append(pending, a.Module+" on "+a.Machine) + case inventory.BackupFailed: + failed = append(failed, fmt.Sprintf("%s on %s: %s", a.Module, a.Machine, a.Words)) + } + } + switch p.Backups.State { + case inventory.BackupPending: + p.Note = backupsNote + "waiting for the backup of " + strings.Join(pending, ", ") + case inventory.BackupFailed: + p.Note = backupsNote + "held — the backup of " + strings.Join(failed, "; ") + "; nothing is asked or sent. " + + "`delivery-backups-again` asks it again" + default: + if strings.HasPrefix(p.Note, backupsNote) { + p.Note = "" + } + } +} + +// waitingOnBackups says a plan has not asked its first tier because its backups are pending or failed: no tier +// late, as S3 and the lines of `plans` read it. +func waitingOnBackups(p inventory.Plan) bool { + return p.Open() && p.Release == nil && p.Tier == 0 && !askedAny(p) && p.Backups != nil && + p.Backups.State != inventory.BackupDone +} + +// askBackupsAgain clears the failed backups of a held plan, so the next step asks those again and no other. It +// says which. Refused for a plan with no failed backup. +func askBackupsAgain(p *inventory.Plan, by, why string) ([]inventory.PlanBackup, error) { + if !p.Open() || p.Backups == nil || p.Backups.State != inventory.BackupFailed { + return nil, fmt.Errorf("%s has no failed backup: there is nothing to ask again", p.ID) + } + var again []inventory.PlanBackup + for i := range p.Backups.Asked { + a := &p.Backups.Asked[i] + if a.State != inventory.BackupFailed { + continue + } + again = append(again, *a) + a.State, a.AskedAt, a.Words = inventory.BackupPending, nil, "" + } + p.Backups.State = backupsStateOf(p.Backups.Asked) + p.Note = backupsNote + "asked again by " + by + ": " + why + return again, nil +} + +// backupsAgain is askBackupsAgain for the plan with this id, kept: the verb `delivery-backups-again`. +func backupsAgain(ctx context.Context, inv *inventory.Inventory, id, by, why string) (inventory.Plan, []inventory.PlanBackup, error) { + release, err := inv.HoldPlans(ctx, true) + if err != nil { + return inventory.Plan{}, nil, err + } + defer release() + p, err := inv.PlanByID(ctx, id) + if err != nil { + return inventory.Plan{}, nil, err + } + again, err := askBackupsAgain(&p, by, why) + if err != nil { + return p, nil, err + } + if err := inv.SavePlan(ctx, &p); err != nil { + return p, nil, err + } + return p, again, nil +} + +// kindBackupHeld is a delivery plan held for a backup that failed or stayed silent; backupHeldProbe its row. +const ( + kindBackupHeld = "backup-held" + backupHeldProbe = "DK" +) + +// observeBackupHolds is one condition per plan held for its backups, with the backups' own words as its evidence. +func observeBackupHolds(plans []inventory.Plan) []conditions.Observation { + var out []conditions.Observation + for _, p := range plans { + if !p.Open() || p.Backups == nil || p.Backups.State != inventory.BackupFailed { + continue + } + var said, machines []string + for _, a := range p.Backups.Asked { + if a.State == inventory.BackupFailed { + said = append(said, fmt.Sprintf("%s on %s: %s", a.Module, a.Machine, a.Words)) + machines = append(machines, a.Machine) + } + } + out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: p.ID, Token: "backup-held", + Kind: kindBackupHeld, Severity: conditions.Urgent, Resolver: conditions.ResolverOperator, + Machine: machines[0], Also: machines[1:], + Summary: fmt.Sprintf("the delivery %s is held: %d of its backups failed or did not answer, so nothing is "+ + "asked or sent", p.ID, len(said)), + Said: strings.Join(said, "; ")}) + } + return out +} + +// probeBackupHeld reads the open plans for the ones held for their backups. +func probeBackupHeld(ctx context.Context, d *doctor) ([]conditions.Observation, error) { + plans, err := d.open.inventory.OpenPlans(ctx) + if err != nil { + return nil, err + } + return observeBackupHolds(plans), nil +} diff --git a/cmd/mesh-controller/plan_backups_hold_test.go b/cmd/mesh-controller/plan_backups_hold_test.go new file mode 100644 index 00000000..41e71296 --- /dev/null +++ b/cmd/mesh-controller/plan_backups_hold_test.go @@ -0,0 +1,117 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A backup that failed or stayed silent holds the plan, is said, and a verb asks it again (novox/hq ADR 0404 +// decision 4, issue 1066). + +// aHeldPlan is a plan whose backup on ace failed and whose backup on g14 is done. +func aHeldPlan(t *testing.T) (*inventory.Plan, backupFacts) { + t.Helper() + entries := []inventory.Entry{aModuleWith(t, "mail", []string{"ace", "g14"}, anItem("a", ""))} + facts := backupFactsFrom(entries, holders("ace", "g14")) + seat := &aFakeBackupSeat{nights: map[string]*night{}} + seat.install(t) + p := aPlanOf("mail") + stepBackups(t.Context(), p, facts, backupT0) + seat.nights["ace/mail"] = &night{OK: false, At: backupT0.Add(time.Minute), Error: "no space left on device"} + seat.nights["g14/mail"] = &night{OK: true, At: backupT0.Add(time.Minute)} + if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(2*time.Minute)); proceed || err != nil { + t.Fatalf("proceed=%v err=%v", proceed, err) + } + return p, facts +} + +// The held plan says which machine, which module and the backup's own words in its note, and is not called tried again. +func TestAHeldPlanSaysTheMachineTheModuleAndTheWordsInItsNote(t *testing.T) { + p, _ := aHeldPlan(t) + for _, want := range []string{"ace", "mail", "no space left on device", "delivery-backups-again"} { + if !strings.Contains(p.Note, want) { + t.Fatalf("note %q lacks %q", p.Note, want) + } + } + if strings.Contains(p.Note, "tried again") || p.State != inventory.PlanBuilding { + t.Fatalf("a held plan is not failed and not retried: %q %s", p.Note, p.State) + } + if !waitingOnBackups(*p) { + t.Fatal("a plan held by its backups is no tier late (S3 leaves it out)") + } +} + +// delivery-backups-again clears the failed records, and the next step asks those backups again and only those. +func TestBackupsAgainClearsTheFailedRecordsAndTheNextStepAsksThemAgain(t *testing.T) { + p, facts := aHeldPlan(t) + again, err := askBackupsAgain(p, "mesh-delivery", "a person released it") + if err != nil || len(again) != 1 || again[0].Machine != "ace" || again[0].Module != "mail" { + t.Fatalf("again=%+v err=%v", again, err) + } + if p.Backups.State != inventory.BackupPending { + t.Fatalf("state %q", p.Backups.State) + } + seat := &aFakeBackupSeat{nights: map[string]*night{}} + seat.install(t) + now := backupT0.Add(10 * time.Minute) + if proceed, err := stepBackups(t.Context(), p, facts, now); proceed || err != nil { + t.Fatalf("proceed=%v err=%v", proceed, err) + } + if strings.Join(seat.asked, " ") != "ace/mail" { + t.Fatalf("asked %v; only the failed backup is asked again", seat.asked) + } + seat.nights["ace/mail"] = &night{OK: true, At: now.Add(time.Minute)} + if proceed, _ := stepBackups(t.Context(), p, facts, now.Add(2*time.Minute)); !proceed { + t.Fatal("the plan must go on once the backup it asked again is done") + } + if strings.Contains(p.Note, "delivery-backups-again") { + t.Fatalf("the hold's note stays after it ended: %q", p.Note) + } +} + +// Refused for a plan with no failed backup. +func TestBackupsAgainIsRefusedForAPlanWithNoFailedBackup(t *testing.T) { + if _, err := askBackupsAgain(aPlanOf("mail"), "x", "y"); err == nil { + t.Fatal("a plan with no backups was asked again") + } + p := aPlanOf("mail") + p.Backups = &inventory.PlanBackups{State: inventory.BackupPending, + Asked: []inventory.PlanBackup{{Machine: "ace", Module: "mail", State: inventory.BackupPending}}} + if _, err := askBackupsAgain(p, "x", "y"); err == nil { + t.Fatal("a plan whose backups are pending was asked again") + } +} + +// The verb's argument vector: the plan and a reason are required. +func TestTheVerbDeliveryBackupsAgainTakesAPlanAndAWhy(t *testing.T) { + got, err := argvFor("delivery-backups-again", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen"}) + want := []string{"delivery", "backups-again", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen"} + if err != nil || strings.Join(got, "|") != strings.Join(want, "|") { + t.Fatalf("got %v %v, want %v", got, err, want) + } + if _, err := argvFor("delivery-backups-again", map[string]any{"plan": "plan-1"}); err == nil { + t.Fatal("no reason was accepted") + } +} + +// One condition per held plan, the backup one, with the words in its evidence; none for a plan only waiting. +func TestAHeldPlanRaisesOneConditionWithTheBackupsWords(t *testing.T) { + held, _ := aHeldPlan(t) + waiting := aPlanOf("mail") + waiting.ID = "plan-w" + waiting.Backups = &inventory.PlanBackups{State: inventory.BackupPending, + Asked: []inventory.PlanBackup{{Machine: "ace", Module: "mail", State: inventory.BackupPending}}} + got := observeBackupHolds([]inventory.Plan{*held, *waiting}) + if len(got) != 1 { + t.Fatalf("observations %+v", got) + } + o := got[0] + if o.Kind != kindBackupHeld || o.Scope != conditions.ScopePlan || o.ID != held.ID || + !strings.Contains(o.Said, "no space left on device") || !strings.Contains(o.Said, "mail") || !strings.Contains(o.Said, "ace") { + t.Fatalf("%+v", o) + } +} diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 3576be92..5b5d1590 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -1730,6 +1730,9 @@ func planLineSaid(p inventory.Plan, now time.Time, pause pauseView, bound time.D // Nor is waiting for the walk before it to end (a related walk, novox/hq ADR 0276, ADR 0407). return fmt.Sprintf("%s · %s, %s, for %s", name, where, p.Note, since) } + if waitingOnBackups(p) { + return fmt.Sprintf("%s · %s, %s, for %s", name, where, p.Note, since) + } if waiting, paused := pausedWaiting(p, pause, now); paused { return fmt.Sprintf("%s · %s, %s", name, where, waiting) } diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index bf64027e..99c1204e 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -444,6 +444,15 @@ func (a *verbArguments) commandLine() ([]string, error) { argv = append(argv, "--without", w) } return argv, nil + case "delivery-backups-again": + if err := need("plan", "why"); err != nil { + return nil, err + } + argv := []string{"delivery", "backups-again", str("plan"), "--why", str("why")} + if b := str("by"); b != "" { + argv = append(argv, "--by", catalogue.DeliverySeat+" for "+b) + } + return argv, nil case "delivery-walks": argv := []string{"delivery", "walks"} if id := str("plan"); id != "" { diff --git a/cmd/mesh-controller/watchdogs.go b/cmd/mesh-controller/watchdogs.go index fccf15e2..1c708fb5 100644 --- a/cmd/mesh-controller/watchdogs.go +++ b/cmd/mesh-controller/watchdogs.go @@ -538,6 +538,11 @@ func gatherPlans(ctx context.Context, inv *inventory.Inventory, now time.Time, b if deferred(p) { continue } + // A walk waiting on, or held by, its backups is no tier late either: the backup condition is the one voice + // (novox/hq ADR 0404 decision 4, issue 1066). + if waitingOnBackups(p) { + continue + } _, paused := pausedWaiting(p, pause, now) bound := bounds.of(p.Repository) out = append(out, planFacts{id: p.ID, repository: p.Repository, commit: p.Commit, tier: p.Tier, diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index a9c1b774..cd4c2d9c 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -177,6 +177,12 @@ var ControllerVerbs = []Verb{ Input: schema(map[string]string{"plan": "the walk's id", "why": "why", "by": "who stopped the delivery", "without": "merges to leave out, @, comma-separated"}, []string{"plan", "why"})}, + {Name: "delivery-backups-again", Description: "Ask again the backups a walk was held for (novox/hq ADR 0404 " + + "decision 4): a walk whose backup before its send failed or did not answer is held, nothing asked or sent; this " + + "clears those failed backups and the next look at the plans asks them again, and only them. Refused for a walk " + + "with no failed backup. The delivery's release calls it. The answer names the walk and the backups asked again.", + Input: schema(map[string]string{"plan": "the held walk's id", "why": "a person's reason, as release passes it", + "by": "who released the delivery"}, []string{"plan", "why"})}, {Name: "delivery-walks", Description: "The walks the controller keeps (novox/hq ADR 0239): every open one and the " + "last ended ones, each whole — its tiers, each module's state, first machines and first-node gate with its readings, and its " + "phases from its merge to every machine running it (novox/hq ADR 0282) — and whether the delivery seat has a " + diff --git a/module.json b/module.json index 815ce35d..169d08cf 100644 --- a/module.json +++ b/module.json @@ -57,6 +57,7 @@ "delivery-check", "deliver", "delivery-stop", + "delivery-backups-again", "delivery-walks", "plan", "assign",