From 9a20547dbd220c517e5e9995b6c1fb880877108e Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 17 Oct 2026 22:45:09 +0200 Subject: [PATCH] Ask the backup seats before the first tier (issue 1065) --- cmd/mesh-controller/plan_backups.go | 294 +++++++++++++++++++++++ cmd/mesh-controller/plan_backups_test.go | 224 +++++++++++++++++ cmd/mesh-controller/release_plan.go | 9 + internal/broker/nats.go | 3 +- internal/inventory/plan_backups_test.go | 35 +++ internal/inventory/plans.go | 36 +++ 6 files changed, 600 insertions(+), 1 deletion(-) create mode 100644 cmd/mesh-controller/plan_backups.go create mode 100644 cmd/mesh-controller/plan_backups_test.go create mode 100644 internal/inventory/plan_backups_test.go diff --git a/cmd/mesh-controller/plan_backups.go b/cmd/mesh-controller/plan_backups.go new file mode 100644 index 00000000..08111c26 --- /dev/null +++ b/cmd/mesh-controller/plan_backups.go @@ -0,0 +1,294 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "sync" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// **A backup before the send** (novox/hq ADR 0404 decisions 2 and 3, design 47, issue 1065). Once a delivery plan +// has started and before it asks its first tier, the controller asks the node-backup holder of every machine that +// runs a moved module with an item protected by a backup to back that module up now, and asks nothing of a builder +// and sends nothing until every one of them answered done. The plan keeps the record (inventory.PlanBackups). + +// backupsWithin is how long a backup is given from its ask to say it is done, and backupsPollEvery how soon the plan +// is looked at again while it waits: variables so a test waits for neither. +var ( + backupsWithin = 10 * time.Minute + backupsPollEvery = 10 * time.Second +) + +// night is one module's last night as a holder's `backed-up` says it (lastNight). +type night struct { + OK bool `json:"ok"` + At time.Time `json:"at"` + Error string `json:"error,omitempty"` +} + +// askBackupNow asks a machine's backup holder to back a module up now; the holder answers when it has started. A +// variable so a test needs no bus. An error is the holder's own words when it refused, or why it could not be asked. +var askBackupNow = func(ctx context.Context, machine, module string) error { + return onTheBus(func(conn *nats.Conn) error { + answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "now", machine, + map[string]any{"module": module}, 30*time.Second) + if err != nil { + return &backupNotAsked{fmt.Errorf("%s's backup holder was not asked: %w", machine, err)} + } + if answer.Error != "" { + return fmt.Errorf("%s", answer.Error) + } + return nil + }) +} + +// backupNotAsked is an ask that could not be made, as opposed to one the holder refused: the step is tried again. +type backupNotAsked struct{ error } + +func (e *backupNotAsked) Unwrap() error { return e.error } + +// readLastNights reads a machine's holder's `backed-up` by module: how each module's last backup went. A variable +// so a test needs no bus. +var readLastNights = func(ctx context.Context, machine string) (map[string]*night, error) { + var out map[string]*night + err := onTheBus(func(conn *nats.Conn) error { + answer, err := link.AskSeatTool(ctx, conn, catalogue.BackupSeat, "backed-up", machine, map[string]any{}, 10*time.Second) + if err != nil { + return err + } + if answer.Error != "" { + return fmt.Errorf("%s", answer.Error) + } + var read []struct { + Module string `json:"module"` + LastNight *night `json:"lastNight"` + } + if err := json.Unmarshal(answer.Result, &read); err != nil { + return fmt.Errorf("its answer is not readable: %w", err) + } + out = map[string]*night{} + for _, m := range read { + out[m.Module] = m.LastNight + } + return nil + }) + return out, err +} + +// backupFacts are what the step needs of the mesh: the modules with an item protected by a backup, where each +// runs, and which machines hold node-backup. +type backupFacts struct { + backedUp map[string][]string // module → machines running it + holder map[string]bool +} + +// backupFactsFrom reads the facts from the catalogue's entries and the seat holdings. A module is backed up when +// one of its own items is (catalogue.DataItem.BackedUp: never a cache, `backup: none` or redundancy alone). A +// module whose policy records (ADR 0377) is decided per machine after the build, so every machine running it is +// asked: a superset costs time and never leaves a send without its restore point. +func backupFactsFrom(entries []inventory.Entry, held []catalogue.Held) backupFacts { + f := backupFacts{backedUp: map[string][]string{}, holder: map[string]bool{}} + for _, e := range entries { + for _, it := range e.Manifest.DataItems() { + if it.BackedUp() { + f.backedUp[e.Manifest.Module] = slices.Clone(e.On) + break + } + } + } + for _, h := range held { + if s, known := catalogue.SeatNamed(h.Claim); known && s.Name == catalogue.BackupSeat { + f.holder[h.Node] = true + } + } + return f +} + +// loadBackupFacts reads the facts from the inventory. +func loadBackupFacts(ctx context.Context, inv *inventory.Inventory) (backupFacts, error) { + entries, err := inv.Catalogued(ctx) + if err != nil { + return backupFacts{}, err + } + held, err := inv.Holdings(ctx) + if err != nil { + return backupFacts{}, err + } + return backupFactsFrom(entries, held), nil +} + +// stepBackups takes one step of a plan's backups and says whether the plan may go on to ask its first tier. It +// asks what is not asked, reads how the asked ones went, and records it all in p.Backups; it waits (false, nil) +// while any is pending, and stays stopped (false, nil) once one failed, which part B holds and says. A plan whose +// moved modules have no backed-up item keeps no record and goes on at once. An error is an ask that could not be +// made or a read the store refused: the step is tried again, the asks already made kept. +func stepBackups(ctx context.Context, p *inventory.Plan, facts backupFacts, now time.Time) (bool, error) { + if p.Backups == nil { + b := wantedBackups(*p, facts) + if b == nil { + return true, nil + } + p.Backups = b + } + b := p.Backups + switch b.State { + case inventory.BackupDone: + return true, nil + case inventory.BackupFailed: + return false, nil + } + nights := map[string]map[string]*night{} + for i := range b.Asked { + a := &b.Asked[i] + if a.State != inventory.BackupPending { + continue + } + if a.AskedAt == nil { + asked := now + if err := askBackupNow(ctx, a.Machine, a.Module); err != nil { + var unasked *backupNotAsked + if errors.As(err, &unasked) { + return false, err + } + a.State, a.Words = inventory.BackupFailed, err.Error() + a.AskedAt = &asked + continue + } + a.AskedAt = &asked + continue + } + if _, read := nights[a.Machine]; !read { + n, err := readLastNights(ctx, a.Machine) + if err != nil { + n = nil // not read now: the bound below still ends the wait + } + nights[a.Machine] = n + } + if n := nights[a.Machine][a.Module]; n != nil && n.At.After(*a.AskedAt) { + if n.OK { + a.State, a.Words = inventory.BackupDone, "" + } else { + a.State, a.Words = inventory.BackupFailed, n.Error + if a.Words == "" { + a.Words = "the backup failed and did not say why" + } + } + continue + } + if now.Sub(*a.AskedAt) >= backupsWithin { + a.State = inventory.BackupFailed + a.Words = "did not answer in " + wordsOfBound(backupsWithin) + } + } + b.State = backupsStateOf(b.Asked) + switch b.State { + case inventory.BackupDone: + return true, nil + case inventory.BackupPending: + lookAgainForBackups(p.ID) + } + return false, nil +} + +// wantedBackups is the record of backups a plan has to take, or nil for none: one entry per machine with a holder +// running a moved module that has a backed-up item, and the machines with none. +func wantedBackups(p inventory.Plan, facts backupFacts) *inventory.PlanBackups { + var asked []inventory.PlanBackup + noHolder := map[string]bool{} + for _, tier := range p.Tiers { + for _, module := range tier { + for _, machine := range facts.backedUp[module] { + if !facts.holder[machine] { + noHolder[machine] = true + continue + } + asked = append(asked, inventory.PlanBackup{Machine: machine, Module: module, State: inventory.BackupPending}) + } + } + } + if len(asked) == 0 && len(noHolder) == 0 { + return nil + } + sort.SliceStable(asked, func(i, j int) bool { + if asked[i].Machine != asked[j].Machine { + return asked[i].Machine < asked[j].Machine + } + return asked[i].Module < asked[j].Module + }) + b := &inventory.PlanBackups{Asked: asked} + for m := range noHolder { + b.NoHolder = append(b.NoHolder, m) + } + sort.Strings(b.NoHolder) + b.State = backupsStateOf(asked) + return b +} + +// backupsStateOf is the whole of the backups' states: failed if any failed, done if all are (or none are asked). +func backupsStateOf(asked []inventory.PlanBackup) string { + state := inventory.BackupDone + for _, a := range asked { + switch a.State { + case inventory.BackupFailed: + return inventory.BackupFailed + case inventory.BackupPending: + state = inventory.BackupPending + } + } + return state +} + +func wordsOfBound(d time.Duration) string { + if d == 10*time.Minute { + return "ten minutes" + } + if d%time.Minute == 0 { + return fmt.Sprintf("%d minutes", int(d/time.Minute)) + } + return d.String() +} + +var ( + backupLookMu sync.Mutex + backupLooks = map[string]bool{} +) + +// lookAgainForBackups has the serving controller look at the plans again soon, once per waiting plan: a backup ending +// is no event the controller hears, so the wait is looked at by a timer, never by blocking the hold on the plans. +func lookAgainForBackups(plan string) { + backupLookMu.Lock() + if backupLooks[plan] { + backupLookMu.Unlock() + return + } + backupLooks[plan] = true + backupLookMu.Unlock() + time.AfterFunc(backupsPollEvery, func() { + backupLookMu.Lock() + delete(backupLooks, plan) + backupLookMu.Unlock() + askToLook() + }) +} + +// backupsFirst is the step advanceOnce takes before it asks a walk's first tier: whether the plan may go on. +func backupsFirst(ctx context.Context, inv *inventory.Inventory, p *inventory.Plan) (bool, error) { + facts := backupFacts{} + if p.Backups == nil { + f, err := loadBackupFacts(ctx, inv) + if err != nil { + return false, err + } + facts = f + } + return stepBackups(ctx, p, facts, time.Now().UTC()) +} diff --git a/cmd/mesh-controller/plan_backups_test.go b/cmd/mesh-controller/plan_backups_test.go new file mode 100644 index 00000000..804dde4e --- /dev/null +++ b/cmd/mesh-controller/plan_backups_test.go @@ -0,0 +1,224 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// The backups before a delivery plan asks anything (novox/hq ADR 0404 decisions 2 and 3, issue 1065). + +// aModuleWith is a catalogue entry for a module keeping the given data items, on the given machines. +func aModuleWith(t *testing.T, name string, on []string, items ...string) inventory.Entry { + t.Helper() + raw := `{"module":"` + name + `","version":"1"` + if len(items) > 0 { + raw += `,"data":{"own":[` + strings.Join(items, ",") + `]}` + } + raw += `}` + var m catalogue.Manifest + if err := json.Unmarshal([]byte(raw), &m); err != nil { + t.Fatal(err) + } + return inventory.Entry{Manifest: m, On: on} +} + +func anItem(id, extra string) string { + if extra != "" { + extra = "," + extra + } + return `{"id":"` + id + `","path":"/x/` + id + `","class":"valuable","why":"w"` + extra + `}` +} + +func holders(nodes ...string) []catalogue.Held { + var out []catalogue.Held + for _, n := range nodes { + out = append(out, catalogue.Held{Claim: catalogue.BackupSeat, Node: n, Module: "restic"}) + } + return out +} + +// aFakeBackupSeat stands in for the machines' node-backup holders. +type aFakeBackupSeat struct { + asked []string // "machine/module", in order + nights map[string]*night + refuse map[string]string +} + +func (f *aFakeBackupSeat) install(t *testing.T) { + t.Helper() + was, wasRead := askBackupNow, readLastNights + t.Cleanup(func() { askBackupNow, readLastNights = was, wasRead }) + askBackupNow = func(_ context.Context, machine, module string) error { + f.asked = append(f.asked, machine+"/"+module) + if why := f.refuse[machine+"/"+module]; why != "" { + return fmt.Errorf("%s", why) + } + return nil + } + readLastNights = func(_ context.Context, machine string) (map[string]*night, error) { + out := map[string]*night{} + for k, n := range f.nights { + if m, mod, _ := strings.Cut(k, "/"); m == machine { + out[mod] = n + } + } + return out, nil + } +} + +func aPlanOf(modules ...string) *inventory.Plan { + p := &inventory.Plan{ID: "plan-b", State: inventory.PlanBuilding, Tiers: [][]string{modules}, + Modules: map[string]*inventory.PlanModule{}} + for _, m := range modules { + p.Modules[m] = &inventory.PlanModule{} + } + return p +} + +var backupT0 = time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC) + +// Only a module with an item protected by a backup is asked, and on the machines that run it. +func TestOnlyModulesWithAnItemProtectedByABackupAreAsked(t *testing.T) { + seat := &aFakeBackupSeat{} + seat.install(t) + entries := []inventory.Entry{ + aModuleWith(t, "copied", []string{"ace", "g14"}, anItem("a", "")), + aModuleWith(t, "dumped", []string{"ace"}, anItem("b", `"backup":{"dump":"pg_dump","into":"d"}`), anItem("d", "")), + aModuleWith(t, "mirrored", []string{"ace"}, anItem("c", `"redundancy":"a mirror"`)), + aModuleWith(t, "unbacked", []string{"ace"}, anItem("e", `"backup":"none"`)), + aModuleWith(t, "cached", []string{"ace"}, `{"id":"f","path":"/x/f","class":"cache"}`), + aModuleWith(t, "dataless", []string{"ace"}), + } + p := aPlanOf("copied", "dumped", "mirrored", "unbacked", "cached", "dataless") + facts := backupFactsFrom(entries, holders("ace", "g14")) + proceed, err := stepBackups(t.Context(), p, facts, backupT0) + if err != nil || proceed { + t.Fatalf("proceed=%v err=%v; the plan must wait for the backups it asked", proceed, err) + } + want := "ace/copied ace/dumped g14/copied" + if got := strings.Join(seat.asked, " "); got != want { + t.Fatalf("asked %q, want %q", got, want) + } + if p.Backups == nil || p.Backups.State != inventory.BackupPending || len(p.Backups.Asked) != 3 { + t.Fatalf("the plan's record: %+v", p.Backups) + } +} + +// A machine with no backup holder is asked nothing, the plan says so, and it does not hold the plan. +func TestANodeWithNoBackupHolderAsksNothingAndThePlanSaysSo(t *testing.T) { + seat := &aFakeBackupSeat{} + seat.install(t) + entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace", "bare"}, anItem("a", ""))} + p := aPlanOf("copied") + proceed, err := stepBackups(t.Context(), p, backupFactsFrom(entries, holders("ace")), backupT0) + if err != nil || proceed { + t.Fatalf("proceed=%v err=%v", proceed, err) + } + if strings.Join(seat.asked, " ") != "ace/copied" { + t.Fatalf("asked %v", seat.asked) + } + if p.Backups == nil || len(p.Backups.NoHolder) != 1 || p.Backups.NoHolder[0] != "bare" { + t.Fatalf("the plan must say bare has no holder: %+v", p.Backups) + } + // Only the no-holder machine: nothing to wait for, the plan goes on, and still says it. + only := []inventory.Entry{aModuleWith(t, "copied", []string{"bare"}, anItem("a", ""))} + q := aPlanOf("copied") + proceed, err = stepBackups(t.Context(), q, backupFactsFrom(only, holders("ace")), backupT0) + if err != nil || !proceed || q.Backups == nil || q.Backups.NoHolder[0] != "bare" || q.Backups.State != inventory.BackupDone { + t.Fatalf("proceed=%v err=%v record=%+v", proceed, err, q.Backups) + } +} + +// Nothing of the plan goes on until every backup answered done; a backup is not asked twice. +func TestNothingIsAskedOfTheBuildOrSentUntilEveryBackupAnsweredDone(t *testing.T) { + seat := &aFakeBackupSeat{nights: map[string]*night{}} + seat.install(t) + entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace", "g14"}, anItem("a", ""))} + facts := backupFactsFrom(entries, holders("ace", "g14")) + p := aPlanOf("copied") + if proceed, _ := stepBackups(t.Context(), p, facts, backupT0); proceed { + t.Fatal("went on before any backup answered") + } + // ace is done, g14 still running (its last night is from before the ask). + seat.nights["ace/copied"] = &night{OK: true, At: backupT0.Add(time.Minute)} + seat.nights["g14/copied"] = &night{OK: true, At: backupT0.Add(-time.Hour)} + if proceed, _ := stepBackups(t.Context(), p, facts, backupT0.Add(2*time.Minute)); proceed { + t.Fatal("went on while g14's backup had not answered") + } + seat.nights["g14/copied"] = &night{OK: true, At: backupT0.Add(3 * time.Minute)} + proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(4*time.Minute)) + if err != nil || !proceed || p.Backups.State != inventory.BackupDone { + t.Fatalf("proceed=%v err=%v record=%+v", proceed, err, p.Backups) + } + if len(seat.asked) != 2 { + t.Fatalf("asked %v; each backup is asked once", seat.asked) + } +} + +// A failed backup, and a silent one past the bound, are recorded with the backup's own words and the plan stops. +func TestAFailedOrSilentBackupIsRecordedAndThePlanStopsAsking(t *testing.T) { + seat := &aFakeBackupSeat{nights: map[string]*night{}} + seat.install(t) + entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace", "g14"}, anItem("a", ""))} + facts := backupFactsFrom(entries, holders("ace", "g14")) + p := aPlanOf("copied") + stepBackups(t.Context(), p, facts, backupT0) + seat.nights["ace/copied"] = &night{OK: false, At: backupT0.Add(time.Minute), Error: "no space left on device"} + if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(backupsWithin+time.Minute)); proceed || err != nil { + t.Fatalf("proceed=%v err=%v", proceed, err) + } + got := map[string]inventory.PlanBackup{} + for _, b := range p.Backups.Asked { + got[b.Machine] = b + } + if got["ace"].State != inventory.BackupFailed || got["ace"].Words != "no space left on device" { + t.Fatalf("ace: %+v", got["ace"]) + } + if got["g14"].State != inventory.BackupFailed || !strings.Contains(got["g14"].Words, "did not answer in ten minutes") { + t.Fatalf("g14: %+v", got["g14"]) + } + if p.Backups.State != inventory.BackupFailed { + t.Fatalf("state %q", p.Backups.State) + } + // It stays stopped, asks nothing more, and says no retry in the plan's note. + n := len(seat.asked) + if proceed, err := stepBackups(t.Context(), p, facts, backupT0.Add(time.Hour)); proceed || err != nil || len(seat.asked) != n { + t.Fatalf("a failed backup must stay stopped: proceed=%v err=%v asked=%v", proceed, err, seat.asked) + } +} + +// A holder that refuses the ask says why, in the record. +func TestAHolderThatRefusesTheAskIsRecordedFailedWithItsWords(t *testing.T) { + seat := &aFakeBackupSeat{refuse: map[string]string{"ace/copied": "copied declares nothing to back up"}} + seat.install(t) + entries := []inventory.Entry{aModuleWith(t, "copied", []string{"ace"}, anItem("a", ""))} + p := aPlanOf("copied") + if proceed, err := stepBackups(t.Context(), p, backupFactsFrom(entries, holders("ace")), backupT0); proceed || err != nil { + t.Fatalf("proceed=%v err=%v", proceed, err) + } + if b := p.Backups.Asked[0]; b.State != inventory.BackupFailed || b.Words != "copied declares nothing to back up" { + t.Fatalf("%+v", b) + } +} + +// A plan moving no backed-up item asks nothing and keeps no record (design 47: costs nothing). +func TestAPlanThatMovesNoBackedUpItemAsksNothing(t *testing.T) { + seat := &aFakeBackupSeat{} + seat.install(t) + entries := []inventory.Entry{ + aModuleWith(t, "dataless", []string{"ace"}), + aModuleWith(t, "mirrored", []string{"ace"}, anItem("c", `"redundancy":"a mirror"`)), + } + p := aPlanOf("dataless", "mirrored") + proceed, err := stepBackups(t.Context(), p, backupFactsFrom(entries, holders("ace")), backupT0) + if err != nil || !proceed || p.Backups != nil || len(seat.asked) != 0 { + t.Fatalf("proceed=%v err=%v record=%+v asked=%v", proceed, err, p.Backups, seat.asked) + } +} diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 69aa1f8d..3576be92 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -807,6 +807,15 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, return changed, nil } } + // **A backup before the send** (novox/hq ADR 0404, issue 1065): a walk that starts asks the backup holders of + // the machines running a moved module that keeps data, and asks its first tier only when every one is done. + if p.Tier == 0 && !askedInTier(*p) { + if proceed, err := backupsFirst(ctx, inv, p); err != nil { + return false, err + } else if !proceed { + return false, nil + } + } if len(unasked) < len(tier) { fmt.Printf("%s: tier %d was asked in part; asking the rest: %s\n", p.ID, p.Tier, strings.Join(unasked, ", ")) } diff --git a/internal/broker/nats.go b/internal/broker/nats.go index a1af12fe..07d30883 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -245,7 +245,8 @@ var VerbsTheSelfCheckAsks = []SeatVerb{{Seat: "node-intrusion-prevention", Verb: // VerbsTheBusStepAsks are the seat verbs the bus's planned step calls (novox/hq to-be 45 §8, ADR 0236): // the bus machine's backup holder takes the bus's snapshot now, before the bus is replaced (ADR 0235's // dump), and says when it is taken (`backed-up`, granted with the self-check's). The one verb of -// the controller's grant that acts, and only through the step a person starts. +// the controller's grant that acts: through the step a person starts, and through the backups a delivery plan +// takes before it asks its first tier (novox/hq ADR 0404). var VerbsTheBusStepAsks = []SeatVerb{{Seat: "node-backup", Verb: "now"}} // VerbsTheControllerAsksForASecret are the seat verbs `give` calls (novox/hq ADR 0259 §10): the operator's diff --git a/internal/inventory/plan_backups_test.go b/internal/inventory/plan_backups_test.go new file mode 100644 index 00000000..40a9f301 --- /dev/null +++ b/internal/inventory/plan_backups_test.go @@ -0,0 +1,35 @@ +package inventory + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + "time" +) + +// The plan's backups are the contract mesh-delivery reads from `plan-moved` (issue 1065): their JSON field names +// are fixed here, and a plan keeps them through a round trip. +func TestAPlanKeepsItsBackupsThroughItsJSONUnderTheNamesTheDeliveryReads(t *testing.T) { + at := time.Date(2026, 10, 17, 12, 0, 0, 0, time.UTC) + p := Plan{ID: "p", Backups: &PlanBackups{State: BackupFailed, NoHolder: []string{"bare"}, + Asked: []PlanBackup{{Machine: "ace", Module: "mail", State: BackupFailed, AskedAt: &at, Words: "no space left"}}}} + raw, err := json.Marshal(p) + if err != nil { + t.Fatal(err) + } + for _, name := range []string{`"backups":{"state":"failed","asked":[{"machine":"ace","module":"mail","state":"failed",` + + `"asked_at":"2026-10-17T12:00:00Z","words":"no space left"}],"no_holder":["bare"]}`} { + if !strings.Contains(string(raw), name) { + t.Fatalf("plan JSON %s lacks %s", raw, name) + } + } + var back Plan + if err := json.Unmarshal(raw, &back); err != nil || !reflect.DeepEqual(back.Backups, p.Backups) { + t.Fatalf("round trip: %v %+v", err, back.Backups) + } + raw, _ = json.Marshal(Plan{ID: "q"}) + if strings.Contains(string(raw), "backups") { + t.Fatalf("a plan with no backups says none: %s", raw) + } +} diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index c2e2bed4..d2e15402 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -62,12 +62,48 @@ type Plan struct { // Phases is the walk's time from its merge, phase by phase (ADR 0282 decision 6): never kept, worked out // from the walk's own moments by whoever says the walk (`delivery walks`, `plan-moved`). Phases *WalkPhases `json:"phases,omitempty"` + // Backups is the backups the walk takes before it asks its first tier (novox/hq ADR 0404 decisions 2 and 3): + // per machine and module, what was asked and how it went, and the machines with no backup holder. Nil for a + // walk that moves no module with an item protected by a backup, and for a walk kept before it was. + Backups *PlanBackups `json:"backups,omitempty"` // WalkSaid is, on the save that started or ended the walk, which of the two it was (novox/hq ADR 0308 // decision 1): set by the save, never kept, read by whoever says the save (`plan-moved`). Empty on every // other save. WalkSaid string `json:"-"` } +// The states of a plan's backups, as a whole and each. +const ( + BackupPending = "pending" + BackupDone = "done" + BackupFailed = "failed" +) + +// PlanBackups is a walk's record of the backups it takes before it asks anything (novox/hq ADR 0404): read by +// mesh-delivery from `plan-moved`, so its JSON field names are a contract between the two repositories. +type PlanBackups struct { + // State is the whole: pending while any backup is asked or unasked and none failed, failed as soon as one + // failed, done when every one is. + State string `json:"state"` + // Asked is one entry per machine and module, sorted by machine then module. + Asked []PlanBackup `json:"asked"` + // NoHolder are the machines that run a module with an item protected by a backup and hold no node-backup: + // nothing is asked of them, and the plan says so. + NoHolder []string `json:"no_holder,omitempty"` +} + +// PlanBackup is one backup asked of one machine's backup holder for one module. +type PlanBackup struct { + Machine string `json:"machine"` + Module string `json:"module"` + // State is pending, done or failed. + State string `json:"state"` + // AskedAt is when the holder was asked; nil while it is not asked yet. + AskedAt *time.Time `json:"asked_at,omitempty"` + // Words are the backup's own: the holder's error for a failed one, "did not answer in ten minutes" for a silent one. + Words string `json:"words,omitempty"` +} + // PlanTimes are a walk's own moments beside its modules' (novox/hq ADR 0282 decision 6). type PlanTimes struct { // WindowClosed is when its batch's merge window closed: no merge for the window's length, or its maximum.