From 914099774e4e31389b94e17f7d4c35b5afad7739 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 17 Oct 2026 22:42:21 +0200 Subject: [PATCH 1/5] Record a backup that cannot start as the module's last night (issue 1067) --- modules/restic/cmd/restic-backups/hand.go | 10 ++++++- .../restic/cmd/restic-backups/hand_test.go | 27 +++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/modules/restic/cmd/restic-backups/hand.go b/modules/restic/cmd/restic-backups/hand.go index 16f97f6f..4fc0dba5 100644 --- a/modules/restic/cmd/restic-backups/hand.go +++ b/modules/restic/cmd/restic-backups/hand.go @@ -26,8 +26,16 @@ import ( const HandTag = "hand" // BackUpNow is a backup asked for now, of every module or one: as a night, each restore point tagged HandTag. +// +// One that cannot start (the lock cannot be taken, the repository cannot be opened, the module declares nothing) +// is recorded as that module's last night, failed and with the reason: the mesh asking before a delivery reads +// it from backed-up, and would otherwise read silence (novox/hq issue 1067). func (b *Backups) BackUpNow(ctx context.Context, only string) (map[string]Night, error) { - return b.backUp(ctx, only, true) + out, err := b.backUp(ctx, only, true) + if err != nil && only != "" { + b.record(only, Night{OK: false, At: b.Now().UTC(), Error: err.Error()}) + } + return out, err } // isHand is whether a restore point was taken by hand. diff --git a/modules/restic/cmd/restic-backups/hand_test.go b/modules/restic/cmd/restic-backups/hand_test.go index f59a8ef5..22af7160 100644 --- a/modules/restic/cmd/restic-backups/hand_test.go +++ b/modules/restic/cmd/restic-backups/hand_test.go @@ -324,3 +324,30 @@ func TestAThinningThatFailsAfterLettingGoSaysWhatItLetGo(t *testing.T) { t.Errorf("said %q, want it to say aaaaaaaa was let go", why) } } + +// A backup asked for now that cannot start leaves its failure as the module's last night, so the mesh asking +// for it before a delivery reads the backup's own words from backed-up, not silence (novox/hq issue 1067). +func TestABackupAskedForNowThatCannotStartIsRecordedAsThatModulesLastNight(t *testing.T) { + h := &handRestic{} + asked := time.Date(2026, 10, 17, 22, 50, 0, 0, time.UTC) + b := &Backups{Where: placed(t, "# pg\npath /\n"), Run: h.run, Now: func() time.Time { return asked }, Say: quiet} + must(t, os.Mkdir(filepath.Join(b.Where.State, nightLock), 0o700)) // a directory cannot be opened read-write: the lock cannot be taken + if _, err := b.BackUpNow(context.Background(), "pg"); err == nil { + t.Fatal("a backup started without its lock") + } + night, ok := b.Nights()["pg"] + if !ok || night.OK || night.Error == "" || !night.At.Equal(asked) { + t.Fatalf("pg's last night is %+v, want a failure at %s with the reason", night, asked) + } + if !strings.Contains(night.Error, nightLock) { + t.Errorf("the reason is %q, want it to name the lock", night.Error) + } + // Asked of a module that declares nothing here, once the lock can be taken: recorded too, with the reason. + must(t, os.Remove(filepath.Join(b.Where.State, nightLock))) + if _, err := b.BackUpNow(context.Background(), "mail"); err == nil { + t.Fatal("a backup of a module that declares nothing started") + } + if n := b.Nights()["mail"]; n.OK || !strings.Contains(n.Error, "declares nothing") { + t.Errorf("mail's last night is %+v, want a failure saying it declares nothing", n) + } +} -- 2.54.0 From 8f7aeeeb81d4aa7d3188b741fb78ac4d6cb63025 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 17 Oct 2026 22:45:03 +0200 Subject: [PATCH 2/5] Hold a delivering delivery for a failed backup before the send; release asks the backups again (issue 1068) --- modules/mesh-delivery/README.md | 10 ++ .../cmd/mesh-delivery/backup_test.go | 126 ++++++++++++++++++ .../cmd/mesh-delivery/bus_test.go | 59 ++++++++ .../cmd/mesh-delivery/fakes_test.go | 22 ++- .../mesh-delivery/cmd/mesh-delivery/main.go | 2 +- .../cmd/mesh-delivery/manifest_test.go | 2 +- .../mesh-delivery/cmd/mesh-delivery/model.go | 34 +++++ .../mesh-delivery/cmd/mesh-delivery/ports.go | 8 ++ .../mesh-delivery/cmd/mesh-delivery/table.go | 25 ++++ .../cmd/mesh-delivery/table_test.go | 5 +- .../mesh-delivery/cmd/mesh-delivery/verbs.go | 15 +++ modules/mesh-delivery/module.json | 1 + 12 files changed, 305 insertions(+), 4 deletions(-) create mode 100644 modules/mesh-delivery/cmd/mesh-delivery/backup_test.go diff --git a/modules/mesh-delivery/README.md b/modules/mesh-delivery/README.md index 18b78dcb..c2b64c5b 100644 --- a/modules/mesh-delivery/README.md +++ b/modules/mesh-delivery/README.md @@ -7,3 +7,13 @@ The `mesh-delivery` module of the mesh catalogue: claims the seat `mesh-delivery Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416). - `${dir:state}`: everything in it is the program's; the mesh moves nothing out (ADR 0422 §2). + +## A backup that failed before the send + +The controller backs up the data of the modules a delivery moves before it asks or sends anything (ADR 0404). When its +walk's record, in `plan-moved`, names a backup that failed or stayed silent and nothing of the plan was sent, the +delivery goes from delivering to held (the observed row `backup failed`); the held reason names the machine, the +module and the backup's words, and the pull request's comment shows it. The hold does not lift by itself. `release` +of such a delivery asks the controller (`delivery-backups-again`) to take the backups again and the delivery goes on +delivering; a record from before the controller cleared the failure does not hold it again. A held delivery already +ends as covered or delivered when a later merge covers it. diff --git a/modules/mesh-delivery/cmd/mesh-delivery/backup_test.go b/modules/mesh-delivery/cmd/mesh-delivery/backup_test.go new file mode 100644 index 00000000..c95c3af3 --- /dev/null +++ b/modules/mesh-delivery/cmd/mesh-delivery/backup_test.go @@ -0,0 +1,126 @@ +package main + +import ( + "strings" + "testing" +) + +// A failed backup before the send holds the delivery, and a release asks the backups again (novox/hq ADR 0404 +// clause 4, issue 1068). The walk's field names are provisional until the controller's contract settles them. + +func failedBackup() *WalkBackups { + return &WalkBackups{Failed: []WalkBackupFailure{{Node: "ace", Module: "pg", Error: "no space left on device"}}} +} + +// aDeliveringWalk is a delivery whose walk was let go and runs, nothing sent yet. +func aDeliveringWalk(t *testing.T) (*world, string, string) { + t.Helper() + w := newWorld(t) + id := IDOf("novox/app", head) + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.h.Checked(verdict("novox/app", 7, head, "pass")) + w.h.PullMerged(merged("novox/app", 7, head, merge)) + w.h.WalkMoved(w.ctl.put(aWalk("plan-1", "novox/app", merge, true))) + w.settleAll() + if s := w.state(id); s != Delivering { + t.Fatalf("setup: %s, want delivering", s) + } + return w, id, "plan-1" +} + +func TestABackupFailedTakesADeliveringDeliveryToHeldOnlyWhileNothingWasSent(t *testing.T) { + w, id, plan := aDeliveringWalk(t) + sent := w.ctl.walk(plan) + sent.Backups = failedBackup() + sent.Modules["app"].SentAt = &w.now + w.h.WalkMoved(w.ctl.put(sent)) + w.settleAll() + if s := w.state(id); s == Held { + t.Fatal("held for a failed backup after a module of the plan was sent") + } + w, id, plan = aDeliveringWalk(t) + held := w.ctl.walk(plan) + held.Backups = failedBackup() + w.h.WalkMoved(w.ctl.put(held)) + w.settleAll() + d := w.delivery(id) + if d.State != Held { + t.Fatalf("with a failed backup and nothing sent it is %s", d.State) + } + for _, want := range []string{"ace", "pg", "no space left on device"} { + if !strings.Contains(d.HeldWhy, want) { + t.Errorf("the held reason %q does not name %q", d.HeldWhy, want) + } + } + // It does not lift by itself: the same facts read again leave it held. + w.h.WalkMoved(w.ctl.put(w.ctl.walk(plan))) + w.settleAll() + if s := w.state(id); s != Held { + t.Fatalf("read again it is %s, want it still held", s) + } +} + +func TestAReleaseOfADeliveryHeldForAFailedBackupAsksTheControllerAgainAndGoesToDelivering(t *testing.T) { + w, id, plan := aDeliveringWalk(t) + held := w.ctl.walk(plan) + held.Backups = failedBackup() + w.h.WalkMoved(w.ctl.put(held)) + w.settleAll() + if s := w.state(id); s != Held { + t.Fatalf("setup: %s, want held", s) + } + lettingGo := len(w.ctl.delivers) + if _, err := w.h.Release(id, "freed some space", "jochen"); err != nil { + t.Fatal(err) + } + if got := w.ctl.backupsAgain; len(got) != 1 || got[0] != plan { + t.Fatalf("the controller was asked to take the backups again %v, want once for %s", got, plan) + } + if len(w.ctl.delivers) != lettingGo { + t.Errorf("a started walk was let go again: %v", w.ctl.delivers) + } + if s := w.state(id); s != Delivering { + t.Fatalf("released, it is %s", s) + } + // The record the controller cleared is read: it stays delivering. + w.h.WalkMoved(w.ctl.walk(plan)) + w.settleAll() + if s := w.state(id); s != Delivering { + t.Fatalf("with the backups cleared it is %s", s) + } + // Another failure, in a later record, holds it again. + again := w.ctl.walk(plan) + again.Backups = failedBackup() + w.h.WalkMoved(w.ctl.put(again)) + w.settleAll() + if s := w.state(id); s != Held { + t.Fatalf("a failed backup in a later record left it %s", s) + } +} + +func TestARecordThatStillHoldsTheFailureAfterAReleaseDoesNotHoldItAgain(t *testing.T) { + w, id, plan := aDeliveringWalk(t) + held := w.ctl.walk(plan) + held.Backups = failedBackup() + stale := w.ctl.put(held) + w.h.WalkMoved(stale) + w.settleAll() + if _, err := w.h.Release(id, "freed some space", "jochen"); err != nil { + t.Fatal(err) + } + w.h.WalkMoved(stale) // the record as it was before the controller cleared it + w.settleAll() + if s := w.state(id); s != Delivering { + t.Fatalf("a record from before the release held it again: %s", s) + } +} + +func TestADeliveryHeldForAFailedBackupIsNotLetGoByItsWalkAlone(t *testing.T) { + // The controller's own path starts a walk with nobody's word: the held-to-delivering row for that must not + // undo the backup hold. + d := &Delivery{ID: "novox/app@c0", State: Held, MergedAs: "aaaa", Walk: &WalkSeen{ID: "plan-1", State: walkBuilding, Revision: 3, + BackupsFailed: failedBackup().Failed}} + if _, err := Apply(d, EvGo, false, Facts{}); err == nil { + t.Fatal("a walk started on the controller's own path lifted the backup hold") + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/bus_test.go b/modules/mesh-delivery/cmd/mesh-delivery/bus_test.go index 832b1ac9..36298d3c 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/bus_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/bus_test.go @@ -562,3 +562,62 @@ func TestAGroupOfTwoWithGreenChecksGetsOneAskAndOnItsApproveBothMergeInOnePassOv t.Fatalf("a second ask was made: %d", again) } } + +// **The acceptance of a failed backup before the send, on the mesh-delivery side** (novox/hq ADR 0404 clause 4, design 47 +// milestone 4, issue 1068): a merged delivery whose walk runs is held when the walk's record, as the controller's +// `plan-moved` carries it, names a backup that failed and nothing was sent; the hold names the machine, the module and the +// backup's words, and nothing asks the controller to let it go. One test function, one bus. The walk's backups field names +// are provisional until the controller's contract (issue 1065) settles them. +func TestAPlanMovedCarryingAFailedBackupHoldsTheDeliveryWithTheBackupsWordsOverARealBus(t *testing.T) { + b := newTestBus(t) + rt := b.start(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rt.deliverEvents(ctx) + rt.until("the module to subscribe to the events it follows", 20*time.Second, func() bool { + rt.mu.Lock() + defer rt.mu.Unlock() + return len(rt.subs) >= len(followed)+1 + }) + const head, merge = "3333333333333333333333333333333333333333", "4444444444444444444444444444444444444444" + announced := pr("novox/app", 9, head, "fix/y", "src/a.go") + announced.BaseSHA = "t1t1t1t1t1t1" + b.publish(t, "mesh.mod.gitea.event.pull.updated", announced) + b.publish(t, "mesh.seat."+ControllerSeat+".event.checked", judgedAt("novox/app", 9, head, "pass", "t1t1t1t1t1t1")) + b.publish(t, "mesh.mod.gitea.event.pull.merged", merged("novox/app", 9, head, merge)) + + // Its walk runs, let go: the record names no failed backup yet. + now := time.Now() + walk := aWalk("plan-1", "novox/app", merge, true) + walk.Created, walk.State, walk.Revision = now, walkRolling, 1 + walk.Delivery.Go, walk.Delivery.By = &now, "mesh-delivery" + b.publish(t, "mesh.seat."+ControllerSeat+".event.plan-moved", walk) + deliveryOf := func() Delivery { + e, err := b.kvs["deliveries"].Get(context.Background(), kvKey(IDOf("novox/app", head))) + var d Delivery + if err == nil { + _ = json.Unmarshal(e.Value(), &d) + } + return d + } + rt.until("the delivery to be delivering", 10*time.Second, func() bool { return deliveryOf().State == Delivering }) + + // A backup failed before the send: the delivery is held with the backup's words. + walk.Revision, walk.Backups = 2, &WalkBackups{Failed: []WalkBackupFailure{{Node: "ace", Module: "pg", Error: "no space left on device"}}} + b.publish(t, "mesh.seat."+ControllerSeat+".event.plan-moved", walk) + rt.until("the delivery to be held for the failed backup", 10*time.Second, func() bool { return deliveryOf().State == Held }) + d := deliveryOf() + for _, want := range []string{"ace", "pg", "no space left on device"} { + if !strings.Contains(d.HeldWhy, want) { + t.Fatalf("the held reason %q does not name %q", d.HeldWhy, want) + } + } + // It stays held, and the controller is asked nothing to let it go. + time.Sleep(500 * time.Millisecond) + if s := deliveryOf().State; s != Held { + t.Fatalf("the hold lifted by itself: %s", s) + } + if n := len(rt.asksOf("seat:"+ControllerSeat+".deliver")) + len(rt.asksOf("seat:"+ControllerSeat+".delivery-backups-again")); n != 0 { + t.Fatalf("the controller was asked %d times to let the held walk go or take its backups again", n) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go b/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go index da842249..4535513f 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go @@ -92,7 +92,9 @@ type fakeController struct { checks []string asked int delivers []string - stops []string + // backupsAgain are the walks whose failed backups were asked again. + backupsAgain []string + stops []string // withouts are the stops that left merges out (novox/hq ADR 0299): " without ,…". withouts []string // refuseWithout refuses every stop with merges to leave out, as a controller before ADR 0299 does; mergedAt are @@ -121,6 +123,24 @@ type fakeController struct { forgot []string } +// BackupsAgain records the word, and clears the walk's failed backups as the controller does. +func (c *fakeController) BackupsAgain(walk, why string) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.down { + return errors.New("down") + } + w, known := c.walks[walk] + if !known || w.Backups == nil || len(w.Backups.Failed) == 0 { + return fmt.Errorf("%s holds no failed backup", walk) + } + w.Backups = nil + w.Revision++ + c.walks[walk] = w + c.backupsAgain = append(c.backupsAgain, walk) + return nil +} + func (c *fakeController) ForgetChecks(owner, repo string, number int) error { c.mu.Lock() defer c.mu.Unlock() diff --git a/modules/mesh-delivery/cmd/mesh-delivery/main.go b/modules/mesh-delivery/cmd/mesh-delivery/main.go index a8952b94..2574d401 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/main.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/main.go @@ -329,7 +329,7 @@ func tools(h *Holder, l *listening) []stdio.Tool { return h.Recheck(strArg(a, "id"), strArg(a, "why"), aPerson) }}, {Name: seat + "release", - Description: "A person's word that a held delivery goes on: it starts delivering. With why.", + Description: "A person's word that a held delivery goes on: it starts delivering. A delivery held for a failed backup asks the controller to take the backups again first. With why.", Input: map[string]any{"type": "object", "properties": map[string]any{"id": str("the delivery's id"), "why": str("why, kept with the transition")}, "required": []string{"id", "why"}}, Run: func(a map[string]any) (any, error) { diff --git a/modules/mesh-delivery/cmd/mesh-delivery/manifest_test.go b/modules/mesh-delivery/cmd/mesh-delivery/manifest_test.go index 3c2a3088..0c866f56 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/manifest_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/manifest_test.go @@ -63,7 +63,7 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { } // It calls exactly what its ports ask: the controller's verbs and the forge's tools. want := []string{} - for _, v := range []string{"delivery-plan", "delivery-check", "deliver", "delivery-stop", "delivery-walks", "forget-checks"} { + for _, v := range []string{"delivery-plan", "delivery-check", "deliver", "delivery-backups-again", "delivery-stop", "delivery-walks", "forget-checks"} { want = append(want, "seat:"+ControllerSeat+"."+v) } want = append(want, "gitea.gitea_note_append", "gitea.gitea_delivery_view", "gitea.gitea_commit_status", diff --git a/modules/mesh-delivery/cmd/mesh-delivery/model.go b/modules/mesh-delivery/cmd/mesh-delivery/model.go index f15dc573..3637cfa4 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/model.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/model.go @@ -5,6 +5,7 @@ import ( "fmt" "maps" "regexp" + "slices" "sort" "strings" "time" @@ -74,6 +75,10 @@ type Delivery struct { MergedAs string `json:"merged_as,omitempty"` MergedAt time.Time `json:"merged_at"` HeldWhy string `json:"held_why,omitempty"` + // BackupsAgainAfter is the revision of its walk when a person released it for a failed backup: a record at that + // revision or before still holds the failure the controller was asked to clear, and does not hold it again + // (novox/hq ADR 0404, issue 1068). + BackupsAgainAfter int64 `json:"backups_again_after,omitempty"` // FromForge is a delivery made from the forge's word at its merge: its head was never heard by this owner. // History is why it is history, once it is (novox/hq issue 313). FromForge bool `json:"from_forge,omitempty"` @@ -500,11 +505,26 @@ type Walk struct { Revision int64 `json:"revision"` Release *WalkRelease `json:"release,omitempty"` Delivery *WalkDelivery `json:"delivery,omitempty"` + // Backups is what the backups the controller asks before it sends anything came to (novox/hq ADR 0404, issue + // 1068). PROVISIONAL field name and shape until the controller's contract (issue 1065) settles them. + Backups *WalkBackups `json:"backups,omitempty"` // Phases is the walk's time from its merge, phase by phase, as the controller works it out (novox/hq ADR // 0282): what a delivery's time is kept from. Absent from a controller that does not say it. Phases *WalkPhases `json:"phases,omitempty"` } +// WalkBackups is the backups of a walk before its send: the failed ones, each with the backup's own words. +type WalkBackups struct { + Failed []WalkBackupFailure `json:"failed,omitempty"` +} + +// WalkBackupFailure is one backup that failed or stayed silent past its bound. +type WalkBackupFailure struct { + Node string `json:"node"` + Module string `json:"module"` + Error string `json:"error"` +} + // WalkModule is one module of a walk. type WalkModule struct { State string `json:"state,omitempty"` @@ -712,6 +732,17 @@ type WalkSeen struct { // Built is, per module, what the walk's build of it did (novox/hq ADR 0350 §4): "reused from " or // "built: " — what `show` says beside the build phase. Built map[string]string `json:"built,omitempty"` + // BackupsFailed are the backups before the send that failed, each with the backup's words (ADR 0404). + BackupsFailed []WalkBackupFailure `json:"backups_failed,omitempty"` +} + +// backupWords says the failed backups: the machine, the module and the backup's own words. +func (w *WalkSeen) backupWords() string { + var parts []string + for _, f := range w.BackupsFailed { + parts = append(parts, fmt.Sprintf("%s on %s: %s", f.Module, f.Node, f.Error)) + } + return "a backup before the send failed, and nothing was sent: " + strings.Join(parts, "; ") } // Seen is a walk as a delivery keeps it. @@ -745,6 +776,9 @@ func (w Walk) Seen() *WalkSeen { } sort.Strings(s.Moved) s.MovedSaid = true + if w.Backups != nil && len(w.Backups.Failed) > 0 { + s.BackupsFailed = slices.Clone(w.Backups.Failed) + } return s } diff --git a/modules/mesh-delivery/cmd/mesh-delivery/ports.go b/modules/mesh-delivery/cmd/mesh-delivery/ports.go index d2370732..01c85fcf 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/ports.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/ports.go @@ -53,6 +53,9 @@ type Controller interface { Check(group string, members []Member) (string, error) // Deliver lets a waiting walk start. Deliver(walk, why string) error + // BackupsAgain asks the controller to take a held walk's failed backups again, and to go on when they pass + // (novox/hq ADR 0404 clause 4). PROVISIONAL verb name until the controller's contract settles it. + BackupsAgain(walk, why string) error // Stop ends a walk, said as stopped by whom. Stop(walk, why, by string) error // StopWithout ends a walk and walks its other merges again without the merges named, each @@ -259,6 +262,11 @@ func (c seatController) Deliver(walk, why string) error { return err } +func (c seatController) BackupsAgain(walk, why string) error { + _, err := c.verb("delivery-backups-again", map[string]any{"plan": walk, "why": why}) + return err +} + func (c seatController) ForgetChecks(owner, repo string, number int) error { _, err := c.verb("forget-checks", map[string]any{"owner": owner, "repo": repo, "number": strconv.Itoa(number)}) return err diff --git a/modules/mesh-delivery/cmd/mesh-delivery/table.go b/modules/mesh-delivery/cmd/mesh-delivery/table.go index 8f2da771..d2e0b33e 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/table.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/table.go @@ -85,6 +85,7 @@ const ( EvCovered Event = "covered" // a later merge into the same trunk delivered every module it moves EvLifted Event = "lifted" // what held it for a person is no longer so: its group's composed check passed, or a walk now answers its merge EvRewalk Event = "rewalk" // the walk that carried a later commit for it failed, or was stopped for another merge (ADR 0299): it waits for the walk that walks it again + EvBackupFailed Event = "backup failed" // a backup before the send failed or stayed silent: nothing of its plan was sent (novox/hq ADR 0404) EvTrunkMoved Event = "trunk moved" // another merge moved its trunk, and its verdict names the trunk commit before (novox/hq ADR 0323) ) @@ -335,6 +336,9 @@ var Table = []Row{ Guard: "its walk started on a word that was not this owner's: the controller's own path, which waits for " + "nobody, or a person's `plans go`", Holds: func(d *Delivery, f Facts) error { + if d.Walk != nil && len(d.Walk.BackupsFailed) > 0 && d.Walk.Revision > d.BackupsAgainAfter { + return errors.New("a backup before the send failed: only a release asks the backups again") + } return unless(d.Walk != nil && d.Walk.Started() && (!d.Walk.Waited() || d.Walk.LetGoByAPerson()), "its walk was not started by a person or the controller's own path") }}, @@ -351,6 +355,24 @@ var Table = []Row{ return unless(f.Why != "" && f.By != "", "a release is a person's word, with why") }}, + // **A failed backup holds the delivery while nothing was sent** (novox/hq ADR 0404 clause 4): the controller asks + // the backups before it asks or sends anything, and a failed or silent one holds its plan. Observed, from the + // plan's record; it does not lift by itself: a person's release asks the controller to take the backups again. + {From: []State{Delivering}, Event: EvBackupFailed, To: Held, + Guard: "its walk's record names a backup before the send that failed or stayed silent, and nothing of the plan " + + "was sent; not one the controller was already asked to take again", + Holds: func(d *Delivery, f Facts) error { + switch { + case d.Walk == nil || len(d.Walk.BackupsFailed) == 0: + return errors.New("no backup failed") + case len(d.Walk.Moved) > 0: + return errors.New("a module of the plan was sent already") + case d.Walk.Revision <= d.BackupsAgainAfter: + return errors.New("the controller was asked to take the backups again, and has not said so yet") + } + return nil + }, + Says: func(d *Delivery) string { return d.Walk.backupWords() }}, {From: []State{Delivering}, Event: EvSuperseded, To: Superseded, Guard: "its walk was superseded and names no walk that took it over: a walk that took it over answers its merge, and it follows that walk (novox/hq ADR 0276)", Holds: func(d *Delivery, f Facts) error { @@ -505,6 +527,9 @@ func Apply(d *Delivery, ev Event, act bool, f Facts) (Transition, error) { } t := Transition{At: f.Now, From: d.State, To: r.To, Event: ev, Why: why, By: by} d.State, d.Since = r.To, f.Now + if r.Event == EvBackupFailed { + d.HeldWhy = why + } d.Transitions = append(d.Transitions, t) if len(d.Transitions) > keptTransitions { d.Transitions = d.Transitions[len(d.Transitions)-keptTransitions:] diff --git a/modules/mesh-delivery/cmd/mesh-delivery/table_test.go b/modules/mesh-delivery/cmd/mesh-delivery/table_test.go index c1f902cf..b382b9c8 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/table_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/table_test.go @@ -88,6 +88,9 @@ func aDeliveryFor(r Row) (*Delivery, Facts) { case EvLifted: d.MergedAs, d.Lifted = "aaaa1111", "its group's composed check passed" d.Walk = &WalkSeen{ID: "plan-1", State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery"} + case EvBackupFailed: + d.Walk = &WalkSeen{ID: "plan-1", State: walkBuilding, Revision: 4, + BackupsFailed: []WalkBackupFailure{{Node: "ace", Module: "pg", Error: "no space left on device"}}} case EvFailed: d.Walk = &WalkSeen{ID: "plan-1", State: walkFailed} case EvSuperseded: @@ -132,7 +135,7 @@ func TestEveryRowOfTheTableIsTakenWhenItsGuardHoldsAndRefusedWhenNot(t *testing. func TestEveryPairTheTableDoesNotHoldIsRefusedByName(t *testing.T) { events := []Event{EvAnnounced, EvAppeared, EvAdopted, EvChecked, EvAccepted, EvRefused, EvRecheck, EvReask, EvNewHead, EvClosed, EvMerged, EvMergedUnchecked, EvGo, EvHold, EvRelease, EvDone, EvFailed, EvSuperseded, EvStop, - EvHistory, EvRetire, EvCovered, EvLifted, EvRewalk, EvTrunkMoved} + EvHistory, EvRetire, EvCovered, EvLifted, EvRewalk, EvTrunkMoved, EvBackupFailed} for _, from := range append([]State{None}, AllStates...) { for _, ev := range events { for _, act := range []bool{false, true} { diff --git a/modules/mesh-delivery/cmd/mesh-delivery/verbs.go b/modules/mesh-delivery/cmd/mesh-delivery/verbs.go index 9657c2b8..634399bc 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/verbs.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/verbs.go @@ -368,8 +368,23 @@ func (h *Holder) Release(id, why, by string) (string, error) { h.mu.Unlock() return "", err } + // Held for a failed backup before the send: the controller is asked to take the backups again (novox/hq ADR + // 0404 clause 4); its record at this revision or before still holds the failure, and does not hold it again. + backupsOf := "" + if d.Walk != nil && len(d.Walk.BackupsFailed) > 0 { + backupsOf, d.BackupsAgainAfter = d.Walk.ID, d.Walk.Revision + } d.HeldWhy = "" h.moved(d, []Transition{t}) + if backupsOf != "" { + h.mu.Unlock() + if err := h.Controller.BackupsAgain(backupsOf, "released by "+by+": "+why); err != nil { + h.again("a walk's backups the controller refused or did not answer") + return "", fmt.Errorf("%s is delivering on %s's word, and the controller could not be asked to take the backups of "+ + "%s again (%v): the failed backup holds it again when its record is read", id, by, backupsOf, err) + } + return fmt.Sprintf("%s is delivering on %s's word; the backups of its walk %s are asked again", id, by, backupsOf), nil + } var walk string if d.Walk != nil && d.Walk.Waits { walk = d.Walk.ID diff --git a/modules/mesh-delivery/module.json b/modules/mesh-delivery/module.json index 2cab76d1..4359918c 100644 --- a/modules/mesh-delivery/module.json +++ b/modules/mesh-delivery/module.json @@ -40,6 +40,7 @@ "seat:mesh-controller.delivery-plan", "seat:mesh-controller.delivery-check", "seat:mesh-controller.deliver", + "seat:mesh-controller.delivery-backups-again", "seat:mesh-controller.delivery-stop", "seat:mesh-controller.delivery-walks", "seat:mesh-controller.forget-checks", -- 2.54.0 From 4799f792424cd40a6b72b5f9c71a630902d7f907 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 17 Oct 2026 22:45:46 +0200 Subject: [PATCH 3/5] Read the plan's backups as the controller says them: machine, module, state, words (issue 1068) --- .../cmd/mesh-delivery/backup_test.go | 8 ++-- .../cmd/mesh-delivery/bus_test.go | 6 +-- .../cmd/mesh-delivery/fakes_test.go | 2 +- .../mesh-delivery/cmd/mesh-delivery/model.go | 37 +++++++++++++------ .../mesh-delivery/cmd/mesh-delivery/ports.go | 2 +- .../cmd/mesh-delivery/table_test.go | 2 +- 6 files changed, 37 insertions(+), 20 deletions(-) diff --git a/modules/mesh-delivery/cmd/mesh-delivery/backup_test.go b/modules/mesh-delivery/cmd/mesh-delivery/backup_test.go index c95c3af3..e30022bf 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/backup_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/backup_test.go @@ -6,10 +6,12 @@ import ( ) // A failed backup before the send holds the delivery, and a release asks the backups again (novox/hq ADR 0404 -// clause 4, issue 1068). The walk's field names are provisional until the controller's contract settles them. +// clause 4, issue 1068). The walk's backups field is the controller's contract. func failedBackup() *WalkBackups { - return &WalkBackups{Failed: []WalkBackupFailure{{Node: "ace", Module: "pg", Error: "no space left on device"}}} + return &WalkBackups{State: "failed", Asked: []WalkBackupAsked{ + {Machine: "ace", Module: "mail", State: "done"}, + {Machine: "ace", Module: "pg", State: "failed", Words: "no space left on device"}}} } // aDeliveringWalk is a delivery whose walk was let go and runs, nothing sent yet. @@ -119,7 +121,7 @@ func TestADeliveryHeldForAFailedBackupIsNotLetGoByItsWalkAlone(t *testing.T) { // The controller's own path starts a walk with nobody's word: the held-to-delivering row for that must not // undo the backup hold. d := &Delivery{ID: "novox/app@c0", State: Held, MergedAs: "aaaa", Walk: &WalkSeen{ID: "plan-1", State: walkBuilding, Revision: 3, - BackupsFailed: failedBackup().Failed}} + BackupsFailed: []WalkBackupFailure{{Machine: "ace", Module: "pg", Words: "no space left on device"}}}} if _, err := Apply(d, EvGo, false, Facts{}); err == nil { t.Fatal("a walk started on the controller's own path lifted the backup hold") } diff --git a/modules/mesh-delivery/cmd/mesh-delivery/bus_test.go b/modules/mesh-delivery/cmd/mesh-delivery/bus_test.go index 36298d3c..f05fa263 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/bus_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/bus_test.go @@ -566,8 +566,8 @@ func TestAGroupOfTwoWithGreenChecksGetsOneAskAndOnItsApproveBothMergeInOnePassOv // **The acceptance of a failed backup before the send, on the mesh-delivery side** (novox/hq ADR 0404 clause 4, design 47 // milestone 4, issue 1068): a merged delivery whose walk runs is held when the walk's record, as the controller's // `plan-moved` carries it, names a backup that failed and nothing was sent; the hold names the machine, the module and the -// backup's words, and nothing asks the controller to let it go. One test function, one bus. The walk's backups field names -// are provisional until the controller's contract (issue 1065) settles them. +// backup's words, and nothing asks the controller to let it go. One test function, one bus. The walk's backups field is the controller's +// contract (issue 1065). func TestAPlanMovedCarryingAFailedBackupHoldsTheDeliveryWithTheBackupsWordsOverARealBus(t *testing.T) { b := newTestBus(t) rt := b.start(t) @@ -603,7 +603,7 @@ func TestAPlanMovedCarryingAFailedBackupHoldsTheDeliveryWithTheBackupsWordsOverA rt.until("the delivery to be delivering", 10*time.Second, func() bool { return deliveryOf().State == Delivering }) // A backup failed before the send: the delivery is held with the backup's words. - walk.Revision, walk.Backups = 2, &WalkBackups{Failed: []WalkBackupFailure{{Node: "ace", Module: "pg", Error: "no space left on device"}}} + walk.Revision, walk.Backups = 2, &WalkBackups{State: "failed", Asked: []WalkBackupAsked{{Machine: "ace", Module: "pg", State: "failed", Words: "no space left on device"}}} b.publish(t, "mesh.seat."+ControllerSeat+".event.plan-moved", walk) rt.until("the delivery to be held for the failed backup", 10*time.Second, func() bool { return deliveryOf().State == Held }) d := deliveryOf() diff --git a/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go b/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go index 4535513f..bf616f19 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go @@ -131,7 +131,7 @@ func (c *fakeController) BackupsAgain(walk, why string) error { return errors.New("down") } w, known := c.walks[walk] - if !known || w.Backups == nil || len(w.Backups.Failed) == 0 { + if !known || w.Backups == nil || w.Backups.State != "failed" { return fmt.Errorf("%s holds no failed backup", walk) } w.Backups = nil diff --git a/modules/mesh-delivery/cmd/mesh-delivery/model.go b/modules/mesh-delivery/cmd/mesh-delivery/model.go index 3637cfa4..822c5a97 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/model.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/model.go @@ -5,7 +5,6 @@ import ( "fmt" "maps" "regexp" - "slices" "sort" "strings" "time" @@ -506,23 +505,35 @@ type Walk struct { Release *WalkRelease `json:"release,omitempty"` Delivery *WalkDelivery `json:"delivery,omitempty"` // Backups is what the backups the controller asks before it sends anything came to (novox/hq ADR 0404, issue - // 1068). PROVISIONAL field name and shape until the controller's contract (issue 1065) settles them. + // 1068); absent when the plan moves no backed-up item. Backups *WalkBackups `json:"backups,omitempty"` // Phases is the walk's time from its merge, phase by phase, as the controller works it out (novox/hq ADR // 0282): what a delivery's time is kept from. Absent from a controller that does not say it. Phases *WalkPhases `json:"phases,omitempty"` } -// WalkBackups is the backups of a walk before its send: the failed ones, each with the backup's own words. +// WalkBackups is the backups of a walk before its send, as the controller's plan record says them: state pending, +// done or failed (failed as soon as any entry failed), each backup asked, and the machines with no backup holder. type WalkBackups struct { - Failed []WalkBackupFailure `json:"failed,omitempty"` + State string `json:"state"` + Asked []WalkBackupAsked `json:"asked,omitempty"` + NoHolder []string `json:"no_holder,omitempty"` } -// WalkBackupFailure is one backup that failed or stayed silent past its bound. +// WalkBackupAsked is one backup asked of a machine for a module: its state, and the backup's own words. +type WalkBackupAsked struct { + Machine string `json:"machine"` + Module string `json:"module"` + State string `json:"state"` + AskedAt *time.Time `json:"asked_at,omitempty"` + Words string `json:"words,omitempty"` +} + +// WalkBackupFailure is one backup that failed or stayed silent past its bound, as a delivery keeps it. type WalkBackupFailure struct { - Node string `json:"node"` - Module string `json:"module"` - Error string `json:"error"` + Machine string `json:"machine"` + Module string `json:"module"` + Words string `json:"words"` } // WalkModule is one module of a walk. @@ -740,7 +751,7 @@ type WalkSeen struct { func (w *WalkSeen) backupWords() string { var parts []string for _, f := range w.BackupsFailed { - parts = append(parts, fmt.Sprintf("%s on %s: %s", f.Module, f.Node, f.Error)) + parts = append(parts, fmt.Sprintf("%s on %s: %s", f.Module, f.Machine, f.Words)) } return "a backup before the send failed, and nothing was sent: " + strings.Join(parts, "; ") } @@ -776,8 +787,12 @@ func (w Walk) Seen() *WalkSeen { } sort.Strings(s.Moved) s.MovedSaid = true - if w.Backups != nil && len(w.Backups.Failed) > 0 { - s.BackupsFailed = slices.Clone(w.Backups.Failed) + if w.Backups != nil { + for _, a := range w.Backups.Asked { + if a.State == "failed" { + s.BackupsFailed = append(s.BackupsFailed, WalkBackupFailure{Machine: a.Machine, Module: a.Module, Words: a.Words}) + } + } } return s } diff --git a/modules/mesh-delivery/cmd/mesh-delivery/ports.go b/modules/mesh-delivery/cmd/mesh-delivery/ports.go index 01c85fcf..c893fe0f 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/ports.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/ports.go @@ -54,7 +54,7 @@ type Controller interface { // Deliver lets a waiting walk start. Deliver(walk, why string) error // BackupsAgain asks the controller to take a held walk's failed backups again, and to go on when they pass - // (novox/hq ADR 0404 clause 4). PROVISIONAL verb name until the controller's contract settles it. + // (novox/hq ADR 0404 clause 4). BackupsAgain(walk, why string) error // Stop ends a walk, said as stopped by whom. Stop(walk, why, by string) error diff --git a/modules/mesh-delivery/cmd/mesh-delivery/table_test.go b/modules/mesh-delivery/cmd/mesh-delivery/table_test.go index b382b9c8..573e4935 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/table_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/table_test.go @@ -90,7 +90,7 @@ func aDeliveryFor(r Row) (*Delivery, Facts) { d.Walk = &WalkSeen{ID: "plan-1", State: walkBuilding, Waits: true, WaitedFor: "mesh-delivery"} case EvBackupFailed: d.Walk = &WalkSeen{ID: "plan-1", State: walkBuilding, Revision: 4, - BackupsFailed: []WalkBackupFailure{{Node: "ace", Module: "pg", Error: "no space left on device"}}} + BackupsFailed: []WalkBackupFailure{{Machine: "ace", Module: "pg", Words: "no space left on device"}}} case EvFailed: d.Walk = &WalkSeen{ID: "plan-1", State: walkFailed} case EvSuperseded: -- 2.54.0 From 82693a441cf0854a12db3c48404a596c61f12475 Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 17 Oct 2026 22:53:43 +0200 Subject: [PATCH 4/5] A release the controller refuses to take leaves the delivery held for its failed backup (issue 1068) --- .../cmd/mesh-delivery/backup_test.go | 26 +++++++++++++++++++ .../cmd/mesh-delivery/fakes_test.go | 7 ++++- .../mesh-delivery/cmd/mesh-delivery/verbs.go | 13 +++++++--- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery/cmd/mesh-delivery/backup_test.go b/modules/mesh-delivery/cmd/mesh-delivery/backup_test.go index e30022bf..1b802538 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/backup_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/backup_test.go @@ -126,3 +126,29 @@ func TestADeliveryHeldForAFailedBackupIsNotLetGoByItsWalkAlone(t *testing.T) { t.Fatal("a walk started on the controller's own path lifted the backup hold") } } + +func TestAReleaseWhoseBackupsTheControllerRefusedToAskAgainLeavesTheDeliveryHeld(t *testing.T) { + w, id, plan := aDeliveringWalk(t) + held := w.ctl.walk(plan) + held.Backups = failedBackup() + w.h.WalkMoved(w.ctl.put(held)) + w.settleAll() + w.ctl.refuseBackupsAgain = true + _, err := w.h.Release(id, "freed some space", "jochen") + if err == nil || !strings.Contains(err.Error(), "held again") { + t.Fatalf("the release of a refused word answered %v, want an error saying it is held again", err) + } + w.settleAll() + d := w.delivery(id) + if d.State != Held || d.BackupsAgainAfter != 0 || !strings.Contains(d.HeldWhy, "no space left on device") { + t.Fatalf("after the refusal it is %s (again-after %d, why %q), want held for the backup", d.State, d.BackupsAgainAfter, d.HeldWhy) + } + // The controller takes the word the next time: the delivery goes on. + w.ctl.refuseBackupsAgain = false + if _, err := w.h.Release(id, "freed some space", "jochen"); err != nil { + t.Fatal(err) + } + if s := w.state(id); s != Delivering { + t.Fatalf("released again, it is %s", s) + } +} diff --git a/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go b/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go index bf616f19..c40666e8 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/fakes_test.go @@ -94,7 +94,9 @@ type fakeController struct { delivers []string // backupsAgain are the walks whose failed backups were asked again. backupsAgain []string - stops []string + // refuseBackupsAgain refuses the word to take a walk's backups again. + refuseBackupsAgain bool + stops []string // withouts are the stops that left merges out (novox/hq ADR 0299): " without ,…". withouts []string // refuseWithout refuses every stop with merges to leave out, as a controller before ADR 0299 does; mergedAt are @@ -130,6 +132,9 @@ func (c *fakeController) BackupsAgain(walk, why string) error { if c.down { return errors.New("down") } + if c.refuseBackupsAgain { + return errors.New("the controller refuses the backups") + } w, known := c.walks[walk] if !known || w.Backups == nil || w.Backups.State != "failed" { return fmt.Errorf("%s holds no failed backup", walk) diff --git a/modules/mesh-delivery/cmd/mesh-delivery/verbs.go b/modules/mesh-delivery/cmd/mesh-delivery/verbs.go index 634399bc..06c4d7db 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/verbs.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/verbs.go @@ -370,7 +370,7 @@ func (h *Holder) Release(id, why, by string) (string, error) { } // Held for a failed backup before the send: the controller is asked to take the backups again (novox/hq ADR // 0404 clause 4); its record at this revision or before still holds the failure, and does not hold it again. - backupsOf := "" + backupsOf, againBefore := "", d.BackupsAgainAfter if d.Walk != nil && len(d.Walk.BackupsFailed) > 0 { backupsOf, d.BackupsAgainAfter = d.Walk.ID, d.Walk.Revision } @@ -379,9 +379,16 @@ func (h *Holder) Release(id, why, by string) (string, error) { if backupsOf != "" { h.mu.Unlock() if err := h.Controller.BackupsAgain(backupsOf, "released by "+by+": "+why); err != nil { + // The controller did not clear the failure: its record still holds it, so the delivery is held for it again now. + h.mu.Lock() + if d := h.deliveries[id]; d != nil { + d.BackupsAgainAfter = againBefore + h.settleAndKeep(d) + } + h.mu.Unlock() h.again("a walk's backups the controller refused or did not answer") - return "", fmt.Errorf("%s is delivering on %s's word, and the controller could not be asked to take the backups of "+ - "%s again (%v): the failed backup holds it again when its record is read", id, by, backupsOf, err) + return "", fmt.Errorf("%s was not released: the controller could not be asked to take the backups of %s again "+ + "(%v), so it is held again for the failed backup", id, backupsOf, err) } return fmt.Sprintf("%s is delivering on %s's word; the backups of its walk %s are asked again", id, by, backupsOf), nil } -- 2.54.0 From c898a937a1e333c49ddd91470d1d9573b175660c Mon Sep 17 00:00:00 2001 From: jochen Date: Sat, 17 Oct 2026 22:57:44 +0200 Subject: [PATCH 5/5] Order the events of the failed-backup bus test: each after the module took the one before (issue 1068) --- .../cmd/mesh-delivery/bus_test.go | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/modules/mesh-delivery/cmd/mesh-delivery/bus_test.go b/modules/mesh-delivery/cmd/mesh-delivery/bus_test.go index f05fa263..34c5b329 100644 --- a/modules/mesh-delivery/cmd/mesh-delivery/bus_test.go +++ b/modules/mesh-delivery/cmd/mesh-delivery/bus_test.go @@ -582,16 +582,6 @@ func TestAPlanMovedCarryingAFailedBackupHoldsTheDeliveryWithTheBackupsWordsOverA const head, merge = "3333333333333333333333333333333333333333", "4444444444444444444444444444444444444444" announced := pr("novox/app", 9, head, "fix/y", "src/a.go") announced.BaseSHA = "t1t1t1t1t1t1" - b.publish(t, "mesh.mod.gitea.event.pull.updated", announced) - b.publish(t, "mesh.seat."+ControllerSeat+".event.checked", judgedAt("novox/app", 9, head, "pass", "t1t1t1t1t1t1")) - b.publish(t, "mesh.mod.gitea.event.pull.merged", merged("novox/app", 9, head, merge)) - - // Its walk runs, let go: the record names no failed backup yet. - now := time.Now() - walk := aWalk("plan-1", "novox/app", merge, true) - walk.Created, walk.State, walk.Revision = now, walkRolling, 1 - walk.Delivery.Go, walk.Delivery.By = &now, "mesh-delivery" - b.publish(t, "mesh.seat."+ControllerSeat+".event.plan-moved", walk) deliveryOf := func() Delivery { e, err := b.kvs["deliveries"].Get(context.Background(), kvKey(IDOf("novox/app", head))) var d Delivery @@ -600,6 +590,21 @@ func TestAPlanMovedCarryingAFailedBackupHoldsTheDeliveryWithTheBackupsWordsOverA } return d } + // Each event is published after the module has taken the one before it: they are on two subjects, and the runner + // may hand over a later one first, which the table would rightly judge on its own (merged before checked). + b.publish(t, "mesh.mod.gitea.event.pull.updated", announced) + rt.until("the pull request to be taken", 10*time.Second, func() bool { return deliveryOf().State != "" }) + b.publish(t, "mesh.seat."+ControllerSeat+".event.checked", judgedAt("novox/app", 9, head, "pass", "t1t1t1t1t1t1")) + rt.until("the verdict to be taken", 10*time.Second, func() bool { d := deliveryOf(); return d.State != "" && d.State != Proposed }) + b.publish(t, "mesh.mod.gitea.event.pull.merged", merged("novox/app", 9, head, merge)) + rt.until("the merge to be taken", 10*time.Second, func() bool { return deliveryOf().MergedAs != "" }) + + // Its walk runs, let go: the record names no failed backup yet. + now := time.Now() + walk := aWalk("plan-1", "novox/app", merge, true) + walk.Created, walk.State, walk.Revision = now, walkRolling, 1 + walk.Delivery.Go, walk.Delivery.By = &now, "mesh-delivery" + b.publish(t, "mesh.seat."+ControllerSeat+".event.plan-moved", walk) rt.until("the delivery to be delivering", 10*time.Second, func() bool { return deliveryOf().State == Delivering }) // A backup failed before the send: the delivery is held with the backup's words. -- 2.54.0