From 31697e696df4d9ef649970f03bf16839180deb22 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:01:44 +0100 Subject: [PATCH] A-113: a revert whose restore fails starts the newer build again; a form change not known waits; holds say wait-held MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G A-113, §297, §560, §541, §629 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- internal/agent/agent.go | 2 +- internal/agent/kept.go | 53 +++++++++++++++++++++++++++++++------ internal/agent/kept_test.go | 28 +++++++++++++++++++- internal/link/have.go | 40 ++++++++++++++++++++++++++++ internal/link/have_test.go | 11 ++++++++ 5 files changed, 124 insertions(+), 10 deletions(-) diff --git a/internal/agent/agent.go b/internal/agent/agent.go index ba620a77..77fee06f 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -1638,7 +1638,7 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo } // A kept data on the bus whose form changes is taken as a restore point first, or restored at a revert (design 47 G §296, §297). - held, catchUps := a.keptStep(ctx, module, w, comp, ranBefore) + held, catchUps := a.keptStep(ctx, module, w, comp, ranBefore, others) if held { return } diff --git a/internal/agent/kept.go b/internal/agent/kept.go index a4f38113..056728fa 100644 --- a/internal/agent/kept.go +++ b/internal/agent/kept.go @@ -10,6 +10,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" "git.novox.be/novox/mesh-sdk/go/catalogue" + "github.com/novox/mesh-host/internal/compositions" ) // A module's kept data on the bus, taken as a restore point before a form change moves it and restored at a revert (design 47 G @@ -206,7 +207,7 @@ func (a *Agent) askAndWait(ctx context.Context, h Have, kind string, k KeptMove, // keptStep takes the restore points a move of a module needs before it applies and restores the ones a revert needs: it // answers whether the module is held (its have says why, and the running build is kept) and, after a restore, the catch-up // of the point to call once the reverted build runs. A want that says to go on without a backup asks neither. -func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp compNow, ranBefore bool) (held bool, catchUps []meshrecovery.CatchUp) { +func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp compNow, ranBefore bool, others []compositions.Composed) (held bool, catchUps []meshrecovery.CatchUp) { r := a.env.Recovery if !ranBefore || r == nil || r.FormChanges == nil || w.Want.Backup == meshwants.WantsByWantNodeModuleValueBackupSkip { return false, nil @@ -225,10 +226,17 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com } changes, err := r.FormChanges(ctx, module, prev.Fp, fp) if err != nil { - // Not known whether the move changes a form: said, and the move goes on as before, since holding every module for a - // build record that is gone would stop deliveries a form never changed. - a.sayOnce("kept:"+module+":"+fp, fmt.Sprintf("%s: whether its kept data changes form between %s and %s is not known: %v", module, prev.Fp, fp, err)) - return false, nil + // Whether the move changes a form is not known: it waits for the entries and never moves without its restore point (design 47 G + // A-113). No attempt is kept, so the composer handing the composition again, or any settle, asks again. + words := fmt.Sprintf("whether %s's kept data changes form between %s and %s is not known, so it is not moved: %v", module, prev.Fp, fp, err) + a.sayOnce("kept:"+module+":"+fp, module+" waits: "+words) + since := r.now() + a.writeHave(ctx, Have{Module: module, FP: prev.Fp, Composed: prev.Composed, Phase: string(nodeengineclient.PhaseHeld), Detail: words, + Wait: &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepCompose, Reason: "entry-missing", Since: since, Words: words}}) + if a.env.Waiting != nil { + a.env.Waiting(module, fp) + } + return true, nil } take, restore := keptMoves(changes, prev.Fp, fp) if len(take)+len(restore) == 0 { @@ -265,16 +273,26 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com return false, nil } // A revert: the newer build stops first (it removes nothing), so the live data is not written while it is set aside. + stopped := false if a.env.Stop != nil { if res := a.env.Stop(ctx, module); res.Failed { hold("the newer build could not be stopped before the restore: "+res.Detail, false) return true, nil } + stopped = true + } + // A revert that cannot restore starts the newer build again and holds the move, saying why: a module is never left stopped + // (design 47 G A-113). + holdRevert := func(words string, interrupted bool) { + if stopped { + words += "; " + a.startAgain(ctx, module, prev, others) + } + hold(words, interrupted) } for _, m := range restore { pts, err := r.KeptPoints(ctx, meshrecovery.KeptPointsInput{Module: module, Kept: m.Name}) if err != nil { - hold(fmt.Sprintf("restore point missing: the points of %s's %s cannot be read: %v", module, m.Name, err), false) + holdRevert(fmt.Sprintf("restore point missing: the points of %s's %s cannot be read: %v", module, m.Name, err), false) return true, nil } var point *meshrecovery.Point @@ -287,7 +305,7 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com } } if point == nil { - hold(fmt.Sprintf("restore point missing: %s's %s has none of form %d taken before %s", module, m.Name, m.Form, m.Fingerprint), false) + holdRevert(fmt.Sprintf("restore point missing: %s's %s has none of form %d taken before %s", module, m.Name, m.Form, m.Fingerprint), false) return true, nil } why := fmt.Sprintf("revert of %s's %s from %s", module, m.Name, m.Fingerprint) @@ -297,7 +315,7 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com return got.Accepted, got.Busy, got.Error, err }) if held != nil { - hold(fmt.Sprintf("%s's %s (form %d) is not restored: %s", module, m.Name, m.Form, held.Words), held.Interrupted) + holdRevert(fmt.Sprintf("%s's %s (form %d) is not restored: %s", module, m.Name, m.Form, held.Words), held.Interrupted) return true, nil } if point.CatchUp != nil { @@ -307,6 +325,25 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com return false, catchUps } +// startAgain starts the build a module ran before a revert stopped it, from the composition the agent applied or holds of its +// fingerprint, and says whether it did (design 47 G A-113). +func (a *Agent) startAgain(ctx context.Context, module string, prev nodeengine.HaveByNodeModuleValue, others []compositions.Composed) string { + a.mu.Lock() + c, ok := a.applied[module] + if !ok || c.C.Module == "" { + c, ok = a.compositionFor(module, prev.Fp) + } + a.mu.Unlock() + if !ok || a.env.Apply == nil { + return "the newer build could not be started again: its composition is not held" + } + res := a.env.Apply(context.WithoutCancel(ctx), c.C, others) + if res.Failed { + return "the newer build could not be started again: " + res.Detail + } + return "the newer build runs again" +} + // catchUp calls, once the reverted build of a module runs, the catch-up of each restore point restored, as the module's own // tool through the tool runner (design 47 G A-108). A tool that fails is said: the build runs, and the tool can be called by hand. func (a *Agent) catchUp(ctx context.Context, module string, calls []meshrecovery.CatchUp) { diff --git a/internal/agent/kept_test.go b/internal/agent/kept_test.go index 5c2f571b..144c5f86 100644 --- a/internal/agent/kept_test.go +++ b/internal/agent/kept_test.go @@ -27,6 +27,7 @@ type recoveryFake struct { calls []string // stop, ask, apply, catch-up, in order catchUp []meshrecovery.CatchUp expired chan struct{} + unknown error // FormChanges answers it } type fakeAnswer struct { @@ -52,7 +53,7 @@ func (f *recoveryFake) next(verb string, in any) (string, string, string) { func (f *recoveryFake) env() *Recovery { return &Recovery{ - FormChanges: func(context.Context, string, string, string) ([]catalogue.FormChange, error) { return f.changes, nil }, + FormChanges: func(context.Context, string, string, string) ([]catalogue.FormChange, error) { return f.changes, f.unknown }, SnapshotKept: func(_ context.Context, in meshrecovery.SnapshotKeptInput) (meshrecovery.SnapshotKeptAnswer, error) { a, b, e := f.next("snapshot-kept", in) return meshrecovery.SnapshotKeptAnswer{Accepted: a, Busy: b, Error: e}, nil @@ -235,6 +236,31 @@ func testAMoveAcrossAFormChange(t *testing.T) { } }) + t.Run("a revert whose restore fails starts the newer build again and holds the move, saying why (A-113)", func(t *testing.T) { + f := &recoveryFake{changes: down, + points: []meshrecovery.Point{{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("a"), Snapshot: "snap-1", At: time.Now()}}, + answers: []fakeAnswer{{accepted: "r1", then: []Ending{failed("r1", "restore point damaged")}}}} + b := moveDB(t, f, "a", "b", heldHave) + if got := f.seen(); !slices.Equal(got, []string{"stop", "kept-points", "restore-kept", "apply"}) { + t.Fatalf("calls %v", got) + } + if h := b.haves.last("db"); !contains(h.Detail, "restore point damaged") || !contains(h.Detail, "the newer build runs again") || + h.Wait == nil || h.Wait.Step != nodeengine.WaitStepCompose { + t.Fatalf("the have %+v", h) + } + }) + + t.Run("a form change not known moves nothing and waits for the entries (A-113)", func(t *testing.T) { + f := &recoveryFake{unknown: fmt.Errorf("the entry is asked")} + b := moveDB(t, f, "a", "b", heldHave) + if h := b.haves.last("db"); h.FP != fp("a") || !contains(h.Detail, "is not known") || h.Wait == nil || h.Wait.Reason != "entry-missing" { + t.Fatalf("the have %+v", h) + } + if n := len(b.appliedList()); n != 1 || len(f.seen()) != 0 { + t.Fatalf("applied %d, calls %v", n, f.seen()) + } + }) + t.Run("a restart waits for the act its have names and asks nothing again", func(t *testing.T) { f := &recoveryFake{changes: up} b := moveDB(t, f, "a", "b", applied, func(b *bed, dig string) { diff --git a/internal/link/have.go b/internal/link/have.go index e4c6bcf1..5d190a18 100644 --- a/internal/link/have.go +++ b/internal/link/have.go @@ -69,6 +69,9 @@ type haveState struct { userListNoKey map[string]bool // runningSaid is, by module, the fingerprint and manifest its event running last said, so each is said once. runningSaid map[string]string + // waitSaid is, by module, the wait its event wait-held last said (since, step, reason), so each is said once and its end + // is said when the have no longer has it (design 47 G §541, §629, A-113). + waitSaid map[string]nodeengine.Wait // unjudged is each module whose composition moved (an apply began, or the fact names another build) and that no look // has judged since it stopped moving: its health is not yet judged, never the last look's of what ran before. unjudged map[string]bool @@ -440,6 +443,13 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) { if ev, ok := s.runningToSay(module, h, q.manifestOf()); ok { said = append(said, func() { q.sayRunning(ev) }) } + if held, ended, ok := s.waitToSay(module, h.Wait); ok { + if h.Wait != nil { + said = append(said, func() { q.sayWait(nodeengine.EventWaitHeld, nodeengine.SubjectWaitHeld(node, module), held) }) + } else { + said = append(said, func() { q.sayWait(nodeengine.EventWaitEnded, nodeengine.SubjectWaitEnded(node, module), ended) }) + } + } } if !q.writeRemoved(write, bucket, node, &said) { return @@ -931,6 +941,36 @@ func (s *haveState) runningToSay(module string, h nodeengine.HaveByNodeModuleVal return nodeengine.Running{V: sdkkeys.Version, Module: module, Fingerprint: h.Fp, Manifest: digest}, true } +// waitToSay is the event wait-held for a have written with a wait the last one did not say, or wait-ended for one written with +// none after a wait said (design 47 G §541, §629): a wait of the same step, reason and since is said once. Under s.mu. +func (s *haveState) waitToSay(module string, w *nodeengine.Wait) (held, ended nodeengine.Wait, ok bool) { + last, said := s.waitSaid[module] + if w == nil { + if !said { + return held, ended, false + } + delete(s.waitSaid, module) + return held, last, true + } + if said && last.Step == w.Step && last.Reason == w.Reason && last.Since.Equal(w.Since) && last.Words == w.Words { + return held, ended, false + } + if s.waitSaid == nil { + s.waitSaid = map[string]nodeengine.Wait{} + } + s.waitSaid[module] = *w + return *w, ended, true +} + +// sayWait says the event wait-held or wait-ended of a wait, with the time its bound counts from. +func (q *Session) sayWait(event, subject string, w nodeengine.Wait) { + if w.ClockFrom == nil { + since := w.Since + w.ClockFrom = &since + } + q.sayEngineOn(subject, event, w) +} + // sayRunning says the event running, on its subject `running..`. func (q *Session) sayRunning(ev nodeengine.Running) { ev.Node, ev.At = q.Membership.Node, time.Now().UTC() diff --git a/internal/link/have_test.go b/internal/link/have_test.go index 7a76f716..4c2ceb52 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -704,6 +704,17 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth( !strings.Contains(got[0], `"fingerprint":"`+fp+`"`) || !strings.Contains(got[0], `"manifest":"sha256:`+strings.Repeat("c", 64)+`"`) { t.Fatalf("running said %v, want one for the healthy build", got) } + // A have with a wait says wait-held once on wait-held.., and wait-ended when the wait is gone (design 47 G §541, A-113). + hold := &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepCompose, Reason: "restore point missing", Since: time.Now().UTC(), Words: "w"} + q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig, Wait: hold}) + q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig, Wait: hold}) + if got := l.saidOn("mesh.seat.node-engine.event.wait-held.n1.web "); len(got) != 1 || !strings.Contains(got[0], `"reason":"restore point missing"`) || !strings.Contains(got[0], `"clock-from"`) { + t.Fatalf("wait-held said %v, want one", got) + } + q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig}) + if got := l.saidOn("mesh.seat.node-engine.event.wait-ended.n1.web "); len(got) != 1 { + t.Fatalf("wait-ended said %v, want one", got) + } // Starting: it waits. Past its start window the check fails: failed. q.SayHealth(context.Background(), *healthOf(map[string]string{"web": StateStarting})) if got := l.have(t, "n1.web"); got.Health != nodeengineclient.HealthWaits || got.Fp != fp || got.Composed != dig {