From d217056cf745c5841362be51c30523bd2ed42192 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 20 Nov 2026 18:56:43 +0100 Subject: [PATCH] H-A-177: move a data module without its restore point where no recovery holder is placed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The node-engine held every data-holding module's take-in for good: it asked the seat mesh-recovery for the restore point, and nothing answers because no holder of the seat is placed on the mesh (H §456). It now asks only where the assigner's placements it holds name a holder (and while they are not read yet); where none is placed it moves the module without a point and raises the finding restore-point-missing (warning, scope machine, id the module), cleared at the module's next take-in that takes its point. A holder placed that does not answer still holds the move. Builds: design 47 H §456, H-A-177 --- cmd/mesh-host/nodeagent.go | 8 +++++ cmd/mesh-host/seatlist.go | 2 +- internal/agent/kept.go | 29 +++++++++++++++ internal/agent/kept_test.go | 53 ++++++++++++++++++++++++++++ internal/seatwait/restorepoint.go | 29 +++++++++++++++ internal/seatwait/seatwait_test.go | 14 ++++++++ internal/selfcompose/entries.go | 20 +++++++++++ internal/selfcompose/entries_test.go | 19 ++++++++++ internal/selfcompose/selfcompose.go | 6 ++++ 9 files changed, 179 insertions(+), 1 deletion(-) create mode 100644 internal/seatwait/restorepoint.go diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 68b29564..a32bda60 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -714,6 +714,14 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus apply.UseEngineModule(composer.EngineModule) // the guard of the engine's own trees fixes no name (§527) env.Composed, env.SelfComposed, env.Policy, env.ComposedToday = composer, true, composer.Policy, composer.ComposedToday env.Recovery = recoveryEnv(session, composer) + // A restore point is asked where the assigner's placements name a holder of mesh-recovery; where none is placed on the + // mesh the module moves without one and says it as the finding restore-point-missing (design 47 H, H-A-177, H §456). + env.Recovery.Placed = composer.RecoveryPlaced + env.Recovery.PointMissing = func(ctx context.Context, module, words string, missing bool) { + if err := seatwait.SayRestorePointMissing(ctx, findings, node, findings.observer(), module, words, missing, time.Now); err != nil { + say("the finding that " + module + " moved without its restore point cannot be kept: " + err.Error()) + } + } env.WhyUnmade, env.Waiting, env.RunningManifest = composer.WhyUnmade, composer.Waiting, composer.RunningManifest keptFetchFn := keptArtifactFetch(composer.KeptBuildOf, session.AskSeatOn, session.KeptObject) keptFetch.Store(&keptFetchFn) diff --git a/cmd/mesh-host/seatlist.go b/cmd/mesh-host/seatlist.go index 15b0fd0c..14b07135 100644 --- a/cmd/mesh-host/seatlist.go +++ b/cmd/mesh-host/seatlist.go @@ -125,7 +125,7 @@ func newFindingsOfTheNode(session *link.Session, node, module string, say func(s Module: module, Node: node, State: link.FindingsState{Session: session}, - Kinds: append([]string{seatwait.Kind, seatwait.HeldKind, seatwait.EntryNotFoundKind, seatwait.EntryUnansweredKind}, seatwait.NodeKinds...), + Kinds: append([]string{seatwait.Kind, seatwait.HeldKind, seatwait.EntryNotFoundKind, seatwait.EntryUnansweredKind, seatwait.RestorePointMissingKind}, seatwait.NodeKinds...), Asked: condseat.SubjectFindingsAsked, Logf: func(format string, args ...any) { say(fmt.Sprintf(format, args...)) }, Publish: session.PublishEvent, diff --git a/internal/agent/kept.go b/internal/agent/kept.go index 185e9a0d..f45f315c 100644 --- a/internal/agent/kept.go +++ b/internal/agent/kept.go @@ -3,6 +3,7 @@ package agent import ( "context" "fmt" + "strings" "time" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" @@ -51,6 +52,13 @@ type Recovery struct { Bound func(ctx context.Context, what, id, module string, since time.Time, d time.Duration) (expired <-chan struct{}, end func(), err error) // CatchUp calls the module's tool on this node through the tool runner (design 47 G A-108). CatchUp func(ctx context.Context, module string, c meshrecovery.CatchUp) error + // Placed says whether the assigner's placements name a holder of the seat mesh-recovery anywhere on the mesh, and whether + // the placements are read yet (design 47 H, H-A-177). Nil, or placements not yet read, reads as a holder placed: the point is + // asked as before. + Placed func() (placed, read bool) + // PointMissing raises (missing) the finding restore-point-missing of a module moved without its restore point, with the words + // naming what moved so, or clears it at a take-in that took its point (H-A-177). Nil raises none. + PointMissing func(ctx context.Context, module, words string, missing bool) // WaitMinutes is the bound in minutes, the config key recovery-wait-minutes; 0 reads as RecoveryWaitMinutes. WaitMinutes int // Now is the clock; nil reads the machine's. @@ -268,6 +276,24 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com a.writeHave(ctx, Have{Module: module, FP: prev.Fp, Composed: prev.Composed, Phase: string(nodeengineclient.PhaseHeld), Detail: words, Wait: &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepCompose, Reason: words, Since: since, Words: words}}) } + // Where the assigner's placements name no holder of the seat mesh-recovery anywhere on the mesh, no restore point can be taken: + // the module moves without one, said as the finding restore-point-missing, never a silent skip and never a hold for good; a + // holder placed that does not answer holds the move as before (design 47 H, H-A-177, H §456). + if len(take) > 0 && r.Placed != nil { + if placed, read := r.Placed(); read && !placed { + var what []string + for _, m := range take { + what = append(what, fmt.Sprintf("%s (form %d)", m.Name, m.Form)) + } + words := fmt.Sprintf("its kept data %s moved to %s with no restore point taken: no holder of the seat %s is placed on the mesh", + strings.Join(what, ", "), fp, meshrecovery.Seat) + a.env.Say(fmt.Sprintf("%s moves without its restore point: %s", module, words)) + if r.PointMissing != nil { + r.PointMissing(ctx, module, words, true) + } + take = nil + } + } backing := Have{Module: module, FP: fp, Composed: dig, Phase: string(nodeengineclient.PhaseBackingUp)} for _, m := range take { why := fmt.Sprintf("form change of %s's %s to %s", module, m.Name, fp) @@ -281,6 +307,9 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com return true, nil } } + if len(take) > 0 && r.PointMissing != nil { + r.PointMissing(ctx, module, "", false) // a take-in with its points clears a move without one (H-A-177) + } if len(restore) == 0 { return false, nil } diff --git a/internal/agent/kept_test.go b/internal/agent/kept_test.go index 3fab81eb..7747b252 100644 --- a/internal/agent/kept_test.go +++ b/internal/agent/kept_test.go @@ -31,6 +31,9 @@ type recoveryFake struct { expired chan struct{} unknown error // FormChanges answers it waited int // times the agent told the composer it waits for a composition + // noHolder is the assigner's placements naming no holder of mesh-recovery, unread their not being read yet (H-A-177). + noHolder, unread bool + missing []string // the finding restore-point-missing raised ("raise: ") or cleared ("clear") } type fakeAnswer struct { @@ -86,6 +89,16 @@ func (f *recoveryFake) env() *Recovery { f.catchUp = append(f.catchUp, c) return nil }, + Placed: func() (bool, bool) { return !f.noHolder, !f.unread }, + PointMissing: func(_ context.Context, module, words string, missing bool) { + f.mu.Lock() + defer f.mu.Unlock() + if missing { + f.missing = append(f.missing, "raise "+module+": "+words) + } else { + f.missing = append(f.missing, "clear "+module) + } + }, Now: time.Now, } } @@ -167,8 +180,48 @@ func testAMoveAcrossAFormChange(t *testing.T) { if f.asks[0] != want { t.Fatalf("asked %s, want %s", f.asks[0], want) } + if !slices.Equal(f.missing, []string{"clear db"}) { + t.Fatalf("a take-in with its point did not clear restore-point-missing: %v", f.missing) + } }) + // Design 47 H, H-A-177, H §456: the point is asked where the placements name a holder of mesh-recovery (and while they are not + // read yet); where none is placed on the mesh the module moves without one, raising restore-point-missing; a holder placed that + // does not answer holds the move. + for _, row := range []struct { + name string + noHolder, unread bool + calls []string + moved bool + missing string + }{ + {name: "no holder of mesh-recovery placed on the mesh moves the module without its point and raises restore-point-missing", + noHolder: true, calls: []string{"apply"}, moved: true, missing: "raise db: its kept data cache (form 1) moved to " + fp("b") + " with no restore point taken: no holder of the seat mesh-recovery is placed on the mesh"}, + {name: "a holder placed that does not answer holds the move and raises nothing", calls: []string{"snapshot-kept"}}, + {name: "placements not read yet ask the point as before", noHolder: true, unread: true, calls: []string{"snapshot-kept"}}, + } { + t.Run(row.name, func(t *testing.T) { + f := &recoveryFake{changes: up, noHolder: row.noHolder, unread: row.unread} + settle := heldHave + if row.moved { + settle = applied + } + b := moveDB(t, f, "a", "b", settle) + if got := f.seen(); !slices.Equal(got, row.calls) { + t.Fatalf("calls %v, want %v", got, row.calls) + } + f.mu.Lock() + missing := slices.Clone(f.missing) + f.mu.Unlock() + if row.missing == "" && len(missing) != 0 || row.missing != "" && !slices.Equal(missing, []string{row.missing}) { + t.Fatalf("restore-point-missing %v, want %q", missing, row.missing) + } + if h := b.haves.last("db"); row.moved && h.Phase == "held" || !row.moved && h.FP != fp("a") { + t.Fatalf("the have %+v", h) + } + }) + } + t.Run("busy waits for the other act's ending and asks again, and a stopped act is asked again; another act's failure wakes nothing", func(t *testing.T) { f := &recoveryFake{changes: up, answers: []fakeAnswer{ {busy: "x1", then: []Ending{failed("zz", "cap"), done("x1")}}, diff --git a/internal/seatwait/restorepoint.go b/internal/seatwait/restorepoint.go new file mode 100644 index 00000000..2dc86df2 --- /dev/null +++ b/internal/seatwait/restorepoint.go @@ -0,0 +1,29 @@ +package seatwait + +import ( + "context" + "fmt" + "time" + + condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" +) + +// RestorePointMissingKind is the finding the node-engine writes when it moves a module across a form change without its restore +// point, because the assigner's placements name no holder of the seat mesh-recovery anywhere on the mesh (design 47 H, H-A-177, +// H §456): a warning, scope the machine, its id the module. It clears at the module's next take-in with a point. +const RestorePointMissingKind = "restore-point-missing" + +// SayRestorePointMissing writes the finding for module when missing, with the words naming what moved without a point, and clears +// it when no longer: a take-in of the module took its point. A write or a clear that fails is returned for the caller to say. +func SayRestorePointMissing(ctx context.Context, w Writer, node, observer, module, words string, missing bool, now func() time.Time) error { + if !missing { + return w.Clear(ctx, scopeMachine, module, RestorePointMissingKind, "a take-in of the module took its restore point") + } + return w.Write(ctx, condseat.Finding{Observer: observer, Kind: RestorePointMissingKind, Scope: scopeMachine, ID: module, + Severity: condseat.FindingSeverityWarning, Source: RestorePointMissingKind, + Subject: condseat.Subject{Scope: scopeMachine, ID: module, Machine: node}, + Summary: fmt.Sprintf("%s moved on %s without its restore point: %s", module, node, words), + Headline: fmt.Sprintf("%s moved without a restore point on %s", module, node), + Explanation: "No holder of the seat mesh-recovery is placed on the mesh, so the node-engine moved the module across a change of its kept data's form without taking a restore point first: that move cannot be reverted to the data as it was. The finding clears at the module's next take-in that takes its point.", + At: now()}) +} diff --git a/internal/seatwait/seatwait_test.go b/internal/seatwait/seatwait_test.go index 128e1442..a742726b 100644 --- a/internal/seatwait/seatwait_test.go +++ b/internal/seatwait/seatwait_test.go @@ -241,6 +241,20 @@ func caseAnEntryNotFoundIsAFindingOfTheFingerprint(t *testing.T) { if err := SayEntryUnanswered(context.Background(), u, "n1", "mesh-host", fp, 0, false, now); err != nil || len(u.cleared) != 1 || u.cleared[0].Kind != EntryUnansweredKind || u.cleared[0].ID != u.put[0].ID { t.Fatalf("cleared %+v, %v", u.cleared, err) } + // And design 47 H, H-A-177: a module moved without its restore point is the finding restore-point-missing, a warning of scope + // machine with the module as its id, on the node's machine, cleared by the same. + r := &fakeFindings{} + if err := SayRestorePointMissing(context.Background(), r, "n1", "mesh-host", "db", "cache (form 1)", true, now); err != nil { + t.Fatal(err) + } + if len(r.put) != 1 || RestorePointMissingKind != "restore-point-missing" || r.put[0].Kind != RestorePointMissingKind || r.put[0].Scope != "machine" || + r.put[0].ID != "db" || r.put[0].Severity != condseat.FindingSeverityWarning || r.put[0].Subject.Machine != "n1" || !strings.Contains(r.put[0].Summary, "cache (form 1)") { + t.Fatalf("written %+v", r.put) + } + if err := SayRestorePointMissing(context.Background(), r, "n1", "mesh-host", "db", "", false, now); err != nil || len(r.cleared) != 1 || + r.cleared[0].Kind != RestorePointMissingKind || r.cleared[0].ID != "db" { + t.Fatalf("cleared %+v, %v", r.cleared, err) + } } // ADR 0636 §1, the node's finding of a seat list unread. diff --git a/internal/selfcompose/entries.go b/internal/selfcompose/entries.go index 5daf0032..29a7c351 100644 --- a/internal/selfcompose/entries.go +++ b/internal/selfcompose/entries.go @@ -15,6 +15,7 @@ import ( "sync" "time" + "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -577,6 +578,25 @@ func (c *Composer) needsAddressOf(node, module string) bool { return false } +// RecoveryPlaced says whether the assigner's placements this composer holds name a holder of the seat mesh-recovery on any node of +// the mesh, and whether they are read yet: the watch of the seat's assignments has caught up (design 47 H, H-A-177, H §456). +func (c *Composer) RecoveryPlaced() (placed, read bool) { + read = c.assignmentsRead.Load() + c.mu.Lock() + defer c.mu.Unlock() + for _, p := range c.assignments { + for _, h := range p.Holds { + if h.Claim == meshrecovery.Seat { + return true, read + } + if s, ok := catalogue.SeatNamed(h.Claim); ok && s.Name == meshrecovery.Seat { + return true, read + } + } + } + return false, read +} + // askTrunkBuilds asks, for each module another node runs at a build with no entry and whose placement names no build, the trunk // build by the seat's verb `module`, once per running build (A-461b). Called before the lock is taken: the verb is a call. func (c *Composer) askTrunkBuilds() (wait func()) { diff --git a/internal/selfcompose/entries_test.go b/internal/selfcompose/entries_test.go index ae9aec8c..b0d7b36f 100644 --- a/internal/selfcompose/entries_test.go +++ b/internal/selfcompose/entries_test.go @@ -666,6 +666,25 @@ func TestOnlyWhatTheNodeNeedsOfAnotherNodesPlacementIsAsked(t *testing.T) { if len(asked) != 2 || asked["distribution"] != 1 || asked["provider"] != 1 { t.Errorf("the trunk builds asked are %v, want the store's holder and the bound provider alone", asked) } + // Design 47 H, H-A-177: the same placements say whether a holder of the seat mesh-recovery is placed on any node, and whether + // they are read yet. + t.Run("a holder of mesh-recovery is placed where any node's placement holds the seat (H-A-177)", func(t *testing.T) { + if placed, read := c.RecoveryPlaced(); placed || read { + t.Fatalf("placed %v read %v, want neither before the watch caught up and with no holder", placed, read) + } + c.assignmentsRead.Store(true) + if placed, read := c.RecoveryPlaced(); placed || !read { + t.Fatalf("placed %v read %v, want no holder read", placed, read) + } + c.mu.Lock() + p := c.assignments["n2"] + p.Holds = append(p.Holds, meshmoduleassigner.Holding{Claim: "mesh-recovery", Module: "mesh-recovery", Scope: "mesh"}) + c.assignments["n2"] = p + c.mu.Unlock() + if placed, read := c.RecoveryPlaced(); !placed || !read { + t.Fatalf("placed %v read %v, want the holder on n2", placed, read) + } + }) } // A-487a (A-461e): a trunk build the catalog gave none of is not asked again at every pass: it waits laterAgain's step for the module diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 8ff12e5a..e67f1293 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -287,6 +287,9 @@ type Composer struct { passing atomic.Bool // early is true while the early pass of a start runs (design 47 G §683, A-117). early atomic.Bool + // assignmentsRead is set once the watch of the seat's assignments has caught up: from then the placements this composer holds + // are the mesh's (RecoveryPlaced, design 47 H, H-A-177). + assignmentsRead atomic.Bool // last is how the last pass went, under mu. last Pass // watching says the composer's loop runs (Watch), so an ask of a pass can be answered. @@ -656,6 +659,9 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er dirty = true // a secret changed: its new generation is asked of the vault at the compose case t.u.Caught: caught[t.in] = true + if ins[t.in].bucket == buckets.Assignments { + c.assignmentsRead.Store(true) + } if ins[t.in].bucket == buckets.Wants { c.endKeptWants() // the wants the seat says now are the ones composed from (§683) }