Design 47 H1: the node-engine moves a data-holding module without a restore point where no holder of the recovery seat is placed (H-A-177, H-A-178) #241

Merged
mesh-tools merged 2 commits from fix/47-H1-engine-restore-point into main 2026-11-20 18:16:17 +00:00
10 changed files with 192 additions and 1 deletions
+8
View File
@@ -714,6 +714,14 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
apply.UseEngineModule(composer.EngineModule) // the guard of the engine's own trees fixes no name (§527)
env.Composed, env.SelfComposed, env.Policy, env.ComposedToday = composer, true, composer.Policy, composer.ComposedToday
env.Recovery = recoveryEnv(session, composer)
// A restore point is asked where the assigner's placements name a holder of mesh-recovery; where none is placed on the
// mesh the module moves without one and says it as the finding restore-point-missing (design 47 H, H-A-177, H §456).
env.Recovery.Placed = composer.RecoveryPlaced
env.Recovery.PointMissing = func(ctx context.Context, module, words string, missing bool) {
if err := seatwait.SayRestorePointMissing(ctx, findings, node, findings.observer(), module, words, missing, time.Now); err != nil {
say("the finding that " + module + " moved without its restore point cannot be kept: " + err.Error())
}
}
env.WhyUnmade, env.Waiting, env.RunningManifest = composer.WhyUnmade, composer.Waiting, composer.RunningManifest
keptFetchFn := keptArtifactFetch(composer.KeptBuildOf, session.AskSeatOn, session.KeptObject)
keptFetch.Store(&keptFetchFn)
+1 -1
View File
@@ -125,7 +125,7 @@ func newFindingsOfTheNode(session *link.Session, node, module string, say func(s
Module: module,
Node: node,
State: link.FindingsState{Session: session},
Kinds: append([]string{seatwait.Kind, seatwait.HeldKind, seatwait.EntryNotFoundKind, seatwait.EntryUnansweredKind}, seatwait.NodeKinds...),
Kinds: append([]string{seatwait.Kind, seatwait.HeldKind, seatwait.EntryNotFoundKind, seatwait.EntryUnansweredKind, seatwait.RestorePointMissingKind}, seatwait.NodeKinds...),
Asked: condseat.SubjectFindingsAsked,
Logf: func(format string, args ...any) { say(fmt.Sprintf(format, args...)) },
Publish: session.PublishEvent,
+29
View File
@@ -3,6 +3,7 @@ package agent
import (
"context"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
@@ -51,6 +52,13 @@ type Recovery struct {
Bound func(ctx context.Context, what, id, module string, since time.Time, d time.Duration) (expired <-chan struct{}, end func(), err error)
// CatchUp calls the module's tool on this node through the tool runner (design 47 G A-108).
CatchUp func(ctx context.Context, module string, c meshrecovery.CatchUp) error
// Placed says whether the assigner's placements name a holder of the seat mesh-recovery anywhere on the mesh, and whether
// the placements are read yet (design 47 H, H-A-177). Nil, or placements not yet read, reads as a holder placed: the point is
// asked as before.
Placed func() (placed, read bool)
// PointMissing raises (missing) the finding restore-point-missing of a module moved without its restore point, with the words
// naming what moved so, or clears it at a take-in that took its point (H-A-177). Nil raises none.
PointMissing func(ctx context.Context, module, words string, missing bool)
// WaitMinutes is the bound in minutes, the config key recovery-wait-minutes; 0 reads as RecoveryWaitMinutes.
WaitMinutes int
// Now is the clock; nil reads the machine's.
@@ -268,6 +276,24 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com
a.writeHave(ctx, Have{Module: module, FP: prev.Fp, Composed: prev.Composed, Phase: string(nodeengineclient.PhaseHeld), Detail: words,
Wait: &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepCompose, Reason: words, Since: since, Words: words}})
}
// Where the assigner's placements name no holder of the seat mesh-recovery anywhere on the mesh, no restore point can be taken:
// the module moves without one, said as the finding restore-point-missing, never a silent skip and never a hold for good; a
// holder placed that does not answer holds the move as before (design 47 H, H-A-177, H §456).
if len(take) > 0 && r.Placed != nil {
if placed, read := r.Placed(); read && !placed {
var what []string
for _, m := range take {
what = append(what, fmt.Sprintf("%s (form %d)", m.Name, m.Form))
}
words := fmt.Sprintf("its kept data %s moved to %s with no restore point taken: no holder of the seat %s is placed on the mesh",
strings.Join(what, ", "), fp, meshrecovery.Seat)
a.env.Say(fmt.Sprintf("%s moves without its restore point: %s", module, words))
if r.PointMissing != nil {
r.PointMissing(ctx, module, words, true)
}
take = nil
}
}
backing := Have{Module: module, FP: fp, Composed: dig, Phase: string(nodeengineclient.PhaseBackingUp)}
for _, m := range take {
why := fmt.Sprintf("form change of %s's %s to %s", module, m.Name, fp)
@@ -281,6 +307,9 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com
return true, nil
}
}
if len(take) > 0 && r.PointMissing != nil {
r.PointMissing(ctx, module, "", false) // a take-in with its points clears a move without one (H-A-177)
}
if len(restore) == 0 {
return false, nil
}
+53
View File
@@ -31,6 +31,9 @@ type recoveryFake struct {
expired chan struct{}
unknown error // FormChanges answers it
waited int // times the agent told the composer it waits for a composition
// noHolder is the assigner's placements naming no holder of mesh-recovery, unread their not being read yet (H-A-177).
noHolder, unread bool
missing []string // the finding restore-point-missing raised ("raise: <words>") or cleared ("clear")
}
type fakeAnswer struct {
@@ -86,6 +89,16 @@ func (f *recoveryFake) env() *Recovery {
f.catchUp = append(f.catchUp, c)
return nil
},
Placed: func() (bool, bool) { return !f.noHolder, !f.unread },
PointMissing: func(_ context.Context, module, words string, missing bool) {
f.mu.Lock()
defer f.mu.Unlock()
if missing {
f.missing = append(f.missing, "raise "+module+": "+words)
} else {
f.missing = append(f.missing, "clear "+module)
}
},
Now: time.Now,
}
}
@@ -167,8 +180,48 @@ func testAMoveAcrossAFormChange(t *testing.T) {
if f.asks[0] != want {
t.Fatalf("asked %s, want %s", f.asks[0], want)
}
if !slices.Equal(f.missing, []string{"clear db"}) {
t.Fatalf("a take-in with its point did not clear restore-point-missing: %v", f.missing)
}
})
// Design 47 H, H-A-177, H §456: the point is asked where the placements name a holder of mesh-recovery (and while they are not
// read yet); where none is placed on the mesh the module moves without one, raising restore-point-missing; a holder placed that
// does not answer holds the move.
for _, row := range []struct {
name string
noHolder, unread bool
calls []string
moved bool
missing string
}{
{name: "no holder of mesh-recovery placed on the mesh moves the module without its point and raises restore-point-missing",
noHolder: true, calls: []string{"apply"}, moved: true, missing: "raise db: its kept data cache (form 1) moved to " + fp("b") + " with no restore point taken: no holder of the seat mesh-recovery is placed on the mesh"},
{name: "a holder placed that does not answer holds the move and raises nothing", calls: []string{"snapshot-kept"}},
{name: "placements not read yet ask the point as before", noHolder: true, unread: true, calls: []string{"snapshot-kept"}},
} {
t.Run(row.name, func(t *testing.T) {
f := &recoveryFake{changes: up, noHolder: row.noHolder, unread: row.unread}
settle := heldHave
if row.moved {
settle = applied
}
b := moveDB(t, f, "a", "b", settle)
if got := f.seen(); !slices.Equal(got, row.calls) {
t.Fatalf("calls %v, want %v", got, row.calls)
}
f.mu.Lock()
missing := slices.Clone(f.missing)
f.mu.Unlock()
if row.missing == "" && len(missing) != 0 || row.missing != "" && !slices.Equal(missing, []string{row.missing}) {
t.Fatalf("restore-point-missing %v, want %q", missing, row.missing)
}
if h := b.haves.last("db"); row.moved && h.Phase == "held" || !row.moved && h.FP != fp("a") {
t.Fatalf("the have %+v", h)
}
})
}
t.Run("busy waits for the other act's ending and asks again, and a stopped act is asked again; another act's failure wakes nothing", func(t *testing.T) {
f := &recoveryFake{changes: up, answers: []fakeAnswer{
{busy: "x1", then: []Ending{failed("zz", "cap"), done("x1")}},
+35
View File
@@ -0,0 +1,35 @@
package seatwait
import (
"context"
"fmt"
"time"
condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
)
// RestorePointMissingKind is the finding the node-engine writes when it moves a module across a form change without its restore
// point, because the assigner's placements name no holder of the seat mesh-recovery anywhere on the mesh (design 47 H, H-A-177,
// H §456): a warning, scope the machine, its id `<module>.<node>` (H-A-178), as a held module's is. It clears at the module's
// next take-in with a point.
const RestorePointMissingKind = "restore-point-missing"
// RestorePointMissingID is the finding's id for a module on a node: findings are keyed by scope, id and kind across the mesh, so
// two machines never clear each other's (H-A-178).
func RestorePointMissingID(module, node string) string { return module + "." + node }
// SayRestorePointMissing writes the finding for module when missing, with the words naming what moved without a point, and clears
// it when no longer: a take-in of the module took its point. A write or a clear that fails is returned for the caller to say.
func SayRestorePointMissing(ctx context.Context, w Writer, node, observer, module, words string, missing bool, now func() time.Time) error {
id := RestorePointMissingID(module, node)
if !missing {
return w.Clear(ctx, scopeMachine, id, RestorePointMissingKind, "a take-in of the module took its restore point")
}
return w.Write(ctx, condseat.Finding{Observer: observer, Kind: RestorePointMissingKind, Scope: scopeMachine, ID: id,
Severity: condseat.FindingSeverityWarning, Source: RestorePointMissingKind,
Subject: condseat.Subject{Scope: scopeMachine, ID: id, Machine: node},
Summary: fmt.Sprintf("%s moved on %s without its restore point: %s", module, node, words),
Headline: fmt.Sprintf("%s moved without a restore point on %s", module, node),
Explanation: "No holder of the seat mesh-recovery is placed on the mesh, so the node-engine moved the module across a change of its kept data's form without taking a restore point first: that move cannot be reverted to the data as it was. The finding clears at the module's next take-in that takes its point.",
At: now()})
}
+14
View File
@@ -241,6 +241,20 @@ func caseAnEntryNotFoundIsAFindingOfTheFingerprint(t *testing.T) {
if err := SayEntryUnanswered(context.Background(), u, "n1", "mesh-host", fp, 0, false, now); err != nil || len(u.cleared) != 1 || u.cleared[0].Kind != EntryUnansweredKind || u.cleared[0].ID != u.put[0].ID {
t.Fatalf("cleared %+v, %v", u.cleared, err)
}
// And design 47 H, H-A-177: a module moved without its restore point is the finding restore-point-missing, a warning of scope
// machine with `<module>.<node>` as its id (H-A-178), on the node's machine, cleared by the same.
r := &fakeFindings{}
if err := SayRestorePointMissing(context.Background(), r, "n1", "mesh-host", "db", "cache (form 1)", true, now); err != nil {
t.Fatal(err)
}
if len(r.put) != 1 || RestorePointMissingKind != "restore-point-missing" || r.put[0].Kind != RestorePointMissingKind || r.put[0].Scope != "machine" ||
r.put[0].ID != "db.n1" || r.put[0].Severity != condseat.FindingSeverityWarning || r.put[0].Subject.Machine != "n1" || !strings.Contains(r.put[0].Summary, "cache (form 1)") {
t.Fatalf("written %+v", r.put)
}
if err := SayRestorePointMissing(context.Background(), r, "n1", "mesh-host", "db", "", false, now); err != nil || len(r.cleared) != 1 ||
r.cleared[0].Kind != RestorePointMissingKind || r.cleared[0].ID != "db.n1" {
t.Fatalf("cleared %+v, %v", r.cleared, err)
}
}
// ADR 0636 §1, the node's finding of a seat list unread.
+20
View File
@@ -15,6 +15,7 @@ import (
"sync"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
@@ -577,6 +578,25 @@ func (c *Composer) needsAddressOf(node, module string) bool {
return false
}
// RecoveryPlaced says whether the assigner's placements this composer holds name a holder of the seat mesh-recovery on any node of
// the mesh, and whether they are read yet: the watch of the seat's assignments has caught up (design 47 H, H-A-177, H §456).
func (c *Composer) RecoveryPlaced() (placed, read bool) {
read = c.assignmentsRead.Load()
c.mu.Lock()
defer c.mu.Unlock()
for _, p := range c.assignments {
for _, h := range p.Holds {
if h.Claim == meshrecovery.Seat {
return true, read
}
if s, ok := catalogue.SeatNamed(h.Claim); ok && s.Name == meshrecovery.Seat {
return true, read
}
}
}
return false, read
}
// askTrunkBuilds asks, for each module another node runs at a build with no entry and whose placement names no build, the trunk
// build by the seat's verb `module`, once per running build (A-461b). Called before the lock is taken: the verb is a call.
func (c *Composer) askTrunkBuilds() (wait func()) {
+19
View File
@@ -666,6 +666,25 @@ func TestOnlyWhatTheNodeNeedsOfAnotherNodesPlacementIsAsked(t *testing.T) {
if len(asked) != 2 || asked["distribution"] != 1 || asked["provider"] != 1 {
t.Errorf("the trunk builds asked are %v, want the store's holder and the bound provider alone", asked)
}
// Design 47 H, H-A-177: the same placements say whether a holder of the seat mesh-recovery is placed on any node, and whether
// they are read yet.
t.Run("a holder of mesh-recovery is placed where any node's placement holds the seat (H-A-177)", func(t *testing.T) {
if placed, read := c.RecoveryPlaced(); placed || read {
t.Fatalf("placed %v read %v, want neither before the watch caught up and with no holder", placed, read)
}
c.assignmentsRead.Store(true)
if placed, read := c.RecoveryPlaced(); placed || !read {
t.Fatalf("placed %v read %v, want no holder read", placed, read)
}
c.mu.Lock()
p := c.assignments["n2"]
p.Holds = append(p.Holds, meshmoduleassigner.Holding{Claim: "mesh-recovery", Module: "mesh-recovery", Scope: "mesh"})
c.assignments["n2"] = p
c.mu.Unlock()
if placed, read := c.RecoveryPlaced(); !placed || !read {
t.Fatalf("placed %v read %v, want the holder on n2", placed, read)
}
})
}
// A-487a (A-461e): a trunk build the catalog gave none of is not asked again at every pass: it waits laterAgain's step for the module
+6
View File
@@ -287,6 +287,9 @@ type Composer struct {
passing atomic.Bool
// early is true while the early pass of a start runs (design 47 G §683, A-117).
early atomic.Bool
// assignmentsRead is set once the watch of the seat's assignments has caught up: from then the placements this composer holds
// are the mesh's (RecoveryPlaced, design 47 H, H-A-177).
assignmentsRead atomic.Bool
// last is how the last pass went, under mu.
last Pass
// watching says the composer's loop runs (Watch), so an ask of a pass can be answered.
@@ -656,6 +659,9 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er
dirty = true // a secret changed: its new generation is asked of the vault at the compose
case t.u.Caught:
caught[t.in] = true
if ins[t.in].bucket == buckets.Assignments {
c.assignmentsRead.Store(true)
}
if ins[t.in].bucket == buckets.Wants {
c.endKeptWants() // the wants the seat says now are the ones composed from (§683)
}
+7
View File
@@ -286,6 +286,10 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
c.holdEntry(t, "oldie", oldieFp, oldie)
b.put(t, buckets.Wants, "want.n1.lemurs", want)
b.put(t, buckets.Wants, "want.n1.node", nodeWant)
// Design 47 H, H-A-177, H-A-178: before the watch of the assignments has caught up, the placements are not read.
if _, read := c.RecoveryPlaced(); read {
t.Error("the placements are read before the watch of the assignments caught up")
}
for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, ResolvedBucket} {
b.send(bucket, agent.Update{Caught: true})
}
@@ -294,6 +298,9 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
for {
u := next(t, out)
if u.Caught {
if placed, read := c.RecoveryPlaced(); !read || placed {
t.Errorf("once the assignments are caught up RecoveryPlaced answers placed %v read %v, want read with no holder", placed, read)
}
break
}
c, err := compositions.Decode(u.Value)