diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 68b29564..a32bda60 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -714,6 +714,14 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus apply.UseEngineModule(composer.EngineModule) // the guard of the engine's own trees fixes no name (§527) env.Composed, env.SelfComposed, env.Policy, env.ComposedToday = composer, true, composer.Policy, composer.ComposedToday env.Recovery = recoveryEnv(session, composer) + // A restore point is asked where the assigner's placements name a holder of mesh-recovery; where none is placed on the + // mesh the module moves without one and says it as the finding restore-point-missing (design 47 H, H-A-177, H §456). + env.Recovery.Placed = composer.RecoveryPlaced + env.Recovery.PointMissing = func(ctx context.Context, module, words string, missing bool) { + if err := seatwait.SayRestorePointMissing(ctx, findings, node, findings.observer(), module, words, missing, time.Now); err != nil { + say("the finding that " + module + " moved without its restore point cannot be kept: " + err.Error()) + } + } env.WhyUnmade, env.Waiting, env.RunningManifest = composer.WhyUnmade, composer.Waiting, composer.RunningManifest keptFetchFn := keptArtifactFetch(composer.KeptBuildOf, session.AskSeatOn, session.KeptObject) keptFetch.Store(&keptFetchFn) diff --git a/cmd/mesh-host/seatlist.go b/cmd/mesh-host/seatlist.go index 15b0fd0c..14b07135 100644 --- a/cmd/mesh-host/seatlist.go +++ b/cmd/mesh-host/seatlist.go @@ -125,7 +125,7 @@ func newFindingsOfTheNode(session *link.Session, node, module string, say func(s Module: module, Node: node, State: link.FindingsState{Session: session}, - Kinds: append([]string{seatwait.Kind, seatwait.HeldKind, seatwait.EntryNotFoundKind, seatwait.EntryUnansweredKind}, seatwait.NodeKinds...), + Kinds: append([]string{seatwait.Kind, seatwait.HeldKind, seatwait.EntryNotFoundKind, seatwait.EntryUnansweredKind, seatwait.RestorePointMissingKind}, seatwait.NodeKinds...), Asked: condseat.SubjectFindingsAsked, Logf: func(format string, args ...any) { say(fmt.Sprintf(format, args...)) }, Publish: session.PublishEvent, diff --git a/internal/agent/kept.go b/internal/agent/kept.go index 185e9a0d..f45f315c 100644 --- a/internal/agent/kept.go +++ b/internal/agent/kept.go @@ -3,6 +3,7 @@ package agent import ( "context" "fmt" + "strings" "time" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" @@ -51,6 +52,13 @@ type Recovery struct { Bound func(ctx context.Context, what, id, module string, since time.Time, d time.Duration) (expired <-chan struct{}, end func(), err error) // CatchUp calls the module's tool on this node through the tool runner (design 47 G A-108). CatchUp func(ctx context.Context, module string, c meshrecovery.CatchUp) error + // Placed says whether the assigner's placements name a holder of the seat mesh-recovery anywhere on the mesh, and whether + // the placements are read yet (design 47 H, H-A-177). Nil, or placements not yet read, reads as a holder placed: the point is + // asked as before. + Placed func() (placed, read bool) + // PointMissing raises (missing) the finding restore-point-missing of a module moved without its restore point, with the words + // naming what moved so, or clears it at a take-in that took its point (H-A-177). Nil raises none. + PointMissing func(ctx context.Context, module, words string, missing bool) // WaitMinutes is the bound in minutes, the config key recovery-wait-minutes; 0 reads as RecoveryWaitMinutes. WaitMinutes int // Now is the clock; nil reads the machine's. @@ -268,6 +276,24 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com a.writeHave(ctx, Have{Module: module, FP: prev.Fp, Composed: prev.Composed, Phase: string(nodeengineclient.PhaseHeld), Detail: words, Wait: &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepCompose, Reason: words, Since: since, Words: words}}) } + // Where the assigner's placements name no holder of the seat mesh-recovery anywhere on the mesh, no restore point can be taken: + // the module moves without one, said as the finding restore-point-missing, never a silent skip and never a hold for good; a + // holder placed that does not answer holds the move as before (design 47 H, H-A-177, H §456). + if len(take) > 0 && r.Placed != nil { + if placed, read := r.Placed(); read && !placed { + var what []string + for _, m := range take { + what = append(what, fmt.Sprintf("%s (form %d)", m.Name, m.Form)) + } + words := fmt.Sprintf("its kept data %s moved to %s with no restore point taken: no holder of the seat %s is placed on the mesh", + strings.Join(what, ", "), fp, meshrecovery.Seat) + a.env.Say(fmt.Sprintf("%s moves without its restore point: %s", module, words)) + if r.PointMissing != nil { + r.PointMissing(ctx, module, words, true) + } + take = nil + } + } backing := Have{Module: module, FP: fp, Composed: dig, Phase: string(nodeengineclient.PhaseBackingUp)} for _, m := range take { why := fmt.Sprintf("form change of %s's %s to %s", module, m.Name, fp) @@ -281,6 +307,9 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com return true, nil } } + if len(take) > 0 && r.PointMissing != nil { + r.PointMissing(ctx, module, "", false) // a take-in with its points clears a move without one (H-A-177) + } if len(restore) == 0 { return false, nil } diff --git a/internal/agent/kept_test.go b/internal/agent/kept_test.go index 3fab81eb..7747b252 100644 --- a/internal/agent/kept_test.go +++ b/internal/agent/kept_test.go @@ -31,6 +31,9 @@ type recoveryFake struct { expired chan struct{} unknown error // FormChanges answers it waited int // times the agent told the composer it waits for a composition + // noHolder is the assigner's placements naming no holder of mesh-recovery, unread their not being read yet (H-A-177). + noHolder, unread bool + missing []string // the finding restore-point-missing raised ("raise: ") or cleared ("clear") } type fakeAnswer struct { @@ -86,6 +89,16 @@ func (f *recoveryFake) env() *Recovery { f.catchUp = append(f.catchUp, c) return nil }, + Placed: func() (bool, bool) { return !f.noHolder, !f.unread }, + PointMissing: func(_ context.Context, module, words string, missing bool) { + f.mu.Lock() + defer f.mu.Unlock() + if missing { + f.missing = append(f.missing, "raise "+module+": "+words) + } else { + f.missing = append(f.missing, "clear "+module) + } + }, Now: time.Now, } } @@ -167,8 +180,48 @@ func testAMoveAcrossAFormChange(t *testing.T) { if f.asks[0] != want { t.Fatalf("asked %s, want %s", f.asks[0], want) } + if !slices.Equal(f.missing, []string{"clear db"}) { + t.Fatalf("a take-in with its point did not clear restore-point-missing: %v", f.missing) + } }) + // Design 47 H, H-A-177, H §456: the point is asked where the placements name a holder of mesh-recovery (and while they are not + // read yet); where none is placed on the mesh the module moves without one, raising restore-point-missing; a holder placed that + // does not answer holds the move. + for _, row := range []struct { + name string + noHolder, unread bool + calls []string + moved bool + missing string + }{ + {name: "no holder of mesh-recovery placed on the mesh moves the module without its point and raises restore-point-missing", + noHolder: true, calls: []string{"apply"}, moved: true, missing: "raise db: its kept data cache (form 1) moved to " + fp("b") + " with no restore point taken: no holder of the seat mesh-recovery is placed on the mesh"}, + {name: "a holder placed that does not answer holds the move and raises nothing", calls: []string{"snapshot-kept"}}, + {name: "placements not read yet ask the point as before", noHolder: true, unread: true, calls: []string{"snapshot-kept"}}, + } { + t.Run(row.name, func(t *testing.T) { + f := &recoveryFake{changes: up, noHolder: row.noHolder, unread: row.unread} + settle := heldHave + if row.moved { + settle = applied + } + b := moveDB(t, f, "a", "b", settle) + if got := f.seen(); !slices.Equal(got, row.calls) { + t.Fatalf("calls %v, want %v", got, row.calls) + } + f.mu.Lock() + missing := slices.Clone(f.missing) + f.mu.Unlock() + if row.missing == "" && len(missing) != 0 || row.missing != "" && !slices.Equal(missing, []string{row.missing}) { + t.Fatalf("restore-point-missing %v, want %q", missing, row.missing) + } + if h := b.haves.last("db"); row.moved && h.Phase == "held" || !row.moved && h.FP != fp("a") { + t.Fatalf("the have %+v", h) + } + }) + } + t.Run("busy waits for the other act's ending and asks again, and a stopped act is asked again; another act's failure wakes nothing", func(t *testing.T) { f := &recoveryFake{changes: up, answers: []fakeAnswer{ {busy: "x1", then: []Ending{failed("zz", "cap"), done("x1")}}, diff --git a/internal/seatwait/restorepoint.go b/internal/seatwait/restorepoint.go new file mode 100644 index 00000000..2dc86df2 --- /dev/null +++ b/internal/seatwait/restorepoint.go @@ -0,0 +1,29 @@ +package seatwait + +import ( + "context" + "fmt" + "time" + + condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" +) + +// RestorePointMissingKind is the finding the node-engine writes when it moves a module across a form change without its restore +// point, because the assigner's placements name no holder of the seat mesh-recovery anywhere on the mesh (design 47 H, H-A-177, +// H §456): a warning, scope the machine, its id the module. It clears at the module's next take-in with a point. +const RestorePointMissingKind = "restore-point-missing" + +// SayRestorePointMissing writes the finding for module when missing, with the words naming what moved without a point, and clears +// it when no longer: a take-in of the module took its point. A write or a clear that fails is returned for the caller to say. +func SayRestorePointMissing(ctx context.Context, w Writer, node, observer, module, words string, missing bool, now func() time.Time) error { + if !missing { + return w.Clear(ctx, scopeMachine, module, RestorePointMissingKind, "a take-in of the module took its restore point") + } + return w.Write(ctx, condseat.Finding{Observer: observer, Kind: RestorePointMissingKind, Scope: scopeMachine, ID: module, + Severity: condseat.FindingSeverityWarning, Source: RestorePointMissingKind, + Subject: condseat.Subject{Scope: scopeMachine, ID: module, Machine: node}, + Summary: fmt.Sprintf("%s moved on %s without its restore point: %s", module, node, words), + Headline: fmt.Sprintf("%s moved without a restore point on %s", module, node), + Explanation: "No holder of the seat mesh-recovery is placed on the mesh, so the node-engine moved the module across a change of its kept data's form without taking a restore point first: that move cannot be reverted to the data as it was. The finding clears at the module's next take-in that takes its point.", + At: now()}) +} diff --git a/internal/seatwait/seatwait_test.go b/internal/seatwait/seatwait_test.go index 128e1442..a742726b 100644 --- a/internal/seatwait/seatwait_test.go +++ b/internal/seatwait/seatwait_test.go @@ -241,6 +241,20 @@ func caseAnEntryNotFoundIsAFindingOfTheFingerprint(t *testing.T) { if err := SayEntryUnanswered(context.Background(), u, "n1", "mesh-host", fp, 0, false, now); err != nil || len(u.cleared) != 1 || u.cleared[0].Kind != EntryUnansweredKind || u.cleared[0].ID != u.put[0].ID { t.Fatalf("cleared %+v, %v", u.cleared, err) } + // And design 47 H, H-A-177: a module moved without its restore point is the finding restore-point-missing, a warning of scope + // machine with the module as its id, on the node's machine, cleared by the same. + r := &fakeFindings{} + if err := SayRestorePointMissing(context.Background(), r, "n1", "mesh-host", "db", "cache (form 1)", true, now); err != nil { + t.Fatal(err) + } + if len(r.put) != 1 || RestorePointMissingKind != "restore-point-missing" || r.put[0].Kind != RestorePointMissingKind || r.put[0].Scope != "machine" || + r.put[0].ID != "db" || r.put[0].Severity != condseat.FindingSeverityWarning || r.put[0].Subject.Machine != "n1" || !strings.Contains(r.put[0].Summary, "cache (form 1)") { + t.Fatalf("written %+v", r.put) + } + if err := SayRestorePointMissing(context.Background(), r, "n1", "mesh-host", "db", "", false, now); err != nil || len(r.cleared) != 1 || + r.cleared[0].Kind != RestorePointMissingKind || r.cleared[0].ID != "db" { + t.Fatalf("cleared %+v, %v", r.cleared, err) + } } // ADR 0636 §1, the node's finding of a seat list unread. diff --git a/internal/selfcompose/entries.go b/internal/selfcompose/entries.go index 5daf0032..29a7c351 100644 --- a/internal/selfcompose/entries.go +++ b/internal/selfcompose/entries.go @@ -15,6 +15,7 @@ import ( "sync" "time" + "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -577,6 +578,25 @@ func (c *Composer) needsAddressOf(node, module string) bool { return false } +// RecoveryPlaced says whether the assigner's placements this composer holds name a holder of the seat mesh-recovery on any node of +// the mesh, and whether they are read yet: the watch of the seat's assignments has caught up (design 47 H, H-A-177, H §456). +func (c *Composer) RecoveryPlaced() (placed, read bool) { + read = c.assignmentsRead.Load() + c.mu.Lock() + defer c.mu.Unlock() + for _, p := range c.assignments { + for _, h := range p.Holds { + if h.Claim == meshrecovery.Seat { + return true, read + } + if s, ok := catalogue.SeatNamed(h.Claim); ok && s.Name == meshrecovery.Seat { + return true, read + } + } + } + return false, read +} + // askTrunkBuilds asks, for each module another node runs at a build with no entry and whose placement names no build, the trunk // build by the seat's verb `module`, once per running build (A-461b). Called before the lock is taken: the verb is a call. func (c *Composer) askTrunkBuilds() (wait func()) { diff --git a/internal/selfcompose/entries_test.go b/internal/selfcompose/entries_test.go index ae9aec8c..b0d7b36f 100644 --- a/internal/selfcompose/entries_test.go +++ b/internal/selfcompose/entries_test.go @@ -666,6 +666,25 @@ func TestOnlyWhatTheNodeNeedsOfAnotherNodesPlacementIsAsked(t *testing.T) { if len(asked) != 2 || asked["distribution"] != 1 || asked["provider"] != 1 { t.Errorf("the trunk builds asked are %v, want the store's holder and the bound provider alone", asked) } + // Design 47 H, H-A-177: the same placements say whether a holder of the seat mesh-recovery is placed on any node, and whether + // they are read yet. + t.Run("a holder of mesh-recovery is placed where any node's placement holds the seat (H-A-177)", func(t *testing.T) { + if placed, read := c.RecoveryPlaced(); placed || read { + t.Fatalf("placed %v read %v, want neither before the watch caught up and with no holder", placed, read) + } + c.assignmentsRead.Store(true) + if placed, read := c.RecoveryPlaced(); placed || !read { + t.Fatalf("placed %v read %v, want no holder read", placed, read) + } + c.mu.Lock() + p := c.assignments["n2"] + p.Holds = append(p.Holds, meshmoduleassigner.Holding{Claim: "mesh-recovery", Module: "mesh-recovery", Scope: "mesh"}) + c.assignments["n2"] = p + c.mu.Unlock() + if placed, read := c.RecoveryPlaced(); !placed || !read { + t.Fatalf("placed %v read %v, want the holder on n2", placed, read) + } + }) } // A-487a (A-461e): a trunk build the catalog gave none of is not asked again at every pass: it waits laterAgain's step for the module diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 8ff12e5a..e67f1293 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -287,6 +287,9 @@ type Composer struct { passing atomic.Bool // early is true while the early pass of a start runs (design 47 G §683, A-117). early atomic.Bool + // assignmentsRead is set once the watch of the seat's assignments has caught up: from then the placements this composer holds + // are the mesh's (RecoveryPlaced, design 47 H, H-A-177). + assignmentsRead atomic.Bool // last is how the last pass went, under mu. last Pass // watching says the composer's loop runs (Watch), so an ask of a pass can be answered. @@ -656,6 +659,9 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er dirty = true // a secret changed: its new generation is asked of the vault at the compose case t.u.Caught: caught[t.in] = true + if ins[t.in].bucket == buckets.Assignments { + c.assignmentsRead.Store(true) + } if ins[t.in].bucket == buckets.Wants { c.endKeptWants() // the wants the seat says now are the ones composed from (§683) }