Merge branch 'feat/47-G-recover-host' into feat/47-G-join8

# Conflicts:
#	internal/selfcompose/entries.go
#	module.json
This commit is contained in:
2026-11-12 00:27:57 +01:00
27 changed files with 1716 additions and 11 deletions
+71
View File
@@ -0,0 +1,71 @@
package main
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildimage"
"git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildnode"
"github.com/novox/mesh-host/internal/apply"
)
// The artifact store's holder takes its own new build from the build node that kept it (design 47): its done build names the
// node and the digest, the node's build seat answers where the bytes are as an object on the bus, and the node-engine
// reads them and checks them against the digest. Never another module's files: the seat's verb is the way.
// keptAskWithin bounds one ask of a build seat; keptObjectWithin one read of the object.
var (
keptAskWithin = 30 * time.Second
keptObjectWithin = 10 * time.Minute
)
// keptBuildSeats are the three build seats, each of which serves the verb kept-artifact: the node that kept the build
// holds the seat of the language it was built in, and the others do not answer.
var keptBuildSeats = []string{nodebuildgo.Seat, nodebuildnode.Seat, nodebuildimage.Seat}
// keptArtifactFetch is the apply's fetch of what a module's done build kept on a build node.
func keptArtifactFetch(buildOf func(context.Context, string, string) (string, string, bool, error),
ask func(ctx context.Context, node, seat, verb string, args any, within time.Duration) (json.RawMessage, error),
object func(ctx context.Context, store, name string, limit int64) ([]byte, error)) apply.KeptFetch {
return func(ctx context.Context, module, digest string) ([]byte, bool, error) {
node, fp, kept, err := buildOf(ctx, module, digest)
if err != nil || !kept {
return nil, false, err
}
hex := strings.TrimPrefix(digest, "sha256:")
var notHeld []string
for _, seat := range keptBuildSeats {
raw, err := ask(ctx, node, seat, nodebuildgo.VerbKeptArtifact,
nodebuildgo.KeptArtifactInput{Module: module, Fp: fp, Digest: hex}, keptAskWithin)
if err != nil {
notHeld = append(notHeld, seat+": "+err.Error())
continue
}
var got nodebuildgo.KeptArtifactAnswer
if json.Unmarshal(raw, &got) != nil {
return nil, true, fmt.Errorf("%s of %s answered with something that is not an answer of kept-artifact", seat, node)
}
switch {
case got.Error != "":
return nil, true, fmt.Errorf("%s of %s could not give the kept build of %s: %s", seat, node, module, got.Error)
case got.Refused != "":
return nil, true, fmt.Errorf("%s of %s refuses the kept build of %s: %s %s", seat, node, module, got.Refused, got.Words)
case got.Store == "" || got.Object == "" || got.Digest != hex:
return nil, true, fmt.Errorf("%s of %s answered for %s without the store and the object of the digest %s", seat, node, module, hex)
}
octx, cancel := context.WithTimeout(ctx, keptObjectWithin)
body, err := object(octx, got.Store, got.Object, got.Size+1)
cancel()
if err != nil {
return nil, true, fmt.Errorf("the build of %s kept on %s could not be read from %s: %w", module, node, got.Store, err)
}
return body, true, nil
}
return nil, true, fmt.Errorf("%s keeps the build of %s for the artifact store and none of its build seats answers (%s)",
node, module, strings.Join(notHeld, "; "))
}
}
+73
View File
@@ -0,0 +1,73 @@
package main
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
)
// A module's done build that kept its artifact on a build node is fetched through that node's build seat (design 47).
func TestAKeptArtifactIsFetchedThroughTheBuildSeatOfItsNode(t *testing.T) {
hex := strings.Repeat("a", 64)
stored := func(string) (json.RawMessage, error) {
return json.RawMessage(`{"module":"distribution","fp":"f1","digest":"` + hex + `","size":5,"store":"kept-builds","object":"` + hex + `"}`), nil
}
for _, row := range []struct {
name string
kept bool
answer func(seat string) (json.RawMessage, error)
object error
wantKept bool
wantErr string
wantSeat string
}{
{"a build kept nothing is not asked", false, nil, nil, false, "", ""},
{"the seat of the language answers", true, func(seat string) (json.RawMessage, error) {
if seat != "node-build-node" {
return nil, errors.New("no responder")
}
return stored(seat)
}, nil, true, "", "node-build-node"},
{"no build seat of the node answers", true, func(string) (json.RawMessage, error) { return nil, errors.New("no responder") },
nil, true, "novox keeps the build of distribution", ""},
{"the seat refuses it", true, func(string) (json.RawMessage, error) {
return json.RawMessage(`{"module":"distribution","fp":"f1","refused":"not kept","words":"swept"}`), nil
}, nil, true, "refuses the kept build of distribution: not kept", ""},
{"the object cannot be read", true, stored, errors.New("no object"), true, "could not be read from kept-builds", "node-build-go"},
} {
t.Run(row.name, func(t *testing.T) {
var seats []string
fetch := keptArtifactFetch(
func(context.Context, string, string) (string, string, bool, error) { return "novox", "f1", row.kept, nil },
func(_ context.Context, node, seat, verb string, _ any, _ time.Duration) (json.RawMessage, error) {
if node != "novox" || verb != "kept-artifact" {
t.Errorf("asked %s of %s on %s", verb, seat, node)
}
seats = append(seats, seat)
return row.answer(seat)
},
func(_ context.Context, store, name string, limit int64) ([]byte, error) {
if store != "kept-builds" || name != hex || limit < 5 {
t.Errorf("read %s/%s up to %d", store, name, limit)
}
return []byte("bytes"), row.object
})
body, kept, err := fetch(context.Background(), "distribution", "sha256:"+hex)
if kept != row.wantKept {
t.Fatalf("kept = %v", kept)
}
if row.wantErr == "" && (err != nil || (kept && string(body) != "bytes")) {
t.Fatalf("got %q, %v", body, err)
}
if row.wantErr != "" && (err == nil || !strings.Contains(err.Error(), row.wantErr)) {
t.Fatalf("the error is %v, want %q", err, row.wantErr)
}
if !row.kept && len(seats) > 0 {
t.Errorf("a build that kept nothing asked %v", seats)
}
})
}
}
+38 -3
View File
@@ -28,6 +28,7 @@ import (
"github.com/novox/mesh-host/internal/seatwait"
"github.com/novox/mesh-host/internal/selfcompose"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
// The node agent's wiring (novox/hq ADR 0429 §4, ADR 0539 §9): the buckets it watches, the haves it writes, the backup
@@ -89,7 +90,7 @@ func (h sessionHaves) Read(ctx context.Context, module string) (agent.Have, bool
return agent.Have{}, false
}
return agent.Have{Module: module, FP: got.Fp, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail,
Sealed: got.SealedAt}, true
Sealed: got.SealedAt, Wait: got.Wait}, true
}
// ReadErr is Read telling a have absent from one the bus could not read (agent.HavesReadErr).
@@ -107,7 +108,7 @@ func (h sessionHaves) ReadErr(ctx context.Context, module string) (agent.Have, b
return agent.Have{}, false, err
}
return agent.Have{Module: module, FP: got.Fp, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail,
Sealed: got.SealedAt}, true, nil
Sealed: got.SealedAt, Wait: got.Wait}, true, nil
}
func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
@@ -124,7 +125,7 @@ func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
}
h.session.NoteModule(ctx, link.ModuleFact{Module: x.Module, FP: x.FP, Composed: x.Composed, Phase: phase,
Failed: phase == string(nodeengineclient.PhaseFailed), Detail: shorten(x.Detail), Judgement: x.Judgement, Waits: x.Waits,
BuildAlone: x.BuildAlone, Kept: x.Kept, Sealed: x.Sealed, PullWait: x.PullWait})
BuildAlone: x.BuildAlone, Kept: x.Kept, Sealed: x.Sealed, PullWait: x.PullWait, Wait: x.Wait})
}
func (h sessionHaves) Removed(ctx context.Context, module string) {
@@ -448,6 +449,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
}
began := time.Now()
node := session.Membership.Node
var keptFetch atomic.Pointer[apply.KeptFetch] // the composer's, set once it exists
composedBucket, err := compositions.BucketFor(node)
if err != nil {
cancel()
@@ -550,6 +552,9 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
Backoff: link.Backoff,
Apply: func(ctx context.Context, c compositions.Composed, others []compositions.Composed) agent.Result {
actx, done := applyContext(session.WithLink(ctx))
if kf := keptFetch.Load(); kf != nil {
actx = apply.WithKept(actx, *kf)
}
// While the grants part's list is followed, the controller's list in the bus module's composition is not
// written: the apply keeps the list on the machine (ADR 0547 clause 8, never both).
held, herr := users.Hold(c)
@@ -574,6 +579,9 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
sayFacts(ctx, ag)
return res
},
Stop: func(ctx context.Context, module string) agent.Result {
return stopComposition(ctx, opts, module)
},
Remove: func(ctx context.Context, module string, others []compositions.Composed) agent.Result {
actx, done := applyContext(session.WithLink(ctx))
res := removeComposition(actx, opts, module, others, ag.Placed(module), sched, say)
@@ -680,7 +688,10 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
go followRunning(run, session, composer, say)
apply.UseEngineModule(composer.EngineModule) // the guard of the engine's own trees fixes no name (§527)
env.Composed, env.SelfComposed, env.Policy, env.ComposedToday = composer, true, composer.Policy, composer.ComposedToday
env.Recovery = recoveryEnv(session, composer)
env.WhyUnmade, env.Waiting = composer.WhyUnmade, composer.Waiting
keptFetchFn := keptArtifactFetch(composer.KeptBuildOf, session.AskSeatOn, session.KeptObject)
keptFetch.Store(&keptFetchFn)
env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does
session.NoteComposes(linked, nodeengineclient.ComposesOwn, "it composes its own modules, and applies no composition made elsewhere")
ag = agent.New(env)
@@ -851,6 +862,30 @@ func removeComposition(ctx context.Context, opts options, module string, others
return agent.Result{}
}
// stopComposition stops a module's units and containers from the node's record and removes nothing (design 47 G A-98,
// §297): what the module kept stays, so a revert restores it before the reverted build starts.
func stopComposition(ctx context.Context, opts options, module string) agent.Result {
applying.Lock()
defer applying.Unlock()
unlock, err := store.Lock(opts.state, nil)
if err != nil {
return agent.Result{Failed: true, Detail: shorten(err.Error())}
}
defer unlock()
known, err := store.Load(opts.state)
if err != nil {
return agent.Result{Failed: true, Detail: shorten(err.Error())}
}
built, err := system.For(builtFor)
if err != nil {
return agent.Result{Failed: true, Detail: shorten(err.Error())}
}
if _, err := apply.StopModule(ctx, built, known, module, opts.run()); err != nil {
return agent.Result{Failed: true, Detail: shorten(err.Error())}
}
return agent.Result{}
}
// busyModules are the modules whose last apply left a container running because it said busy.
func busyModules() []string {
busyMu.Lock()
+89
View File
@@ -0,0 +1,89 @@
package main
import (
"context"
"encoding/json"
"os"
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
"github.com/novox/mesh-host/internal/agent"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/selfcompose"
)
// recoveryEnv is what the node agent asks of the recovery module and the manifests to move a module's kept data (design 47 G §296,
// §297, §554, §560, §630, A-77, A-98, A-108): the seat's verbs asked in the node-engine's own name, the endings heard from the
// seat's events, the bound of a wait kept as a key of the state bounds, the catch-up called as the module's tool.
func recoveryEnv(session *link.Session, composer *selfcompose.Composer) *agent.Recovery {
ask := func(ctx context.Context, verb string, in, out any) error {
raw, err := session.AskMeshSeat(ctx, meshrecovery.Seat, verb, in)
if err != nil {
return err
}
return json.Unmarshal(raw, out)
}
return &agent.Recovery{
FormChanges: composer.FormChanges,
SnapshotKept: func(ctx context.Context, in meshrecovery.SnapshotKeptInput) (a meshrecovery.SnapshotKeptAnswer, err error) {
return a, ask(ctx, meshrecovery.VerbSnapshotKept, in, &a)
},
RestoreKept: func(ctx context.Context, in meshrecovery.RestoreKeptInput) (a meshrecovery.RestoreKeptAnswer, err error) {
return a, ask(ctx, meshrecovery.VerbRestoreKept, in, &a)
},
KeptPoints: func(ctx context.Context, in meshrecovery.KeptPointsInput) (a meshrecovery.KeptPointsAnswer, err error) {
return a, ask(ctx, meshrecovery.VerbKeptPoints, in, &a)
},
Hear: func(_ context.Context, kind, module, kept string) (<-chan agent.Ending, func(), error) {
heard, stop, err := session.HearRecoveryEnding(kind, module, kept)
if err != nil {
return nil, nil, err
}
out := make(chan agent.Ending, 64)
done := make(chan struct{})
go func() {
for {
select {
case ev := <-heard:
select {
case out <- agent.Ending(ev):
case <-done:
return
}
case <-done:
return
}
}
}()
return out, func() { stop(); close(done) }, nil
},
Bound: func(ctx context.Context, what, id, module string, since time.Time, d time.Duration) (<-chan struct{}, func(), error) {
return session.KeepBound(ctx, what, id, module, since, d)
},
CatchUp: func(ctx context.Context, module string, c meshrecovery.CatchUp) error {
return session.CallTool(ctx, module, c.Tool, c.Arguments)
},
WaitMinutes: recoveryWaitMinutes(),
}
}
// recoveryWaitFile is the file the manifest declares with the content ${config:recovery-wait-minutes} (design 47 G §633, A-112): the
// node-engine is no bundle under a runtime, so it reads the setting from its file at its start, as the builder does (A-71).
var recoveryWaitFile = "/var/lib/mesh-host/recovery-wait-minutes"
// recoveryWaitMinutes is the bound of the wait for the recovery module, in minutes: the file's number, the design's 90 when the
// file is missing, empty or not a positive number (design 47 G §554, A-112).
func recoveryWaitMinutes() int {
raw, err := os.ReadFile(recoveryWaitFile)
if err != nil {
return agent.RecoveryWaitMinutes
}
n, err := strconv.Atoi(strings.TrimSpace(string(raw)))
if err != nil || n <= 0 {
return agent.RecoveryWaitMinutes
}
return n
}
+142
View File
@@ -0,0 +1,142 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"os"
"path/filepath"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/meshrecovery"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/nodegrant"
)
// Design 47 G §560, §630, A-77, A-92, A-108: the node agent's recovery step over a real bus. Its session asks the seat's verbs, hears
// the last `restored` already said and the next, keeps a wait's bound as a key of the state bounds whose time ends with the bus's
// marker, and calls the catch-up as the module's tool with its arguments. The recovery module is a stand-in that answers the verbs and
// says the events; the grants part composes the node's subjects from the manifest, so the test adds them to the node's grant.
func TestNatsTheRecoveryStepAsksHearsBoundsAndCatchesUpOverTheBus(t *testing.T) {
perms := nodegrant.For(itNode)
perms.Publish = append(perms.Publish, "$JS.API.>", "$KV.mesh-host_bounds.>", "mesh.seat.mesh-recovery.call.*.node~"+itNode,
"mesh.mod.db.tool.rebuild."+itNode)
perms.Subscribe = append(perms.Subscribe, "mesh.seat.mesh-recovery.event.>", "$KV.mesh-host_bounds.>", "_INBOX.>", "$JS.ACK.>", "$JS.FC.>")
b := newItBedAs(t, perms)
ctx := context.Background()
for _, s := range []struct{ stream, subject string }{{meshrecovery.StreamEnded, meshrecovery.SubjectEndedAll},
{meshrecovery.StreamRestored, meshrecovery.SubjectRestoredAll}} {
if _, err := b.js.AddStream(&nats.StreamConfig{Name: s.stream, Subjects: []string{s.subject}, MaxMsgsPerSubject: 1}); err != nil {
t.Fatal(err)
}
}
js, err := jetstream.New(b.conn)
if err != nil {
t.Fatal(err)
}
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: link.BoundsBucket, History: 1, LimitMarkerTTL: time.Second}); err != nil {
t.Fatal(err)
}
say := func(act, outcome string) {
raw, _ := json.Marshal(meshrecovery.RestoredEvent{Act: act, Module: "db", Kept: "cache", Outcome: outcome, At: time.Now().UTC()})
if _, err := b.js.Publish(meshrecovery.SubjectRestored("db", "cache"), raw); err != nil {
t.Fatal(err)
}
}
var restoreAsked meshrecovery.RestoreKeptInput
for verb, answer := range map[string]func(m *nats.Msg) any{
meshrecovery.VerbKeptPoints: func(*nats.Msg) any {
return meshrecovery.KeptPointsAnswer{Points: []meshrecovery.Point{{Module: "db", Kept: "cache", Form: 1, Fingerprint: "f2", Snapshot: "snap-1",
CatchUp: &meshrecovery.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": true}}}}}
},
meshrecovery.VerbRestoreKept: func(m *nats.Msg) any {
_ = json.Unmarshal(m.Data, &restoreAsked)
say("r1", meshrecovery.RestoredEventOutcomeDone)
return meshrecovery.RestoreKeptAnswer{Accepted: "r1"}
},
} {
answer := answer
sub, err := b.conn.Subscribe("mesh.seat.mesh-recovery.call."+verb+".*", func(m *nats.Msg) {
out, _ := json.Marshal(map[string]any{"result": answer(m)})
_ = m.Respond(out)
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
var toolArgs map[string]any
toolSub, err := b.conn.Subscribe(link.ToolSubject("db", "rebuild", itNode), func(m *nats.Msg) {
_ = json.Unmarshal(m.Data, &toolArgs)
_ = m.Respond([]byte(`{"result":{}}`))
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = toolSub.Unsubscribe() })
// A-112: the bound is the number in the file the manifest declares, 90 where it is missing or not a number.
for content, want := range map[string]int{"45\n": 45, "": 90, "soon\n": 90, "0\n": 90} {
recoveryWaitFile = filepath.Join(t.TempDir(), "w")
if err := os.WriteFile(recoveryWaitFile, []byte(content), 0o644); err != nil {
t.Fatal(err)
}
if got := recoveryWaitMinutes(); got != want {
t.Errorf("the file %q gives %d minutes, want %d", content, got, want)
}
}
recoveryWaitFile = filepath.Join(t.TempDir(), "missing")
if got := recoveryWaitMinutes(); got != 90 {
t.Errorf("no file gives %d minutes, want 90", got)
}
say("old", meshrecovery.RestoredEventOutcomeFailed) // said before anyone listens: the last message of the subject
session := b.startSession()
r := recoveryEnv(session, nil)
events, stop, err := r.Hear(ctx, "restored", "db", "cache")
if err != nil {
t.Fatal(err)
}
defer stop()
if first := <-events; first.Act != "old" || first.Outcome != "failed" {
t.Fatalf("the first event heard is %+v, want the last one said before", first)
}
pts, err := r.KeptPoints(ctx, meshrecovery.KeptPointsInput{Module: "db", Kept: "cache"})
if err != nil || len(pts.Points) != 1 {
t.Fatalf("kept-points: %+v, %v", pts, err)
}
got, err := r.RestoreKept(ctx, meshrecovery.RestoreKeptInput{Module: "db", Kept: "cache", Form: 1, Fingerprint: "f2", Snapshot: pts.Points[0].Snapshot, Why: "w"})
if err != nil || got.Accepted != "r1" || restoreAsked.Snapshot != "snap-1" {
t.Fatalf("restore-kept: %+v, %v, asked %+v", got, err, restoreAsked)
}
if ev := <-events; ev.Act != "r1" || ev.Outcome != "done" {
t.Fatalf("the ending heard is %+v", ev)
}
if err := r.CatchUp(ctx, "db", *pts.Points[0].CatchUp); err != nil || toolArgs["all"] != true {
t.Fatalf("catch-up: %v, arguments %v", err, toolArgs)
}
expired, end, err := r.Bound(ctx, "recovery", "db.restored.cache", "db", time.Now(), time.Second)
if err != nil {
t.Fatal(err)
}
defer end()
select {
case <-expired:
case <-time.After(10 * time.Second):
t.Fatal("the bound's end, the bus's marker, was not heard")
}
}
// startSession is a session on the node's own connection, with no node agent run on it.
func (b *itBed) startSession() *link.Session {
b.session = &link.Session{Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second,
Say: b.say, Keys: func() link.RecordKeys { return link.RecordKeys{} }}
b.session.Attach(context.Background(), link.OverNATS{Conn: b.nodeConn, JS: b.nodeJS, Node: itNode})
return b.session
}
+40
View File
@@ -0,0 +1,40 @@
package main
import (
"context"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/store"
)
// The node agent's Stop stops a module's container from the node's record, asks the machine nothing else and leaves the
// record, so the module's kept data and its record survive (design 47 G A-98).
func TestTheNodeAgentsStopStopsAModuleAndRemovesNothing(t *testing.T) {
was := builtFor
builtFor = "arch"
t.Cleanup(func() { builtFor = was })
path := filepath.Join(t.TempDir(), "state.json")
var s store.State
s.Record(store.Applied{ID: "app.box", Type: "container", Target: "app-box"})
s.Record(store.Applied{ID: "other.box", Type: "container", Target: "other-box"})
if err := store.Save(path, s); err != nil {
t.Fatal(err)
}
var asked []string
opts := options{state: path, runner: func(_ context.Context, name string, args ...string) (string, error) {
asked = append(asked, name+" "+strings.Join(args, " "))
return "", nil
}}
if res := stopComposition(context.Background(), opts, "app"); res.Failed {
t.Fatalf("stop failed: %s", res.Detail)
}
if !strings.Contains(strings.Join(asked, "\n"), "stop app-box") || strings.Contains(strings.Join(asked, "\n"), "other-box") ||
strings.Contains(strings.Join(asked, "\n"), " rm ") {
t.Fatalf("the machine was asked %v", asked)
}
if after, err := store.Load(path); err != nil || len(after.Resources) != 2 {
t.Fatalf("the record changed: %v, %v", after.Resources, err)
}
}