From decbce4c54382c5119fc34ae1b81bc1d8b9e5b9f Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:03:13 +0100 Subject: [PATCH 01/59] H-1: the node-engine claims its seat, says its findings on its own stream and hears running MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §364, §387, §199, §368, §411, §438 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- cmd/mesh-host/settled_manifest_test.go | 39 ++++++++++++++++++++++++++ module.json | 18 +++++++++++- packaging/operator_keys_test.go | 4 +-- 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/cmd/mesh-host/settled_manifest_test.go b/cmd/mesh-host/settled_manifest_test.go index 24d4b75e..395819bf 100644 --- a/cmd/mesh-host/settled_manifest_test.go +++ b/cmd/mesh-host/settled_manifest_test.go @@ -26,3 +26,42 @@ func TestTheSettledEventIsTheManifestsConsume(t *testing.T) { t.Errorf("the engine hears %s, its manifest consumes %v", settledEvent, m.Consumes) } } + +// The manifest claims the seat node-engine, says its findings on its own stream and hears the keeper's ask and the +// node-engine seat's running (design 47 G §364, §387, §199). +func TestTheManifestClaimsItsSeatAndSaysItsFindings(t *testing.T) { + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + var m struct { + Version string `json:"version"` + Consumes []string `json:"consumes"` + Claims []struct { + Name string `json:"name"` + Scope string `json:"scope"` + } `json:"claims"` + Streams []struct { + Name string `json:"name"` + Events []string `json:"events"` + } `json:"streams"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + if len(m.Claims) != 1 || m.Claims[0].Name != "node-engine" || m.Claims[0].Scope != "node" { + t.Errorf("claims = %+v, want the node seat node-engine", m.Claims) + } + if len(m.Streams) != 1 || m.Streams[0].Name != "mesh-host-findings" || + !slices.Equal(m.Streams[0].Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) { + t.Errorf("streams = %+v, want mesh-host-findings with the three finding events", m.Streams) + } + for _, c := range []string{"mesh-conditions.findings-asked", "node-engine.running"} { + if !slices.Contains(m.Consumes, c) { + t.Errorf("consumes %v lacks %s", m.Consumes, c) + } + } + if m.Version != "0.1" { + t.Errorf("version = %q, want MAJOR.MINOR 0.1 (§368, §411)", m.Version) + } +} diff --git a/module.json b/module.json index 8c19c884..52f97f17 100644 --- a/module.json +++ b/module.json @@ -1,6 +1,6 @@ { "module": "mesh-host", - "version": "1", + "version": "0.1", "slug": "host", "build": { "artifacts": [ @@ -14,15 +14,31 @@ } ] }, + "claims": [ + { + "name": "node-engine", + "scope": "node" + } + ], "invokes": [ "seat:mesh-catalog.seat-definitions" ], "consumes": [ "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", + "mesh-conditions.findings-asked", "mesh-delivery-view.settled", + "node-engine.running", "restic.backed-up" ], + "streams": [ + { + "name": "mesh-host-findings", + "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], + "max-age-days": 7, + "max-mib": 16 + } + ], "state": [ { "name": "health" diff --git a/packaging/operator_keys_test.go b/packaging/operator_keys_test.go index 1b86b25a..d620eef7 100644 --- a/packaging/operator_keys_test.go +++ b/packaging/operator_keys_test.go @@ -66,8 +66,8 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) { if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) } - // The backup event, the status view's settled, and the two events of the seat that serves the seat list (ADR 0636 §4). - if want := []string{"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-delivery-view.settled", "restic.backed-up"}; !slices.Equal(m.Consumes, want) { + // The backup event, the status view's settled, and the two events of the seat that serves the seat list (ADR 0636 §4), the keeper's ask (design 47 G §387) and the seat's running (§199). + if want := []string{"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-conditions.findings-asked", "mesh-delivery-view.settled", "node-engine.running", "restic.backed-up"}; !slices.Equal(m.Consumes, want) { t.Fatalf("consumes %v, want %v", m.Consumes, want) } // 774: the acts bucket mesh-host_acts is not declared as state for now: a declared state of a module with no account -- 2.54.0 From 730860b274ae33325cae9d7c4ac4d12386986cff Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:03:47 +0100 Subject: [PATCH 02/59] H-2: every service and container is started with MESH_FINGERPRINT, the fingerprint of the want it applies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §201, §205, §239 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- cmd/mesh-host/nodeagent_integration_test.go | 5 ++- internal/compositions/compare_test.go | 46 +++++++++++++++++++++ internal/compositions/composed.go | 36 ++++++++++++++++ 3 files changed, 86 insertions(+), 1 deletion(-) diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index f8da072e..44c753ad 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -746,7 +746,10 @@ func TestNatsTheBusServerIsTakenOverAndReplacedOnlyWhenItsBuildChangesIt(t *test } folded, err := declaration.Parse([]byte(`{"declaration":1,"resources":[` + `{"id":"nats.server-conf","type":"file","path":"` + conf + `","mode":"0644","content":"listen: old\n"},` + - `{"id":"nats.server","type":"container","name":"` + server + `","image":"` + image + `"}]}`)) + `{"id":"nats.server","type":"container","name":"` + server + `","image":"` + image + `",` + + // As the node-engine starts every container since the fingerprint handover (47/G W13.1): the fingerprint of the + // want it applies, here the first want's, so that want changes the server's spec in nothing. + `"env":{"MESH_FINGERPRINT":"` + fpIt("e") + `"}}]}`)) if err != nil { t.Fatal(err) } diff --git a/internal/compositions/compare_test.go b/internal/compositions/compare_test.go index 8a3c1e5d..a4987123 100644 --- a/internal/compositions/compare_test.go +++ b/internal/compositions/compare_test.go @@ -165,4 +165,50 @@ func TestComparingCompositions(t *testing.T) { if c := CompareComposed(map[string]Composed{"k": comp(file("conf", "a"))}, nil); len(c.Added) != 1 || c.Added[0].Module != "nats" || c.Added[0].Composed == "" { t.Errorf("a composition not run is added: %+v", c) } + + // Design 47, *The fingerprint handover* (the composed resources are split here): a daemon or a container of a module + // is started with MESH_FINGERPRINT, the fingerprint of the want it applies, over any value its module declared of that + // name; a file, or a resource of no module, carries none. + split, err := Split("novox", []json.RawMessage{ + json.RawMessage(`{"id":"d","type":"process","module":"nats","run":["x"],"env":{"A":"1","MESH_FINGERPRINT":"stale"}}`), + json.RawMessage(`{"id":"c","type":"container","module":"nats","image":"i"}`), + file("conf", "a"), + json.RawMessage(`{"id":"own","type":"process","run":["x"]}`), + }, map[string]string{"nats": fp}, nil, NodeOwn{}) + if err != nil { + t.Fatal(err) + } + if len(split) != 2 { + t.Fatalf("split into %d compositions, want the module's and the node's own", len(split)) + } + if n := len(split[mustKey(t, "novox", "nats", fp)].Resources); n != 3 { + t.Fatalf("the module's composition holds %d resources, want 3", n) + } + for _, raw := range split[mustKey(t, "novox", "nats", fp)].Resources { + var r struct { + ID string + Env map[string]string + } + _ = json.Unmarshal(raw, &r) + want := fp + if r.ID == "conf" { + want = "" + } + if r.Env[FingerprintVar] != want || (r.ID == "d" && r.Env["A"] != "1") { + t.Errorf("%s starts with %v, want %s=%q and its own words kept", r.ID, r.Env, FingerprintVar, want) + } + } + for _, raw := range split["novox"].Resources { + if strings.Contains(string(raw), FingerprintVar) { + t.Errorf("the node's own carries %s: %s", FingerprintVar, raw) + } + } +} + +func mustKey(t *testing.T, node, module, fp string) string { + k, err := Key(node, module, fp) + if err != nil { + t.Fatal(err) + } + return k } diff --git a/internal/compositions/composed.go b/internal/compositions/composed.go index 8107422f..16296023 100644 --- a/internal/compositions/composed.go +++ b/internal/compositions/composed.go @@ -234,6 +234,11 @@ func Split(node string, resources []json.RawMessage, fps map[string]string, fill if head.Type == string(declaration.TypeFile) && head.Module != "" { pathOf[head.Path] = key } + if head.Module != "" { + if raw, err = withFingerprint(raw, head.Type, fps[head.Module]); err != nil { + return nil, err + } + } c := out[key] c.Resources = append(c.Resources, raw) out[key] = c @@ -249,3 +254,34 @@ func Split(node string, resources []json.RawMessage, fps map[string]string, fill } return out, nil } + +// FingerprintVar is the environment value every process the node-engine starts, a daemon or a container, holds: the +// fingerprint of the want it applies. mesh-sdk's runtime reads it once at the process's start as the process's own build +// (novox/hq ADR 0656 §1, design 47 *The fingerprint handover*). +const FingerprintVar = "MESH_FINGERPRINT" + +// withFingerprint is raw, when it is a daemon or a container, with FingerprintVar in its env set to fp. The value is +// the node-engine's to say, so one a module declared of that name is replaced. Another resource, or no fingerprint, is +// returned as it is. +func withFingerprint(raw json.RawMessage, kind, fp string) (json.RawMessage, error) { + if fp == "" || (kind != string(declaration.TypeProcess) && kind != string(declaration.TypeContainer)) { + return raw, nil + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + return nil, fmt.Errorf("compositions: a %s resource is not JSON: %w", kind, err) + } + env := map[string]string{} + if had, ok := fields["env"]; ok { + if err := json.Unmarshal(had, &env); err != nil { + return nil, fmt.Errorf("compositions: the env of a %s resource is not words by name: %w", kind, err) + } + } + env[FingerprintVar] = fp + body, err := json.Marshal(env) + if err != nil { + return nil, err + } + fields["env"] = body + return json.Marshal(fields) +} -- 2.54.0 From 26581d87eda357dc63ddb733df2a7c1ee9531587 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:05:18 +0100 Subject: [PATCH 03/59] H-3: the node-engine says composed, running and capabilities-changed on the seat's subjects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §260, §227, §226, §177, §309, §404 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- cmd/mesh-host/nodeagent.go | 8 +- internal/link/engineseat.go | 6 +- internal/link/have.go | 92 ++++++++++++++++++++++ internal/link/have_test.go | 41 ++++++++++ internal/link/session.go | 17 ++++ internal/selfcompose/composedstate.go | 22 ++++++ internal/selfcompose/composedstate_test.go | 16 ++++ internal/selfcompose/selfcompose.go | 15 ++++ 8 files changed, 214 insertions(+), 3 deletions(-) diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index d22910f4..feb6aa62 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -656,8 +656,9 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus // composition bucket) is never applied. While the seat's assignment holds no key for this node it composes // nothing and keeps running what it runs; a assignment it cannot watch stops the agent, said, and it begins again. composer := selfComposer(session, node, hooks, say) - machinesNow.Store(composer) // the machines a failing network part points at, by the assignments (issue 774) - composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone + machinesNow.Store(composer) // the machines a failing network part points at, by the assignments (issue 774) + composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone + session.SetManifestOf(composer.ManifestOf) // the event running names the manifest of the build that runs env.Composed, env.SelfComposed, env.Policy, env.ComposedToday = composer, true, composer.Policy, composer.ComposedToday env.WhyUnmade, env.Waiting = composer.WhyUnmade, composer.Waiting env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does @@ -670,6 +671,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus cancel() current.set(nil) composerNow.Store(nil) + session.SetManifestOf(nil) shadowNow.Store(nil) if ctx.Err() != nil { return @@ -1076,6 +1078,8 @@ func selfComposer(session *link.Session, node string, hooks composerHooks, say f // And every composition it makes in its composition state composed-, a record the provisioner, the assigner // and the view read and nothing applies (novox/hq ADR 0593 §6). env.ComposedState = link.EngineComposed{Session: session, Node: node} + // And each composition it writes there said as the seat's event composed, which the assigner waits on. + env.SayComposed = session.SayComposed return selfcompose.New(env) } diff --git a/internal/link/engineseat.go b/internal/link/engineseat.go index 045e94ed..88e01c1f 100644 --- a/internal/link/engineseat.go +++ b/internal/link/engineseat.go @@ -204,11 +204,15 @@ func EngineEndpointMetadata(v catalogue.Verb, node string) map[string]string { // SayEngine says one of the seat's events about this node, on the events stream, said again through a transport fault // (sayOnce). func (l *natsLink) SayEngine(event string, body any) error { + return l.SayEngineOn(EngineEventSubject(event, l.node), body) +} + +// SayEngineOn says one of the seat's events on its subject, which names the node and, for some, the module. +func (l *natsLink) SayEngineOn(subject string, body any) error { raw, err := json.Marshal(body) if err != nil { return err } - subject := EngineEventSubject(event, l.node) return sayOnce(func(id string) error { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() diff --git a/internal/link/have.go b/internal/link/have.go index 063ac250..ccbf71b9 100644 --- a/internal/link/have.go +++ b/internal/link/have.go @@ -9,6 +9,7 @@ import ( nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" sdkkeys "git.novox.be/novox/mesh-sdk/go/keys" "github.com/novox/mesh-host/internal/compositions" + "slices" "sort" "strings" "sync" @@ -67,6 +68,11 @@ type haveState struct { userList map[string]nodeengine.UserListWritten // userListNoKey is each module whose user list was said to have no have key to ride on, so it is said once. userListNoKey map[string]bool + // runningSaid is, by module, the fingerprint and manifest its event running last said, so each is said once; and + // capsSaid the capabilities the event capabilities-changed last said (nil: none said yet this run). + runningSaid map[string]string + capsSaid []string + capsKnown bool // unjudged is each module whose composition moved (an apply began, or the fact names another build) and that no look // has judged since it stopped moving: its health is not yet judged, never the last look's of what ran before. unjudged map[string]bool @@ -360,6 +366,14 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) { return } s := &q.haves + // What the pass says as the seat's events (running, capabilities-changed) is said once the haves' lock is let go: a + // publish never holds it. + var said []func() + defer func() { + for _, say := range said { + say() + } + }() s.mu.Lock() defer s.mu.Unlock() if s.modules == nil && s.node == nil && len(s.removed) == 0 && len(s.userList) == 0 { @@ -426,6 +440,9 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) { s.written = map[string]string{} } s.written[module] = f.Build + if ev, ok := s.runningToSay(module, h, q.manifestOf()); ok { + said = append(said, func() { q.sayRunning(ev) }) + } } if !q.writeRemoved(write, bucket, node) { return @@ -450,6 +467,9 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) { }) { return } + if ev, ok := s.capabilitiesToSay(node, facts.Record); ok { + said = append(said, func() { q.sayCapabilities(ev) }) + } } } if err != nil { @@ -885,3 +905,75 @@ func nothingJudged(f ModuleFact, looked bool) string { } return nodeengineclient.HealthUnknown } + +// runningToSay is the event running for a module whose have was just written running and healthy at a fingerprint whose +// manifest is known, when it has not been said for that fingerprint and manifest (design 47, *What each step says*): the +// module, the fingerprint and the digest of the manifest of that build. A module not yet healthy, or one whose manifest +// the node does not know, says nothing, and is said at the write that knows. Under s.mu. +func (s *haveState) runningToSay(module string, h nodeengine.HaveByNodeModuleValue, manifest func(module, fp string) string) (nodeengine.Running, bool) { + if h.Phase != nodeengineclient.PhaseRunning || h.Health != nodeengineclient.HealthOK || h.Fp == "" || manifest == nil { + return nodeengine.Running{}, false + } + digest := manifest(module, h.Fp) + if digest == "" { + return nodeengine.Running{}, false + } + if s.runningSaid[module] == h.Fp+"\x00"+digest { + return nodeengine.Running{}, false + } + if s.runningSaid == nil { + s.runningSaid = map[string]string{} + } + s.runningSaid[module] = h.Fp + "\x00" + digest + return nodeengine.Running{V: sdkkeys.Version, Module: module, Fingerprint: h.Fp, Manifest: digest}, true +} + +// capabilitiesToSay is the event capabilities-changed when the node's record says other capabilities than the event last +// said (or none was said this run). Under s.mu. +func (s *haveState) capabilitiesToSay(node string, rec *nodeengine.NodeRecord) (nodeengine.CapabilitiesChanged, bool) { + if rec == nil { + return nodeengine.CapabilitiesChanged{}, false + } + if s.capsKnown && slices.Equal(s.capsSaid, rec.Capabilities) { + return nodeengine.CapabilitiesChanged{}, false + } + s.capsKnown, s.capsSaid = true, slices.Clone(rec.Capabilities) + caps := slices.Clone(rec.Capabilities) + if caps == nil { + caps = []string{} + } + return nodeengine.CapabilitiesChanged{V: sdkkeys.Version, Node: node, Capabilities: caps}, true +} + +// sayRunning says the event running, on its subject `running..`. +func (q *Session) sayRunning(ev nodeengine.Running) { + ev.Node, ev.At = q.Membership.Node, time.Now().UTC() + q.sayEngineOn(nodeengine.SubjectRunning(ev.Node, ev.Module), "running", ev) +} + +// sayCapabilities says the event capabilities-changed, on its subject `capabilities-changed.`. +func (q *Session) sayCapabilities(ev nodeengine.CapabilitiesChanged) { + ev.At = time.Now().UTC() + q.sayEngineOn(nodeengine.SubjectCapabilitiesChanged(ev.Node), "capabilities-changed", ev) +} + +// SetManifestOf gives the session what names the manifest of the build a module runs (the composer's picture of the +// catalog half): its digest, or "" when the node does not know it. +func (q *Session) SetManifestOf(f func(module, fp string) string) { + q.manifest.Lock() + q.manifestFn = f + q.manifest.Unlock() +} + +func (q *Session) manifestOf() func(module, fp string) string { + q.manifest.Lock() + defer q.manifest.Unlock() + return q.manifestFn +} + +// SayComposed says the event composed for a composition written at a fingerprint, on `composed.`. +func (q *Session) SayComposed(module, fingerprint, digest string) { + node := q.Membership.Node + q.sayEngineOn(nodeengine.SubjectComposed(node), "composed", nodeengine.Composed{V: sdkkeys.Version, Node: node, + Module: module, Fingerprint: fingerprint, Digest: digest, At: time.Now().UTC()}) +} diff --git a/internal/link/have_test.go b/internal/link/have_test.go index 18f50d85..a575afee 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -44,6 +44,29 @@ type haveLink struct { // sayFails fails that many of the node seat's says, as a publish past the bus's retry bound; said is every event said. sayFails int said []string + // onSaid are the events said on a subject of their own (running, capabilities-changed), by subject, in order. + onSaid []string +} + +func (l *haveLink) SayEngineOn(subject string, body any) error { + raw, _ := json.Marshal(body) + l.hmu.Lock() + l.onSaid = append(l.onSaid, subject+" "+string(raw)) + l.hmu.Unlock() + return nil +} + +// saidOn is the events said on a subject, as the bus would carry them. +func (l *haveLink) saidOn(prefix string) []string { + l.hmu.Lock() + defer l.hmu.Unlock() + var out []string + for _, e := range l.onSaid { + if strings.HasPrefix(e, prefix) { + out = append(out, e) + } + } + return out } func (l *haveLink) SayEngine(event string, body any) error { @@ -326,6 +349,10 @@ func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) { if l.kv["n1.node"].rev != at { t.Errorf("a report changing nothing wrote the node key again") } + // Design 47: the node's capabilities, said when they change, on capabilities-changed.. + if got := l.saidOn("mesh.seat.node-engine.event.capabilities-changed.n1 "); len(got) != 1 || !strings.Contains(got[0], `"capabilities":["apt","bus-state-calls","ledger-calls","systemd"]`) { + t.Errorf("capabilities-changed said %v, want once for the record's capabilities", got) + } // A runner that does not answer states what it stated before; one answering without any states none (an older runner). l.statedErr = fmt.Errorf("%w: nothing answered", ErrNoAnswer) l.stated = nil @@ -659,6 +686,20 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth( if h.Fp != fp || h.Composed != dig || h.Phase != nodeengineclient.PhaseRunning || h.Health != nodeengineclient.HealthUnknown { t.Fatalf("the have after the apply: %+v", h) } + // Design 47, *What each step says*: running is said once, when the build is healthy and its manifest is known, on + // running.., with the module, the fingerprint and the manifest's digest; never while it is unknown. + q.SetManifestOf(func(module, f string) string { + if module == "web" && f == fp { + return "sha256:" + strings.Repeat("c", 64) + } + return "" + }) + q.SayHealth(context.Background(), *healthOf(map[string]string{"web": StateHealthy})) + q.SayHealth(context.Background(), *healthOf(map[string]string{"web": StateHealthy})) + if got := l.saidOn("mesh.seat.node-engine.event.running.n1.web "); len(got) != 1 || + !strings.Contains(got[0], `"fingerprint":"`+fp+`"`) || !strings.Contains(got[0], `"manifest":"sha256:`+strings.Repeat("c", 64)+`"`) { + t.Fatalf("running said %v, want one for the healthy build", got) + } // Starting: it waits. Past its start window the check fails: failed. q.SayHealth(context.Background(), *healthOf(map[string]string{"web": StateStarting})) if got := l.have(t, "n1.web"); got.Health != nodeengineclient.HealthWaits || got.Fp != fp || got.Composed != dig { diff --git a/internal/link/session.go b/internal/link/session.go index e4cdf90c..2d74daf2 100644 --- a/internal/link/session.go +++ b/internal/link/session.go @@ -103,6 +103,9 @@ type Session struct { // haves is what this node-engine writes beside its facts (novox/hq ADR 0429 §4, issue 1132). haves haveState + // manifestFn names the manifest of the build a module runs, for the event running (SetManifestOf), under manifest. + manifest sync.Mutex + manifestFn func(module, fp string) string // healthWritten is this node's health report as last written or read (WriteNodeHealth): the health loop alone // writes it, one look at a time. healthWritten *NodeHealth @@ -463,6 +466,20 @@ func (q *Session) SayReads() { Node: q.Membership.Node, ManifestReads: manifestReads(), At: time.Now().UTC().Format(time.RFC3339Nano), Why: why}) } +// sayEngineOn says one of the node seat's events on a subject of its own (running, composed: a module in it), over the link +// open now; with none open it is not said. +func (q *Session) sayEngineOn(subject, event string, body any) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + if sayer, ok := bus.(interface{ SayEngineOn(string, any) error }); ok { + if err := sayer.SayEngineOn(subject, body); err != nil { + q.Say("could not say the node seat's " + event + ": " + err.Error()) + } + } +} + // SayEngine says one of the node seat's events over the link open now. With none open, or a publish that failed past the // bus's retry bound, it is kept and said at the next link's opening (SayReads; novox/hq ADR 0668 §4, issue 1966): an // event a reader waits on is never only a line in this engine's log. diff --git a/internal/selfcompose/composedstate.go b/internal/selfcompose/composedstate.go index 6793a857..aef6dfbc 100644 --- a/internal/selfcompose/composedstate.go +++ b/internal/selfcompose/composedstate.go @@ -137,6 +137,7 @@ func (c *Composer) writeComposed(ctx context.Context, at *published) (wrote, del } c.pubKept[key] = ComposedEntry{Value: values[key], Revision: rev} wrote++ + c.saidComposed(key, values[key]) } for _, key := range c.pastKeeping() { if err := c.env.ComposedState.Delete(ctx, key); err != nil { @@ -319,3 +320,24 @@ func ranOf(running []engcomp.Composed) map[string]string { } return out } + +// saidComposed says the event composed for a key just written that holds a module's composition at a fingerprint and was +// not refused: the node's own composition and a refusal say nothing. +func (c *Composer) saidComposed(key string, value []byte) { + if c.env.SayComposed == nil { + return + } + _, module, fp, isNode, err := engcomp.ParseKey(key) + if err != nil || isNode { + return + } + comp, err := engcomp.Decode(value) + if err != nil || comp.Refused != "" { + return + } + digest, err := engcomp.DigestOf(comp) + if err != nil { + return + } + c.env.SayComposed(module, fp, digest) +} diff --git a/internal/selfcompose/composedstate_test.go b/internal/selfcompose/composedstate_test.go index 94a25d70..38c02b51 100644 --- a/internal/selfcompose/composedstate_test.go +++ b/internal/selfcompose/composedstate_test.go @@ -84,6 +84,14 @@ func TestTheNodePublishesItsCompositions(t *testing.T) { state := &fakeComposed{kv: map[string]ComposedEntry{}} env := b.env() env.ComposedState = state + // Design 47, *What each step says*: each module composition written says composed, with its fingerprint and digest. + var saidMu sync.Mutex + var saidComposed []string + env.SayComposed = func(module, fingerprint, digest string) { + saidMu.Lock() + defer saidMu.Unlock() + saidComposed = append(saidComposed, module+" "+fingerprint+" "+digest) + } ctx, cancel := context.WithCancel(context.Background()) defer cancel() c := New(env) @@ -136,6 +144,14 @@ func TestTheNodePublishesItsCompositions(t *testing.T) { t.Errorf("the published composition of lemurs: %+v, %v", comp, err) } } + if comp, err := engcomp.Decode(state.kv[lemursKey].Value); err == nil { + dig, _ := engcomp.DigestOf(comp) + saidMu.Lock() + if len(saidComposed) != 1 || saidComposed[0] != "lemurs "+fp+" "+dig || dig == "" { + t.Errorf("composed said %v, want once for lemurs at %s with its digest %q (the node's own says none)", saidComposed, fp, dig) + } + saidMu.Unlock() + } // A pass that changes nothing writes nothing. setFor(t, &askedWithin, 0) if _, err := c.ComposeNow(ctx); err != nil { diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 3810cab4..1bb6fa1c 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -118,11 +118,26 @@ type Env struct { // SeatList is the node's reader of the seat list (novox/hq ADR 0636 §1): until the list was read, the composer hands // nothing new of a module whose seats are not all bootstrapped, and keeps what runs of it. Nil gates nothing. SeatList SeatGate + // SayComposed says the seat's event composed for a composition of a module written into the composition state at a + // fingerprint (design 47, *What each step says*): the module, the fingerprint and the digest of the composition. Nil says + // nothing. + SayComposed func(module, fingerprint, digest string) // SeatListChanged says the reader's held set changed: the node composes again, so what waited for the list is // handed at the read and not on a timer. Nil hears nothing. SeatListChanged <-chan struct{} } +// ManifestOf is the digest of the manifest of the build a module runs at fp, from the composer's picture of the catalog half; +// "" when the picture holds no such build (the event running names it, so one not known is not said). +func (c *Composer) ManifestOf(module, fp string) string { + c.mu.Lock() + defer c.mu.Unlock() + if m, ok := c.modules[module]; ok && m.Fp2 == fp { + return m.Manifest + } + return "" +} + // SeatGate is what the composer asks of the reader of the seat list: whether it may compose a module that claims, // invokes or provides these seats (seatlist.Reader.MayAct). type SeatGate interface { -- 2.54.0 From 420087ab14a2adbe06f70df780c96b8b77c16d5a Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:06:10 +0100 Subject: [PATCH 04/59] H-4: running says stopped, with no fingerprint, once the have no longer holds the module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §365, §404 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- internal/link/have.go | 13 +++++++++++-- internal/link/have_test.go | 4 ++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/internal/link/have.go b/internal/link/have.go index ccbf71b9..202cac27 100644 --- a/internal/link/have.go +++ b/internal/link/have.go @@ -444,7 +444,7 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) { said = append(said, func() { q.sayRunning(ev) }) } } - if !q.writeRemoved(write, bucket, node) { + if !q.writeRemoved(write, bucket, node, &said) { return } if !q.writeUserListsAlone(write, bucket, node) { @@ -521,7 +521,7 @@ func (q *Session) haveSucceeded() { // writeRemoved writes, once, each module the applied declaration no longer holds and this node-engine wrote a have key // for, with the phase removed over the revision it reads (issue 1369). The key is never deleted, and a key that is not // in the bucket is not created. False when the pass should stop. The caller holds the lock of the haves. -func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string) bool { +func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string, said *[]func()) bool { s := &q.haves gone := make([]string, 0, len(s.removed)) for module := range s.removed { @@ -567,10 +567,19 @@ func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string) b return false } delete(s.removed, module) + // The module no longer runs here: running says stopped, with no fingerprint, on its own subject (design 47 G §365, §404). + delete(s.runningSaid, module) + *said = append(*said, func() { q.sayRunning(stoppedRunning(node, module)) }) } return true } +// stoppedRunning is the event running of a module the node-engine's have no longer holds: stopped, with no fingerprint +// and no manifest (design 47 G §365, §404). +func stoppedRunning(node, module string) nodeengine.Running { + return nodeengine.Running{V: sdkkeys.Version, Node: node, Module: module, Stopped: true} +} + // writeUserListsAlone writes the user list of each module this node-engine holds no fact of (no apply here said its have) // into that module's key as the bucket holds it, keeping every other field the key says (novox/hq ADR 0547 §8). The have // of a module is one value with parts from more than one source: the apply's (fingerprint, digest, build, phase, health) diff --git a/internal/link/have_test.go b/internal/link/have_test.go index a575afee..caaa39fc 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -552,6 +552,10 @@ func TestAModuleLeavingTheDeclarationIsWrittenRemoved(t *testing.T) { if l.kv["n1.b"].rev != revB+1 { t.Errorf("b was written at revision %d, want one over %d", l.kv["n1.b"].rev, revB) } + // Design 47 G §365: a module the have no longer holds says running stopped, with no fingerprint, once, on its own subject. + if got := l.saidOn("mesh.seat.node-engine.event.running.n1.b "); len(got) != 1 || !strings.Contains(got[0], `"stopped":true`) || strings.Contains(got[0], `"fingerprint"`) { + t.Errorf("running said %v for b, want one stopped with no fingerprint", got) + } if l.kv["n1.a"].rev != revA || l.have(t, "n1.a").Phase != nodeengineclient.PhaseRunning { t.Errorf("a changed: %+v at %d", l.have(t, "n1.a"), l.kv["n1.a"].rev) } -- 2.54.0 From f3813a279e23dc22229b3891145a0f3ce820ce3e Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:09:56 +0100 Subject: [PATCH 05/59] H-6: the seat verb process-builds reads the build each process was started with; the engine keeps no callers table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §262, §245, §440, §447 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- cmd/mesh-host/main.go | 14 ++ internal/link/handacts.go | 26 ++-- internal/link/handover_test.go | 16 ++- internal/processbuilds/processbuilds.go | 140 +++++++++++++++++++ internal/processbuilds/processbuilds_test.go | 70 ++++++++++ 5 files changed, 245 insertions(+), 21 deletions(-) create mode 100644 internal/processbuilds/processbuilds.go create mode 100644 internal/processbuilds/processbuilds_test.go diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 1edcaad9..cf5c9e84 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -16,6 +16,7 @@ import ( "errors" "flag" "fmt" + nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshvault" "io" "net" @@ -42,6 +43,7 @@ import ( "github.com/novox/mesh-host/internal/liveness" "github.com/novox/mesh-host/internal/network" "github.com/novox/mesh-host/internal/outward" + "github.com/novox/mesh-host/internal/processbuilds" "github.com/novox/mesh-host/internal/profile" "github.com/novox/mesh-host/internal/reachable" "github.com/novox/mesh-host/internal/store" @@ -1018,6 +1020,18 @@ func runLink(ctx context.Context, opts options) error { queue.Acts.Compose = composeFrom(current, &composerNow) queue.Acts.Packages = packagesFrom(mine.Node, queue, &composerNow) queue.Acts.SeatList, queue.Acts.SeatListAgain = seatListRead, seatListAgain + queue.Acts.ProcessBuilds = func(module string) (any, error) { + d, err := declaredNow(opts.state, nil) + if errors.Is(err, store.ErrNothingDeclared) { + d, err = nil, nil + } + if err != nil { + return nodeengine.ProcessBuilds{Node: mine.Node, Error: "what runs on this node does not read: " + err.Error()}, nil + } + ctx, cancel := context.WithTimeout(context.Background(), lookBound) + defer cancel() + return processbuilds.Reader{Node: mine.Node, Run: processbuilds.Runner(opts.run())}.Read(ctx, d, module), nil + } queue.Acts.ComposeCandidate = composeCandidateFrom(mine.Node, &shadowNow) setRunsHere(runsFromAgent(current)) factsOfNow := func(ctx context.Context, ag *agent.Agent) { diff --git a/internal/link/handacts.go b/internal/link/handacts.go index 186c7cab..28ad5861 100644 --- a/internal/link/handacts.go +++ b/internal/link/handacts.go @@ -55,18 +55,11 @@ const ( // (novox/hq ADR 0636 §7). The node-engine seat's definition carries both verbs. ActSeatList = nodeengine.VerbSeatList ActSeatListAgain = nodeengine.VerbSeatListAgain + // ActProcessBuilds reads the build each process on this node was started with (design 47, *The fingerprint handover*); + // it reads only; who may call it is the seat definition's (design 47 G §440, §447). + ActProcessBuilds = nodeengine.VerbProcessBuilds ) -// SeatListCallers are who may call the engine's two seat-list verbs (novox/hq ADR 0636 §7; the node-engine seat's -// definition: the operator and the healers): every operator's session and the healers, both verbs alike. A call through -// the mesh MCP server, and nox, is proved only as its machine's tool runner (.node-tools, issue 627), so node-tools -// stands for an operator's session until a session is proved; without it the verbs answered nobody but the healers, and -// no healer calls them (issue 1966). -var SeatListCallers = stdio.VerbCallers{ - ActSeatList: {stdio.AnyPerson, "node-tools", "mesh-healers"}, - ActSeatListAgain: {stdio.AnyPerson, "node-tools", "mesh-healers"}, -} - // The states of an act: asked, done and refused are also the seat's events. const ( HandAsked = nodeengine.EventAsked @@ -166,6 +159,8 @@ type HandActs struct { // SeatList answers the seat list the engine holds; SeatListAgain makes it read the list now and answers what it holds. SeatList func() (any, error) SeatListAgain func() (any, error) + // ProcessBuilds answers the build each process of the node was started with, narrowed to a module when one is given. + ProcessBuilds func(module string) (any, error) // ComposeCandidate composes a check's candidate (the indexer's candidate key) on this node in shadow and answers per // module, applying and writing nothing; with input, also the composition input it composed from (the verb // compose-candidate, ADR 0593 §7). @@ -237,17 +232,16 @@ func (h *HandActs) Call(c EngineCall) (any, error) { } return h.Compose(arg("module")) case ActSeatList: - if err := SeatListCallers.Allowed(c.Verb, c.Caller); err != nil { - return nil, err - } if h.SeatList == nil { return nil, errors.New("this engine reads no seat list yet: its node agent has not started") } return h.SeatList() - case ActSeatListAgain: - if err := SeatListCallers.Allowed(c.Verb, c.Caller); err != nil { - return nil, err + case ActProcessBuilds: + if h.ProcessBuilds == nil { + return nil, errors.New("this engine reads no process yet: its node agent has not started") } + return h.ProcessBuilds(arg("module")) + case ActSeatListAgain: if h.SeatListAgain == nil { return nil, errors.New("this engine reads no seat list yet: its node agent has not started") } diff --git a/internal/link/handover_test.go b/internal/link/handover_test.go index 1fc5de3b..3fb5491d 100644 --- a/internal/link/handover_test.go +++ b/internal/link/handover_test.go @@ -271,6 +271,10 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) { ManifestReads: func() ([]string, time.Time) { return []string{"form:a", "seat:b"}, time.Date(2026, 11, 4, 10, 0, 0, 0, time.UTC) }, + // Design 47: the build each process was started with, narrowed by module. + ProcessBuilds: func(module string) (any, error) { + return map[string]any{"processes": []any{}, "module": module}, nil + }, // ADR 0565 §6, §15: the node's key of the engine's state packages. Packages: func(write bool) (any, error) { key := []byte(`{"node":"n1","packages":[{"package":"curl","modules":["backup","web"],"absent":false}]}`) @@ -301,16 +305,18 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) { {"packages", EngineToolSubject("packages", "n1") + " " + `{"write":true}`, `"key":null,"wrote":true`}, {"packages", EngineToolSubject("packages", "n1") + " " + `{"write":"true"}`, `"note":"the engine has written no key for this node yet`}, {"manifest-reads", EngineToolSubject("manifest-reads", "n1"), `"manifest-reads":["form:a","seat:b"],"node":"n1","stated-at":"2026-11-04T10:00:00Z","v":1`}, - // ADR 0636 §7: both seat-list verbs answer every operator's session, proved as a person or, through the mesh MCP - // server, as its machine's tool runner (issue 627, issue 1966), and the healers, no other module. + // Design 47 G §447: the engine keeps no callers table; who may call a verb is the seat definition's, which the + // grants part makes the bus enforce, so a call that reaches the engine is answered whoever the caller is. {"seat-list", "mesh.seat.node-engine.call.seat-list.n1.person~jochen", `"held":true`}, {"seat-list", "mesh.seat.node-engine.call.seat-list.n1.shanks~node-tools", `"held":true`}, - {"seat-list", "mesh.seat.node-engine.call.seat-list.n1.ace~messenger", `not on the holder's caller list for seat-list`}, + {"seat-list", "mesh.seat.node-engine.call.seat-list.n1.ace~messenger", `"held":true`}, {"seat-list-again", "mesh.seat.node-engine.call.seat-list-again.n1.shanks~node-tools", `"read":true`}, {"seat-list-again", "mesh.seat.node-engine.call.seat-list-again.n1.person~jochen", `"read":true`}, {"seat-list-again", "mesh.seat.node-engine.call.seat-list-again.n1.ace~mesh-healers", `"read":true`}, - {"seat-list-again", "mesh.seat.node-engine.call.seat-list-again.n1.ace~messenger", `not on the holder's caller list for seat-list-again`}, - {"seat-list-again", EngineToolSubject("seat-list-again", "n1"), `no caller the tool runner proved`}, + {"seat-list-again", "mesh.seat.node-engine.call.seat-list-again.n1.ace~messenger", `"read":true`}, + + {"process-builds", "mesh.seat.node-engine.call.process-builds.n1.person~jochen " + `{"module":"web"}`, `"module":"web","processes":[]`}, + {"process-builds", "mesh.seat.node-engine.call.process-builds.n1.ace~messenger", `"processes":[]`}, {"unserved", "mesh.seat.node-engine.call.unserved.n1.x", `"error":"the node seat node-engine serves no verb \"unserved\""`}, } { subject, body, ok := strings.Cut(row.subject, " ") diff --git a/internal/processbuilds/processbuilds.go b/internal/processbuilds/processbuilds.go new file mode 100644 index 00000000..9b39cf97 --- /dev/null +++ b/internal/processbuilds/processbuilds.go @@ -0,0 +1,140 @@ +// Package processbuilds reads the build each process of this node was started with (design 47, *The fingerprint handover*): +// the value MESH_FINGERPRINT the node-engine handed a daemon or a container at its start, read from the running process +// itself, never from what the declaration would hand it now. A process started before the node-engine handed the value, +// or one whose value cannot be read, says why its build is unknown instead of a fingerprint. +package processbuilds + +import ( + "context" + "encoding/json" + "fmt" + "os" + "sort" + "strconv" + "strings" + "time" + + nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-sdk/go/declaration" +) + +// Var is the environment value the build is handed in. +const Var = "MESH_FINGERPRINT" + +// Runner runs a command and answers its output. +type Runner func(ctx context.Context, name string, args ...string) (string, error) + +// Reader reads the running processes of a node. +type Reader struct { + Node string + Run Runner + // Environ reads a process's environment by its pid; nil reads /proc//environ. + Environ func(pid int) ([]byte, error) +} + +func (r Reader) environ(pid int) ([]byte, error) { + if r.Environ != nil { + return r.Environ(pid) + } + return os.ReadFile(fmt.Sprintf("/proc/%d/environ", pid)) +} + +// Read answers the build of every daemon and container the declaration holds, narrowed to module when it is not empty, +// ordered by module and process. +func (r Reader) Read(ctx context.Context, d *declaration.Declaration, module string) nodeengine.ProcessBuilds { + out := nodeengine.ProcessBuilds{Node: r.Node, Processes: []nodeengine.ProcessBuild{}} + if d == nil { + out.Error = "this node keeps no declaration of what it runs" + return out + } + for _, res := range d.Resources { + owner := res.DeclaredModule() + if owner == "" { + owner, _ = declaration.ModuleGuess(res.Identity()) + } + if module != "" && owner != module { + continue + } + switch p := res.(type) { + case *declaration.Process: + out.Processes = append(out.Processes, r.daemon(ctx, owner, p.Name)) + case *declaration.Container: + out.Processes = append(out.Processes, r.container(ctx, owner, p.Name)) + } + } + sort.SliceStable(out.Processes, func(i, j int) bool { + a, b := out.Processes[i], out.Processes[j] + if a.Module != b.Module { + return a.Module < b.Module + } + return a.Process < b.Process + }) + return out +} + +func (r Reader) daemon(ctx context.Context, module, name string) nodeengine.ProcessBuild { + unit := name + ".service" + b := nodeengine.ProcessBuild{Module: module, Process: unit, Kind: nodeengine.ProcessBuildKindService} + said, err := r.Run(ctx, "systemctl", "show", "--property=MainPID,ActiveEnterTimestamp", "--", unit) + if err != nil { + b.Unknown = "the unit cannot be read: " + err.Error() + return b + } + props := map[string]string{} + for _, line := range strings.Split(said, "\n") { + if k, v, ok := strings.Cut(strings.TrimSpace(line), "="); ok { + props[k] = v + } + } + pid, _ := strconv.Atoi(props["MainPID"]) + if pid <= 0 { + b.Unknown = "the unit runs no process now" + return b + } + if t, err := time.Parse("Mon 2006-01-02 15:04:05 MST", props["ActiveEnterTimestamp"]); err == nil { + b.Started = &t + } + raw, err := r.environ(pid) + if err != nil { + b.Unknown = "its environment cannot be read: " + err.Error() + return b + } + b.Fingerprint, b.Unknown = fingerprintIn(strings.Split(string(raw), "\x00")) + return b +} + +func (r Reader) container(ctx context.Context, module, name string) nodeengine.ProcessBuild { + b := nodeengine.ProcessBuild{Module: module, Process: name, Kind: nodeengine.ProcessBuildKindContainer} + var said string + var err error + for _, cri := range []string{"docker", "podman"} { + if said, err = r.Run(ctx, cri, "container", "inspect", "--format", "{{json .Config.Env}}\t{{.State.StartedAt}}", name); err == nil { + break + } + } + if err != nil { + b.Unknown = "the container cannot be read: " + err.Error() + return b + } + env, started, _ := strings.Cut(strings.TrimSpace(said), "\t") + var vars []string + if err := json.Unmarshal([]byte(env), &vars); err != nil { + b.Unknown = "its environment cannot be read: " + err.Error() + return b + } + if t, err := time.Parse(time.RFC3339Nano, started); err == nil { + b.Started = &t + } + b.Fingerprint, b.Unknown = fingerprintIn(vars) + return b +} + +// fingerprintIn is the value of the handed variable in a process's environment, or why there is none. +func fingerprintIn(vars []string) (fingerprint, unknown string) { + for _, v := range vars { + if value, ok := strings.CutPrefix(v, Var+"="); ok && value != "" { + return value, "" + } + } + return "", "started before the node-engine handed " + Var + ", or without it" +} diff --git a/internal/processbuilds/processbuilds_test.go b/internal/processbuilds/processbuilds_test.go new file mode 100644 index 00000000..fc31889b --- /dev/null +++ b/internal/processbuilds/processbuilds_test.go @@ -0,0 +1,70 @@ +package processbuilds + +import ( + "context" + "errors" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/declaration" +) + +// Design 47, *The fingerprint handover*: the build of a daemon is read from its running process's environment and of a +// container from its configuration, narrowed by module; one started without the value, one not running and one the +// runtime cannot read each say why their build is unknown, never a guessed fingerprint. +func TestProcessBuildsAreReadFromTheRunningProcesses(t *testing.T) { + fp := "src2:" + strings.Repeat("a", 64) + d := &declaration.Declaration{Resources: []declaration.Resource{ + &declaration.Process{ID: "web.d", Type: declaration.TypeProcess, Module: "web", Name: "web-d"}, + &declaration.Process{ID: "old.d", Type: declaration.TypeProcess, Module: "old", Name: "old-d"}, + &declaration.Process{ID: "down.d", Type: declaration.TypeProcess, Module: "down", Name: "down-d"}, + &declaration.Container{ID: "db.c", Type: declaration.TypeContainer, Module: "db", Name: "db-c"}, + &declaration.Container{ID: "gone.c", Type: declaration.TypeContainer, Module: "gone", Name: "gone-c"}, + &declaration.Directory{ID: "web.dir", Type: declaration.TypeDirectory, Module: "web"}, + }} + run := func(_ context.Context, name string, args ...string) (string, error) { + switch { + case name == "systemctl" && args[len(args)-1] == "web-d.service": + return "MainPID=11\nActiveEnterTimestamp=Mon 2026-11-09 10:00:00 UTC\n", nil + case name == "systemctl" && args[len(args)-1] == "old-d.service": + return "MainPID=12\nActiveEnterTimestamp=\n", nil + case name == "systemctl": + return "MainPID=0\nActiveEnterTimestamp=\n", nil + case name == "docker" && args[len(args)-1] == "db-c": + return `["A=1","MESH_FINGERPRINT=` + fp + `"]` + "\t2026-11-09T10:00:00.5Z\n", nil + } + return "", errors.New("no such container") + } + environ := func(pid int) ([]byte, error) { + if pid == 11 { + return []byte("A=1\x00MESH_FINGERPRINT=" + fp + "\x00"), nil + } + return []byte("A=1\x00"), nil + } + r := Reader{Node: "n1", Run: run, Environ: environ} + got := r.Read(context.Background(), d, "") + by := map[string]string{} + for _, p := range got.Processes { + by[p.Module] = p.Fingerprint + "|" + p.Unknown + } + want := map[string]string{"web": fp + "|", "db": fp + "|", + "old": "|started before the node-engine handed MESH_FINGERPRINT, or without it", + "down": "|the unit runs no process now", "gone": "|the container cannot be read: no such container"} + if len(by) != len(want) || got.Node != "n1" { + t.Fatalf("read %v, want %v", by, want) + } + for m, w := range want { + if by[m] != w { + t.Errorf("%s: %q, want %q", m, by[m], w) + } + } + if got.Processes[0].Module != "db" || got.Processes[len(got.Processes)-1].Module != "web" || got.Processes[len(got.Processes)-1].Started == nil { + t.Errorf("order or start time wrong: %+v", got.Processes) + } + if one := r.Read(context.Background(), d, "web"); len(one.Processes) != 1 || one.Processes[0].Process != "web-d.service" { + t.Errorf("narrowed by module: %+v", one.Processes) + } + if none := r.Read(context.Background(), nil, ""); none.Error == "" || none.Processes == nil { + t.Errorf("no declaration: %+v", none) + } +} -- 2.54.0 From b41a0d59b6b0fc414569555872c0f1ce5b3b18ab Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:12:55 +0100 Subject: [PATCH 06/59] H-14: a module held for its claim is the finding module-held until it is composed or no longer wanted Builds: design 47 main file (A claim problem holds its module), test 55 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- cmd/mesh-host/nodeagent.go | 10 +++ cmd/mesh-host/seatlist.go | 32 ++++++- cmd/mesh-host/seatlist_unread_test.go | 8 ++ internal/link/findings.go | 13 +++ internal/seatwait/held.go | 117 ++++++++++++++++++++++++++ internal/seatwait/seatwait_test.go | 35 +++++++- 6 files changed, 208 insertions(+), 7 deletions(-) create mode 100644 internal/seatwait/held.go diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index feb6aa62..2036a129 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -20,11 +20,13 @@ import ( "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay" "git.novox.be/novox/mesh-sdk/go/declaration" + "git.novox.be/novox/mesh-sdk/go/views" "github.com/novox/mesh-host/internal/agent" "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/compose/frombus" "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/liveness" + "github.com/novox/mesh-host/internal/seatwait" "github.com/novox/mesh-host/internal/selfcompose" "github.com/novox/mesh-host/internal/store" ) @@ -598,6 +600,9 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus } // The marks of the haves run off the composition pass, each bounded (see marker). marks := newMarker(run) + findings := findingsOfTheNode{session} + heldFindings := &seatwait.HeldKeeper{Node: node, Observer: "node-engine", Writer: findings, + Left: func() ([]views.Finding, error) { return findings.left(node) }, Say: say} hooks := composerHooks{ seatList: seats, seatListFailed: err != nil, @@ -626,6 +631,11 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus // A module held alone marks that module, never the others (issue 1126); the marking never holds the pass. marks.set("refused", func(ctx context.Context) { now.NotComposedEach(ctx, why, held) }) } + // And each module held says it as the finding `module-held`, cleared when it is composed again or no longer + // wanted here. A node refused whole (why) says nothing of its modules: their findings stand as they were. + if why == "" && run.Err() == nil { + marks.set("module-held", func(context.Context) { heldFindings.Reconcile(held) }) + } }, // And what it keeps of a module as it runs, for a manifest that does not say it yet: on the module's have (ADR 0601 §4). kept: func(kept map[string][]string) { diff --git a/cmd/mesh-host/seatlist.go b/cmd/mesh-host/seatlist.go index 7ae8c9fe..dd07ddb6 100644 --- a/cmd/mesh-host/seatlist.go +++ b/cmd/mesh-host/seatlist.go @@ -107,10 +107,16 @@ func (realClock) AfterFunc(d time.Duration, f func()) seatwait.Timer { type findingsOfTheNode struct{ session *link.Session } func (w findingsOfTheNode) encode(f views.Finding) (string, []byte, error) { - key := conditions.Key(conditions.ScopeMachine, f.ID, f.Kind) + // The seat list's finding is of the machine; a held module's is of the module on a machine, `.`. + scope, machine, source := conditions.ScopeMachine, f.ID, "seat-list-unread" + if f.Kind == seatwait.HeldKind { + scope, source = conditions.ScopeModule, seatwait.HeldKind + machine = f.ID[strings.LastIndex(f.ID, ".")+1:] + } + key := conditions.Key(scope, f.ID, f.Kind) v := condseat.FindingsByKeyValue{V: 1, Key: key, Kind: f.Kind, Observer: f.Observer, Severity: conditions.Warning, - Subject: condseat.Subject{Scope: conditions.ScopeMachine, ID: f.ID, Machine: f.ID}, Summary: f.Summary, - Headline: f.Headline, Explanation: f.Explanation, Source: "seat-list-unread", At: f.At.UTC()} + Subject: condseat.Subject{Scope: scope, ID: f.ID, Machine: machine}, Summary: f.Summary, + Headline: f.Headline, Explanation: f.Explanation, Source: source, At: f.At.UTC()} if err := conditions.CheckFinding(v); err != nil { return "", nil, err } @@ -118,6 +124,26 @@ func (w findingsOfTheNode) encode(f views.Finding) (string, []byte, error) { return key, raw, err } +// left is the findings of held modules an earlier run of this engine left on node: its keys of this kind. +func (w findingsOfTheNode) left(node string) ([]views.Finding, error) { + keys, err := w.session.FindingKeys() + if err != nil { + return nil, err + } + var out []views.Finding + for _, k := range keys { + id, ok := strings.CutPrefix(k, conditions.ScopeModule+".") + if !ok { + continue + } + if id, ok = strings.CutSuffix(id, "."+seatwait.HeldKind); !ok || !strings.HasSuffix(id, "."+node) { + continue + } + out = append(out, views.Finding{Observer: "node-engine", Kind: seatwait.HeldKind, ID: id}) + } + return out, nil +} + func (w findingsOfTheNode) Put(f views.Finding) error { key, raw, err := w.encode(f) if err != nil { diff --git a/cmd/mesh-host/seatlist_unread_test.go b/cmd/mesh-host/seatlist_unread_test.go index ef6c1208..3e76ac4c 100644 --- a/cmd/mesh-host/seatlist_unread_test.go +++ b/cmd/mesh-host/seatlist_unread_test.go @@ -3,6 +3,8 @@ package main import ( "testing" + "git.novox.be/novox/mesh-sdk/go/views" + "github.com/novox/mesh-host/internal/seatwait" "github.com/novox/mesh-host/internal/selfcompose" ) @@ -19,4 +21,10 @@ func TestAReaderThatCouldNotStartHoldsTheComposer(t *testing.T) { if got := composerEnv(nil, "n1", func(string) {}, composerHooks{}).SeatList; got != nil { t.Errorf("a composer with no reader asked for gates by %v, want none (the shadow)", got) } + // The finding of a held module is of the module on its machine, in a layout the condition keeper accepts (design 47, test 55). + key, raw, err := findingsOfTheNode{}.encode(views.Finding{Observer: "node-engine", Kind: seatwait.HeldKind, + ID: seatwait.HeldID("mesh-grants", "ace"), Summary: "mesh-grants is held on ace: x", Headline: "h", Explanation: "e"}) + if err != nil || key != "module.mesh-grants.ace.module-held" || len(raw) == 0 { + t.Errorf("a held module's finding encodes as %q, %v", key, err) + } } diff --git a/internal/link/findings.go b/internal/link/findings.go index c0356f9c..a4a46390 100644 --- a/internal/link/findings.go +++ b/internal/link/findings.go @@ -57,6 +57,19 @@ func (q *Session) HasFinding(key string) (bool, error) { return true, nil } +// FindingKeys lists the keys of this engine's findings over the link open now; none when the bucket holds none. +func (q *Session) FindingKeys() ([]string, error) { + kv, err := q.findings() + if err != nil { + return nil, err + } + keys, err := kv.Keys() + if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { + return nil, err + } + return keys, nil +} + // DeleteFinding clears one finding of this engine over the link open now; a finding already gone is cleared. func (q *Session) DeleteFinding(key string) error { kv, err := q.findings() diff --git a/internal/seatwait/held.go b/internal/seatwait/held.go new file mode 100644 index 00000000..b2682d50 --- /dev/null +++ b/internal/seatwait/held.go @@ -0,0 +1,117 @@ +// HeldKeeper keeps the node-engine's finding `module-held`: a module the node leaves out of its pass because its +// claim cannot be held (its seat's definition missing, removed or refused, a claim another holder keeps, a credential the +// vault refuses) is no silent leave-out. The finding names the node, the module and why, and the condition keeper keeps the +// condition (novox/hq ADR 0564 §2) until the module is composed again or is no longer wanted on the node, when the finding +// is cleared (design 47, *A claim problem holds its module, never the node's pass*; test 55). +package seatwait + +import ( + "fmt" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/views" +) + +// HeldKind is the finding of a held module; its id is `.`. +const HeldKind = "module-held" + +// HeldKeeper writes one finding for each module the node holds, and clears it when the module is no longer held. +type HeldKeeper struct { + Node string + Observer string + Writer views.FindingWriter + // Left lists the findings of this kind an earlier run of the engine left, once, at the first Reconcile: a module held + // then and composed while the engine was away must not keep its condition. Nil leaves none. + Left func() ([]views.Finding, error) + // Now is the clock; nil is time.Now. + Now func() time.Time + // Say tells the node's journal; nil says nothing. + Say func(string) + + mu sync.Mutex + swept bool + written map[string]views.Finding // by module +} + +// HeldID is the finding's id for a module on a node. +func HeldID(module, node string) string { return module + "." + node } + +// Reconcile makes the findings those of held (each module held and why): one is written for a module not yet written or +// held for other words, and one is cleared for a module no longer held, composed or no longer wanted here. A write or a +// clear that fails is said and tried again at the next Reconcile. +func (k *HeldKeeper) Reconcile(held map[string]string) { + k.mu.Lock() + defer k.mu.Unlock() + if k.written == nil { + k.written = map[string]views.Finding{} + } + if !k.swept && k.Left != nil { + left, err := k.Left() + if err != nil { + k.say(fmt.Sprintf("the findings an earlier run left of held modules cannot be read: %v", err)) + } else { + k.swept = true + for _, f := range left { + module := strings.TrimSuffix(f.ID, "."+k.Node) + if _, ok := k.written[module]; !ok { + k.written[module] = f + } + } + } + } else if k.Left == nil { + k.swept = true + } + names := make([]string, 0, len(k.written)) + for m := range k.written { + names = append(names, m) + } + sort.Strings(names) + for _, m := range names { + if _, still := held[m]; still { + continue + } + if err := k.Writer.Delete(k.written[m]); err != nil { + k.say(fmt.Sprintf("the finding that %s is held on %s cannot be cleared: %v", m, k.Node, err)) + continue + } + delete(k.written, m) + } + names = names[:0] + for m := range held { + names = append(names, m) + } + sort.Strings(names) + for _, m := range names { + why := held[m] + if was, ok := k.written[m]; ok && was.Summary != "" && strings.HasSuffix(was.Summary, ": "+why) { + continue + } + f := views.Finding{Observer: k.Observer, Kind: HeldKind, ID: HeldID(m, k.Node), + Summary: fmt.Sprintf("%s is held on %s: %s", m, k.Node, why), + Headline: fmt.Sprintf("%s is held on %s", m, k.Node), + Explanation: "The node leaves this module out of its pass, keeps what runs of it and composes and applies every other module. The finding clears when the module is composed again or is no longer wanted on the node.", + At: k.now()} + if err := k.Writer.Put(f); err != nil { + k.say(fmt.Sprintf("the finding that %s is held on %s cannot be written: %v", m, k.Node, err)) + continue + } + k.written[m] = f + k.say(f.Summary) + } +} + +func (k *HeldKeeper) now() time.Time { + if k.Now != nil { + return k.Now() + } + return time.Now() +} + +func (k *HeldKeeper) say(s string) { + if k.Say != nil { + k.Say(s) + } +} diff --git a/internal/seatwait/seatwait_test.go b/internal/seatwait/seatwait_test.go index f34aa5cf..3e945bd9 100644 --- a/internal/seatwait/seatwait_test.go +++ b/internal/seatwait/seatwait_test.go @@ -167,13 +167,40 @@ func caseAFindingOfAnEarlierProcessIsClearedAtTheFirstFullRead(t *testing.T) { } } +// Design 47, test 55: a module held for its claim is a finding that names the node, the module and why, written once, +// and cleared when the module is composed again or no longer wanted, also one an earlier run of the engine left. +func caseAHeldModuleIsAFindingClearedWhenItIsComposedOrNoLongerWanted(t *testing.T) { + findings := &fakeFindings{} + k := &HeldKeeper{Node: "ace", Observer: "node-engine", Writer: findings, + Left: func() ([]views.Finding, error) { + return []views.Finding{{Kind: HeldKind, ID: HeldID("gone", "ace")}}, nil + }} + k.Reconcile(map[string]string{"mesh-grants": "its seat is refused", "other": "no claim"}) + if len(findings.put) != 2 || findings.put[0].ID != "mesh-grants.ace" || len(findings.deleted) != 1 || findings.deleted[0].ID != "gone.ace" { + t.Fatalf("wrote %v, cleared %v", findings.put, findings.deleted) + } + k.Reconcile(map[string]string{"mesh-grants": "its seat is refused", "other": "no claim"}) + if len(findings.put) != 2 { + t.Fatalf("a module held for the same words was written again: %v", findings.put) + } + k.Reconcile(map[string]string{"mesh-grants": "its seat is refused"}) + if len(findings.deleted) != 2 || findings.deleted[1].ID != "other.ace" { + t.Fatalf("a module composed again kept its finding: %v", findings.deleted) + } + k.Reconcile(nil) + if len(findings.deleted) != 3 { + t.Fatalf("a module no longer wanted kept its finding: %v", findings.deleted) + } +} + // ADR 0636 §1, the node's finding of a seat list unread. func TestTheSeatListWait(t *testing.T) { for name, run := range map[string]func(*testing.T){ - "five minutes write one finding and the first full read clears it": caseTheNodeWritesOneFindingAfterFiveMinutesAndClearsItAtTheFirstFullRead, - "a read within the bound writes nothing": caseAReadWithinTheBoundWritesNothing, - "a finding that cannot be written is said and tried again": caseAFindingThatCannotBeWrittenIsSaidAndTriedAgain, - "a finding of an earlier process is cleared at the first full read": caseAFindingOfAnEarlierProcessIsClearedAtTheFirstFullRead, + "five minutes write one finding and the first full read clears it": caseTheNodeWritesOneFindingAfterFiveMinutesAndClearsItAtTheFirstFullRead, + "a read within the bound writes nothing": caseAReadWithinTheBoundWritesNothing, + "a finding that cannot be written is said and tried again": caseAFindingThatCannotBeWrittenIsSaidAndTriedAgain, + "a finding of an earlier process is cleared at the first full read": caseAFindingOfAnEarlierProcessIsClearedAtTheFirstFullRead, + "a held module is a finding cleared when it is composed or no longer wanted": caseAHeldModuleIsAFindingClearedWhenItIsComposedOrNoLongerWanted, } { t.Run(name, run) } -- 2.54.0 From 45e2b946c7901344861654192a678ec8ac69b036 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 13:58:33 +0100 Subject: [PATCH 07/59] H-5: the seats a node passes on are decided from what every node said it runs, by SeatsToPassOn's new input MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The node follows the node-engine seat's running events (an ordered read of the last message of each subject, then a durable consumer), keeps what each node runs by `at`, acts on none until the first read ended, and hands the composer each node's heard build with the manifest read by the event's digest; compose resolves the passes through catalogue.SeatsToPassOn (heard, entries) with the trunk's entry for a node that said nothing. Builds: design 47 G §199, §249, §459, §536, §556 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- cmd/mesh-host/nodeagent.go | 27 ++++++ internal/compose/frombus/frombus.go | 35 ++++++++ internal/compose/frombus/frombus_test.go | 17 ++++ internal/compose/input.go | 12 +++ internal/compose/rest.go | 77 ++++++++++++------ internal/compose/rest_test.go | 39 +++++++-- internal/link/running.go | 95 ++++++++++++++++++++++ internal/link/session_test.go | 38 +++++++++ internal/selfcompose/candidate.go | 2 +- internal/selfcompose/composedstate_test.go | 17 ++++ internal/selfcompose/selfcompose.go | 83 ++++++++++++++++++- 11 files changed, 406 insertions(+), 36 deletions(-) create mode 100644 internal/link/running.go diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 2036a129..76873b20 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -669,6 +669,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus machinesNow.Store(composer) // the machines a failing network part points at, by the assignments (issue 774) composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone session.SetManifestOf(composer.ManifestOf) // the event running names the manifest of the build that runs + go followRunning(run, session, composer, say) env.Composed, env.SelfComposed, env.Policy, env.ComposedToday = composer, true, composer.Policy, composer.ComposedToday env.WhyUnmade, env.Waiting = composer.WhyUnmade, composer.Waiting env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does @@ -1133,3 +1134,29 @@ func composerEnv(session *link.Session, node string, say func(string), hooks com } return env } + +// followRunning keeps the composer told what every node runs, from the seat node-engine's event running, for the run it is +// in: the stream or the link not being there is said and tried again after a backoff, never polled. +func followRunning(run context.Context, session *link.Session, composer *selfcompose.Composer, say func(string)) { + said := "" + for try := 1; run.Err() == nil; try++ { + began := time.Now() + err := session.FollowRunning(run, composer.NoteRunning, composer.RunningRead) + composer.RunningLost() + if run.Err() != nil { + return + } + if time.Since(began) > time.Minute { + try = 1 + } + if why := fmt.Sprint(err); why != said { + say("this node does not hear what the nodes run, and tries again: " + why) + said = why + } + select { + case <-run.Done(): + return + case <-time.After(link.Backoff(try)): + } + } +} diff --git a/internal/compose/frombus/frombus.go b/internal/compose/frombus/frombus.go index 89e6e931..7af21b84 100644 --- a/internal/compose/frombus/frombus.go +++ b/internal/compose/frombus/frombus.go @@ -156,6 +156,10 @@ type Scope struct { Assignments map[string]*meshmoduleassigner.Assignments InOwn []string Running []engcomp.Composed + // Heard is, by node and module, what each node said it runs, from the node-engine seat's event `running` kept by the + // composing node's own consumer (design 47 G §199): the passing of a seat reads it, and a module nothing was heard of + // counts by the trunk's entry. Never read from a have key, a want or a call. + Heard map[string]map[string]Heard // HeldToday is, in an answer, the modules the node keeps today as they run ("node" for its own composition), each with // how ("held", "failed", "runs a build of the old form"): a candidate that cannot compose one, or holds one only // behind one of them, keeps it as today, which is said under CandidateAnswer.Held and is no refusal (issue 1126). @@ -551,6 +555,7 @@ func (b base) finish(ctx context.Context, r Reader, build func(ctx context.Conte if err := fromBuilds(ctx, r, build, &in, s, b.assignments[s.Node], b.modules, read, parse, refuse, wanted); err != nil { return compose.Input{}, err } + keepHeard(&in, s, read) // Its own modules' secrets, sealed to its own key by the vault: a composition carries them sealed (ADR 0550 §5). // Asked only when a module placed on it declares one: a node none of whose modules has a secret composes // whether or not the vault answers. @@ -581,6 +586,36 @@ func (b base) finish(ctx context.Context, r Reader, build func(ctx context.Conte return in, nil } +// Heard is one `running` event as the scope keeps it: the fingerprint of the build, the digest of its manifest, or Stopped. +type Heard struct { + Fingerprint string + Manifest string + Stopped bool +} + +// keepHeard puts, as each node's Heard, what it said it runs of each module it is assigned, with the manifest of the build +// read by the event's digest; a digest that cannot be read leaves the entry unread, and the passing says so (§459). +func keepHeard(in *compose.Input, s Scope, read func(module, digest string) (catalogue.Manifest, error)) { + for i := range in.Nodes { + n := &in.Nodes[i] + heard := s.Heard[n.Name] + if len(heard) == 0 { + continue + } + n.Heard = map[string]compose.Heard{} + for _, module := range sortedKeys(heard) { + h := heard[module] + got := compose.Heard{Fingerprint: h.Fingerprint, Stopped: h.Stopped} + if !h.Stopped && h.Manifest != "" { + if man, err := read(module, h.Manifest); err == nil { + got.Manifest, got.Read = man, true + } + } + n.Heard[module] = got + } + } +} + // Compose is the node's compositions over the bus (ADR 0547 §2): input for s, then composed at the fingerprints fps // gives each module in the Reading mode, as the controller composes them, by the one rule its answer to a candidate // follows too (composeEach, issue 1126): a module refused alone (its build unread or not taken in, its composition diff --git a/internal/compose/frombus/frombus_test.go b/internal/compose/frombus/frombus_test.go index 6ae853e8..c34bbd50 100644 --- a/internal/compose/frombus/frombus_test.go +++ b/internal/compose/frombus/frombus_test.go @@ -575,6 +575,23 @@ func TestInput(t *testing.T) { t.Errorf("the want's build is kept on the composing node alone: %v, %v", in.Nodes, err) } + // Design 47 G §199 (test 56): what each node said it runs (the seat node-engine's event running) is that node's Heard, + // with the manifest read by the event's digest; a digest that cannot be read leaves the entry unread, and a stopped + // module has none. + in, err = Input(ctx, kept, Mesh{}, Scope{Node: "laptop", Heard: map[string]map[string]Heard{ + "laptop": {"xterm": {Fingerprint: "f1", Manifest: digest}, "media": {Stopped: true}}, "anchor": {"xterm": {Fingerprint: "f2", Manifest: "gone"}}}}) + if err != nil { + t.Fatal(err) + } + for _, n := range in.Nodes { + h := n.Heard["xterm"] + switch { + case n.Name == "laptop" && (!h.Read || h.Fingerprint != "f1" || h.Manifest.Module != "xterm" || !n.Heard["media"].Stopped): + t.Errorf("laptop's Heard holds %+v", n.Heard) + case n.Name == "anchor" && (h.Read || h.Fingerprint != "f2"): + t.Errorf("anchor's Heard holds %+v, want f2 unread", n.Heard) + } + } // Issue 1126, the seed's refusals. restic and the provisioner's client as the catalogue holds them (mesh-catalog // c05aa5e6), restic moved with an own secret the vault leaves out (it hands it to the client, mesh-vault ownnode.go // handed): composed from the indexer's manifest restic is not handed and stands in for that secret, as novox's, diff --git a/internal/compose/input.go b/internal/compose/input.go index 7c1de99a..d4dd4a07 100644 --- a/internal/compose/input.go +++ b/internal/compose/input.go @@ -240,6 +240,9 @@ type Node struct { // own resolution only, never to another node's, and a declaration carries the kept commit for the module. Kept map[string]catalogue.Manifest KeptBuilds map[string]string + // Heard is, by module, what this node said it runs, from the node-engine seat's event `running` (design 47 G §199): the + // passing of a seat resolves the build each assignment runs from it. Absent for a module nothing was heard of. + Heard map[string]Heard // Contract is the contract of its node-engine's newest statement (inv.HealthOf); Stated is whether it ever stated one. Contract int Stated bool @@ -437,6 +440,15 @@ type Credential struct { Why string } +// Heard is what a node's `running` event for a module said: the build's fingerprint, or Stopped (no fingerprint, §365), with +// the manifest of the build's entry read by the event's manifest digest (Read false when it could not be read). +type Heard struct { + Fingerprint string + Stopped bool + Manifest catalogue.Manifest + Read bool +} + // Run is the build an assignment names: the commit it is of and the build made from it (chain.Ref). type Run struct { Commit string diff --git a/internal/compose/rest.go b/internal/compose/rest.go index f97ef72a..f921f1e1 100644 --- a/internal/compose/rest.go +++ b/internal/compose/rest.go @@ -102,7 +102,7 @@ func (c *composer) theRestOfTheMeshNow(exclude string) (catalogue.World, Passes, } } var passes Passes - holdings, passes = passSeatsOn(holdings, c.assignmentsOf(shelf, everyAssigned)) + holdings, passes = passSeatsOn(holdings, c.assignmentsOf(everyAssigned), c.heard, c.entries(shelf)) for _, n := range nodes { if n.Name == exclude { continue @@ -177,10 +177,11 @@ func (c *composer) theRestOfTheMeshNow(exclude string) (catalogue.World, Passes, } // passSeatsOn finds, in the composition that finds it, the new holder of each seat held once whose holder on record has left -// it and which has exactly one claimant able to hold it (catalogue.SeatsToPassOn, novox/hq ADR 0660), and answers the -// holdings as they are after it, with the passes. Nothing is recorded. -func passSeatsOn(holdings []catalogue.Held, assignments []catalogue.Assignment) ([]catalogue.Held, Passes) { - passes, said := catalogue.SeatsToPassOn(holdings, assignments) +// it and which has exactly one claimant able to hold it (catalogue.SeatsToPassOn, novox/hq ADR 0660, design 47 G §199, §459), and +// answers the holdings as they are after it, with the passes. Nothing is recorded. +func passSeatsOn(holdings []catalogue.Held, assignments []catalogue.Assignment, heard func(node, module string) (catalogue.Heard, bool), + entries catalogue.EntryReader) ([]catalogue.Held, Passes) { + passes, said := catalogue.SeatsToPassOn(holdings, assignments, heard, entries) out := Passes{Said: said} for _, p := range passes { out.Passes = append(out.Passes, p) @@ -197,11 +198,10 @@ func passSeatsOn(holdings []catalogue.Held, assignments []catalogue.Assignment) return holdings, out } -// assignmentsOf are the assignments of every machine, as the passing of a seat reads them (novox/hq ADR 0660 clause 2): each -// with the manifest of the build its node runs where the Input holds it (the node's Kept: what it runs of a module kept at -// that build), else the trunk's on the shelf, which for the composing node's own modules is already the build they are -// wanted at. A module assigned and missing from both is still assigned, marked unread, so its holder is not taken for gone. -func (c *composer) assignmentsOf(shelf map[string]catalogue.Manifest, assigned map[string][]string) []catalogue.Assignment { +// assignmentsOf are the assignments of every machine, as the passing of a seat reads them (§459): a node, a module and no +// placement branch, this composer's view of the catalog half being the trunk's alone. The build each runs is resolved by +// catalogue.SeatsToPassOn from c.heard and c.entries. +func (c *composer) assignmentsOf(assigned map[string][]string) []catalogue.Assignment { var out []catalogue.Assignment names := make([]string, 0, len(assigned)) for node := range assigned { @@ -209,26 +209,55 @@ func (c *composer) assignmentsOf(shelf map[string]catalogue.Manifest, assigned m } sort.Strings(names) for _, node := range names { - var kept map[string]catalogue.Manifest - for _, n := range c.in.Nodes { - if n.Name == node { - kept = n.Kept - } - } for _, module := range assigned[node] { - m, ok := kept[module] - if !ok { - m, ok = shelf[module] - } - if !ok { - m = catalogue.Manifest{Module: module} - } - out = append(out, catalogue.Assignment{Node: node, Manifest: m, Unread: !ok}) + out = append(out, catalogue.Assignment{Node: node, Module: module}) } } return out } +// heard is what the node-engine seat's `running` said of a module on a node (Node.Heard): a fingerprint or stopped. Nothing +// is heard of a module no event named, and the passing then takes the entry of the module on the trunk (§199). +func (c *composer) heard(node, module string) (catalogue.Heard, bool) { + for _, n := range c.in.Nodes { + if n.Name == node { + h, ok := n.Heard[module] + return catalogue.Heard{Fingerprint: h.Fingerprint, Stopped: h.Stopped}, ok + } + } + return catalogue.Heard{}, false +} + +// entries are the composer's reader of the catalog half for the passing of a seat: an entry by the fingerprint a node said it +// runs, read through the digest its event named, and a module's entry on the trunk, the shelf. +func (c *composer) entries(shelf map[string]catalogue.Manifest) catalogue.EntryReader { + byFingerprint := map[string]catalogue.Manifest{} + for _, n := range c.in.Nodes { + for _, h := range n.Heard { + if h.Read && h.Fingerprint != "" { + byFingerprint[h.Fingerprint] = h.Manifest + } + } + } + return entryReader{byFingerprint: byFingerprint, shelf: shelf} +} + +type entryReader struct { + byFingerprint map[string]catalogue.Manifest + shelf map[string]catalogue.Manifest +} + +func (r entryReader) ByFingerprint(fingerprint string) (catalogue.Manifest, bool) { + m, ok := r.byFingerprint[fingerprint] + return m, ok +} + +// OnBranch answers the trunk's entry whatever the branch: this composer's shelf is the trunk's alone. +func (r entryReader) OnBranch(module, _ string) (catalogue.Manifest, bool) { + m, ok := r.shelf[module] + return m, ok +} + // Lines are what the passes say, one each, then what the finding itself says: a pass is composed so and recorded by nobody // (ADR 0660 clause 4). func (p Passes) Lines() []string { diff --git a/internal/compose/rest_test.go b/internal/compose/rest_test.go index 8cb1e61d..8ba82ad0 100644 --- a/internal/compose/rest_test.go +++ b/internal/compose/rest_test.go @@ -162,19 +162,18 @@ func TestASeatWhoseHolderIsAssignedNowhereGoesToItsOnlyClaimant(t *testing.T) { func TestNoSeatIsPassedForTwoClaimantsAnAssignedHolderOrNoClaimant(t *testing.T) { withSeat(t) held := []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}} - _, passes := passSeatsOn(held, []catalogue.Assignment{ - {Node: "novox", Manifest: aBrokerClaiming("gate")}, {Node: "ace", Manifest: aBrokerClaiming("other")}}) + shelf := map[string]catalogue.Manifest{"gate": aBrokerClaiming("gate"), "other": aBrokerClaiming("other"), "old-broker": aBrokerClaiming("old-broker")} + _, passes := passSeatsOn(held, []catalogue.Assignment{{Node: "novox", Module: "gate"}, {Node: "ace", Module: "other"}}, nil, entryReader{shelf: shelf}) said := strings.Join(passes.Said, "\n") if len(passes.Passes) != 0 || !strings.Contains(said, "gate") || !strings.Contains(said, "other") || !strings.Contains(said, "seat mesh-broker --to") { t.Fatalf("passes %+v said %q", passes.Passes, said) } - got, passes := passSeatsOn(held, []catalogue.Assignment{ - {Node: "novox", Manifest: aBrokerClaiming("old-broker")}, {Node: "novox", Manifest: aBrokerClaiming("gate")}}) + got, passes := passSeatsOn(held, []catalogue.Assignment{{Node: "novox", Module: "old-broker"}, {Node: "novox", Module: "gate"}}, nil, entryReader{shelf: shelf}) if len(passes.Passes) != 0 || len(got) != 1 || got[0].Module != "old-broker" { t.Fatalf("holder assigned: %+v %+v", got, passes) } - got, passes = passSeatsOn(held, nil) + got, passes = passSeatsOn(held, nil, nil, entryReader{shelf: shelf}) if len(passes.Passes) != 0 || len(got) != 1 || got[0].Module != "old-broker" { t.Fatalf("no claimant: %+v %+v", got, passes) } @@ -183,13 +182,35 @@ func TestNoSeatIsPassedForTwoClaimantsAnAssignedHolderOrNoClaimant(t *testing.T) // seat_pass_test.go TestAHolderAssignedButNotOnTheShelfIsNotTakenForGone. func TestAHolderAssignedButNotOnTheShelfIsNotTakenForGone(t *testing.T) { withSeat(t) - as := (&composer{}).assignmentsOf(map[string]catalogue.Manifest{"gate": aBrokerClaiming("gate")}, - map[string][]string{"novox": {"gate", "old-broker"}}) + as := (&composer{}).assignmentsOf(map[string][]string{"novox": {"gate", "old-broker"}}) passes, _ := catalogue.SeatsToPassOn( - []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}, as) + []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "novox", Module: "old-broker"}}, as, nil, + entryReader{shelf: map[string]catalogue.Manifest{"gate": aBrokerClaiming("gate")}}) if len(passes) != 0 { t.Fatalf("a holder missing from the shelf lost its seat: %+v", passes) } + // Design 47 G §199 (test 56): the build a node runs is the one it said in its running event, the trunk's entry counting + // only for a node that said nothing. The trunk's old-broker still claims the seat; the one the away node runs does not. + for name, c := range map[string]struct { + heard Heard + passTo string + }{ + "the away node runs a build that no longer claims the seat": {Heard{Fingerprint: "f1", Manifest: catalogue.Manifest{Module: "old-broker"}, Read: true}, "gate"}, + "the away node says it stopped it": {Heard{Stopped: true}, "gate"}, + "the away node's build cannot be read": {Heard{Fingerprint: "f1"}, ""}, + "the away node runs a build that claims the seat": {Heard{Fingerprint: "f1", Manifest: aBrokerClaiming("old-broker"), Read: true}, ""}, + } { + in := Input{ + Catalogue: map[string]catalogue.Manifest{"old-broker": aBrokerClaiming("old-broker"), "gate": aBrokerClaiming("gate")}, + Nodes: []Node{{Name: "away", Assigned: []string{"old-broker"}, Heard: map[string]Heard{"old-broker": c.heard}}, + {Name: "novox", Assigned: []string{"gate"}}}, + Holdings: []catalogue.Held{{Claim: "mesh-broker", Scope: catalogue.ScopeMesh, Node: "away", Module: "old-broker"}}, + } + _, got, _, err := TheRestOfTheMesh(in, "") + if err != nil || (c.passTo == "") != (len(got.Passes) == 0) || (c.passTo != "" && got.Passes[0].To.Module != c.passTo) { + t.Errorf("%s: %v %+v", name, err, got) + } + } } // The not-resolving node of a view with a holder on record: holdings reach the first pass (novox/hq ADR 0131), so a node running @@ -267,7 +288,7 @@ func TestAHolderWhoseManifestDroppedTheSeatGoesToItsOnlyClaimant(t *testing.T) { } // The build the holder's node runs still claims the seat, though the trunk dropped it: the holder keeps it (ADR 0660 §2). running := mk("gate") - running.Nodes[0].Kept = map[string]catalogue.Manifest{"old-broker": aBrokerClaiming("old-broker")} + running.Nodes[0].Heard = map[string]Heard{"old-broker": {Fingerprint: "f", Manifest: aBrokerClaiming("old-broker"), Read: true}} _, passes, _, err = TheRestOfTheMesh(running, "") if err != nil || len(passes.Passes) != 0 || len(passes.Said) != 0 { t.Fatalf("a holder whose running build claims the seat lost it: %v %+v", err, passes) diff --git a/internal/link/running.go b/internal/link/running.go new file mode 100644 index 00000000..722ce19a --- /dev/null +++ b/internal/link/running.go @@ -0,0 +1,95 @@ +package link + +import ( + "context" + "encoding/json" + "fmt" + + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "github.com/nats-io/nats.go" +) + +// The node seat's event running, said by every node-engine for each module it saw running healthy, is kept on the stream +// SEAT_node-engine_running, the last message of each subject with no age limit (design 47 G §199, §260). The node reads it +// at each start through an ordered consumer it makes for that one read (§556), then follows a durable consumer of its own +// from its acknowledged position: what the other nodes run is known from the last message of each subject however long ago it +// was said, never from a have key or a call. + +// RunningBus is a link that hears the seat's event running of every node. +type RunningBus interface { + // FirstRunning calls first with the last message of every subject, through an ordered consumer made for this read, and + // answers when it has read them all. + FirstRunning(ctx context.Context, first func([]byte)) error + // HearRunning calls heard with each message after the durable consumer's acknowledged position (a new consumer begins at + // the last message of each subject), until stop is called or the link ends. + HearRunning(consumer string, heard func([]byte)) (stop func(), err error) +} + +// RunningConsumer is the name of this node's durable consumer of the stream. +func RunningConsumer(node string) string { return "node-engine-" + node + "-running" } + +func (l *natsLink) FirstRunning(ctx context.Context, first func([]byte)) error { + sub, err := l.js.SubscribeSync(nodeengine.SubjectRunningAll, nats.OrderedConsumer(), nats.DeliverLastPerSubject(), + nats.BindStream(nodeengine.StreamRunning)) + if err != nil { + return fmt.Errorf("the stream %s cannot be read: %w", nodeengine.StreamRunning, err) + } + defer sub.Unsubscribe() + info, err := sub.ConsumerInfo() + if err != nil { + return fmt.Errorf("the stream %s cannot be read: %w", nodeengine.StreamRunning, err) + } + for total, got := info.Delivered.Consumer+info.NumPending, uint64(0); got < total; got++ { + msg, err := sub.NextMsgWithContext(ctx) + if err != nil { + return fmt.Errorf("reading the last running of each subject (%d of %d): %w", got, total, err) + } + first(msg.Data) + } + return nil +} + +func (l *natsLink) HearRunning(consumer string, heard func([]byte)) (func(), error) { + sub, err := l.js.Subscribe(nodeengine.SubjectRunningAll, func(m *nats.Msg) { + heard(m.Data) + _ = m.Ack() + }, nats.Durable(consumer), nats.BindStream(nodeengine.StreamRunning), nats.DeliverLastPerSubject(), nats.AckExplicit(), + nats.ManualAck()) + if err != nil { + return nil, fmt.Errorf("the stream %s cannot be followed: %w", nodeengine.StreamRunning, err) + } + return func() { _ = sub.Unsubscribe() }, nil +} + +var _ RunningBus = (*natsLink)(nil) + +// FollowRunning hands every node's running event to heard: the last of each subject first, with readEnded called after them +// (until then the caller acts on none, §556), then each after them, until ctx ends. An error is the stream or the link not +// being there, which the caller tries again. +func (q *Session) FollowRunning(ctx context.Context, heard func(nodeengine.Running), readEnded func()) error { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + h, ok := bus.(RunningBus) + if bus == nil || !ok { + return ErrNoLink + } + decode := func(raw []byte) { + var ev nodeengine.Running + if json.Unmarshal(raw, &ev) == nil && ev.Node != "" && ev.Module != "" { + heard(ev) + } + } + if err := h.FirstRunning(ctx, decode); err != nil { + return err + } + stop, err := h.HearRunning(RunningConsumer(q.Membership.Node), decode) + if err != nil { + return err + } + defer stop() + readEnded() + <-ctx.Done() + return nil +} diff --git a/internal/link/session_test.go b/internal/link/session_test.go index 7fc1fdd6..64783c1d 100644 --- a/internal/link/session_test.go +++ b/internal/link/session_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "github.com/nats-io/nats.go" ) @@ -24,6 +25,26 @@ func newQuietLink() *quietLink { return &quietLink{lost: make(chan error, 1)} } func (l *quietLink) Lost() <-chan error { return l.lost } func (l *quietLink) Close() {} +// runningLink hears the node seat's running events: the last of each subject first, then the durable consumer's. +type runningLink struct { + *quietLink + first, after [][]byte +} + +func (l *runningLink) FirstRunning(_ context.Context, first func([]byte)) error { + for _, raw := range l.first { + first(raw) + } + return nil +} + +func (l *runningLink) HearRunning(_ string, heard func([]byte)) (func(), error) { + for _, raw := range l.after { + heard(raw) + } + return func() {}, nil +} + func aMember(t *testing.T) (Membership, ed25519.PrivateKey) { t.Helper() public, private, err := ed25519.GenerateKey(nil) @@ -46,6 +67,23 @@ func TestFactsWithNoLinkOpenAreNotSaid(t *testing.T) { if s.Linked() || s.SayFacts(context.Background(), Report{}) { t.Fatal("facts were said on a link that was let go") } + + // Design 47 G §199, §556: with no link the running events are not followed; with one, the last message of each subject + // comes first, the read ends, then the durable consumer's, and a body that is no running event is passed over. + if err := s.FollowRunning(context.Background(), func(nodeengine.Running) {}, func() {}); !errors.Is(err, ErrNoLink) { + t.Fatalf("followed with no link: %v", err) + } + rl := &runningLink{quietLink: newQuietLink(), + first: [][]byte{[]byte(`{"node":"a","module":"x","fingerprint":"f1"}`), []byte(`not json`), []byte(`{"module":"x"}`)}, + after: [][]byte{[]byte(`{"node":"b","module":"y","stopped":true}`)}} + s.Attach(context.Background(), rl) + var order []string + ctx, stop := context.WithCancel(context.Background()) + err := s.FollowRunning(ctx, func(ev nodeengine.Running) { order = append(order, ev.Node+"/"+ev.Module) }, + func() { order = append(order, "read"); stop() }) + if err != nil || strings.Join(order, " ") != "a/x b/y read" { + t.Fatalf("followed %v: %v, want a/x b/y read", order, err) + } } // A link opening says the node's facts, off the link's own loop. diff --git a/internal/selfcompose/candidate.go b/internal/selfcompose/candidate.go index 3791f1e2..a070e45b 100644 --- a/internal/selfcompose/candidate.go +++ b/internal/selfcompose/candidate.go @@ -461,7 +461,7 @@ func (c *Composer) composeShadow(ctx context.Context, running []engcomp.Composed c.mu.Lock() c.wants = wants c.mu.Unlock() - scope := frombus.Scope{Node: node, KeptAt: kept, InOwn: inOwn, Running: running, SameValue: c.sameValue} + scope := frombus.Scope{Node: node, KeptAt: kept, InOwn: inOwn, Running: running, SameValue: c.sameValue, Heard: c.heardNow()} if input != nil { input(scope, fps) } diff --git a/internal/selfcompose/composedstate_test.go b/internal/selfcompose/composedstate_test.go index 38c02b51..fb5b10cd 100644 --- a/internal/selfcompose/composedstate_test.go +++ b/internal/selfcompose/composedstate_test.go @@ -152,6 +152,23 @@ func TestTheNodePublishesItsCompositions(t *testing.T) { } saidMu.Unlock() } + // Design 47 G §199, §556 (test 56): what the nodes say they run is acted on once the first read of the stream ended; the + // newest by `at` stands, an older one and the same again change nothing. + at := time.Now() + c.NoteRunning(nodeengine.Running{Node: "n2", Module: "lemurs", Fingerprint: "a", Manifest: "sha256:a", At: at}) + if got := c.heardNow(); got != nil { + t.Errorf("heard %v before the first read ended", got) + } + c.RunningRead() + c.NoteRunning(nodeengine.Running{Node: "n2", Module: "lemurs", Fingerprint: "old", At: at.Add(-time.Hour)}) + c.NoteRunning(nodeengine.Running{Node: "n3", Module: "lemurs", Stopped: true, At: at}) + if got := c.heardNow(); len(got) != 2 || got["n2"]["lemurs"].Fingerprint != "a" || !got["n3"]["lemurs"].Stopped { + t.Errorf("heard %v, want n2's lemurs at a (not the older) and n3's stopped", got) + } + c.RunningLost() + if got := c.heardNow(); got != nil { + t.Errorf("heard %v with the stream lost", got) + } // A pass that changes nothing writes nothing. setFor(t, &askedWithin, 0) if _, err := c.ComposeNow(ctx); err != nil { diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 1bb6fa1c..cba3a1f2 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -176,6 +176,10 @@ type Composer struct { unmadeSaid map[string]string // modulesRev is the revision of the last update of the indexer's modules taken: where the picture of them stands. modulesRev uint64 + // heard is, by node and module, what each node's event running last said it runs, kept from the seat's stream by this + // node's own consumer (NoteRunning), under mu; heardRead is whether the first read of the stream ended (RunningRead). + heard map[string]map[string]heardRun + heardRead bool // rehand asks for a pass that hands again what the agent waits for (Waiting). rehand chan struct{} // sealedOwn is each own secret's sealed value as first handed, by module/name and generation: a value sealed again @@ -998,7 +1002,7 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { var keptMu sync.Mutex var holds map[string]compose.Hold fromManifest := map[string]string{} // by module, why it is composed from the indexer's manifest and not a build - scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running, Kept: func(module, words string) { + scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running, Heard: c.heardNow(), Kept: func(module, words string) { keptMu.Lock() defer keptMu.Unlock() if !slices.Contains(keptRuns[module], words) { @@ -1861,7 +1865,7 @@ func (c *Composer) ComposedToday(ctx context.Context, module, fp string) (string if c.env.Running != nil { running = c.env.Running() } - scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running} + scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running, Heard: c.heardNow()} comps, _, held, err := frombus.Compose(ctx, reader{c: c}, c.env.Mesh, scope, fps) if err != nil || held[module] != "" { return "", false @@ -2272,3 +2276,78 @@ func (c *Composer) WhyUnmade(module string) string { defer c.mu.Unlock() return c.last.Unmade[module] } + +// heardRun is one running event as the composer keeps it, with when it was said. +type heardRun struct { + at time.Time + fingerprint string + manifest string + stopped bool +} + +// NoteRunning takes the seat node-engine's event running of a node, this one's included (design 47 G §199): a module's +// newest by `at` stands, an older one changes nothing (§556). A change of what a node runs asks a pass once the first read of +// the stream has ended; the same value again asks none. +func (c *Composer) NoteRunning(ev nodeengine.Running) { + if ev.Node == "" || ev.Module == "" { + return + } + run := heardRun{at: ev.At, fingerprint: ev.Fingerprint, manifest: ev.Manifest, stopped: ev.Stopped} + c.mu.Lock() + had, ok := c.heard[ev.Node][ev.Module] + if ok && (ev.At.Before(had.at) || had == run) { + c.mu.Unlock() + return + } + if c.heard == nil { + c.heard = map[string]map[string]heardRun{} + } + if c.heard[ev.Node] == nil { + c.heard[ev.Node] = map[string]heardRun{} + } + c.heard[ev.Node][ev.Module] = run + read := c.heardRead + c.mu.Unlock() + if read { + c.askRehand() + } +} + +// RunningRead says the first read of the stream ended: from now the composer acts on what it heard, and a pass is asked. +func (c *Composer) RunningRead() { + c.mu.Lock() + c.heardRead = true + c.mu.Unlock() + c.askRehand() +} + +// RunningLost says the stream is not followed any more: nothing heard is acted on until a new read ends. +func (c *Composer) RunningLost() { + c.mu.Lock() + c.heardRead = false + c.mu.Unlock() +} + +func (c *Composer) askRehand() { + select { + case c.rehand <- struct{}{}: + default: + } +} + +// heardNow is what the nodes said they run, as the passing of a seat reads it; nil until the first read of the stream ended. +func (c *Composer) heardNow() map[string]map[string]frombus.Heard { + c.mu.Lock() + defer c.mu.Unlock() + if !c.heardRead || len(c.heard) == 0 { + return nil + } + out := make(map[string]map[string]frombus.Heard, len(c.heard)) + for node, modules := range c.heard { + out[node] = make(map[string]frombus.Heard, len(modules)) + for module, h := range modules { + out[node][module] = frombus.Heard{Fingerprint: h.fingerprint, Manifest: h.manifest, Stopped: h.stopped} + } + } + return out +} -- 2.54.0 From a073a001c52ab21c1107f6708f9223afd056f2a4 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:00:21 +0100 Subject: [PATCH 08/59] H-7: the node record states the capability want-parts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §525, §94, §192 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- internal/link/have_test.go | 6 +++--- internal/link/record.go | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/internal/link/have_test.go b/internal/link/have_test.go index caaa39fc..041bc536 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -334,7 +334,7 @@ func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) { t.Fatalf("the node key holds no record: %+v, %v", got, err) } wantRecord := nodeengine.NodeRecord{V: sdkkeys.Version, Node: "n1", TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", - Certificate: "cert", Trusts: keys.Trusts, Capabilities: []string{"apt", "bus-state-calls", "ledger-calls", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true, + Certificate: "cert", Trusts: keys.Trusts, Capabilities: []string{"apt", "bus-state-calls", "ledger-calls", "systemd", CapabilityWantParts}, OutwardLinks: []string{"eth0"}, NetworksSaid: true, Networks: []nodeengine.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, Contract: NodeKeyContract, ManifestReads: catalogue.ManifestReads()} // what it reads of a manifest: its contract's (ADR 0614 §4) if !nodeengineclient.SameNodeRecord(*got.Record, wantRecord) { @@ -350,7 +350,7 @@ func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) { t.Errorf("a report changing nothing wrote the node key again") } // Design 47: the node's capabilities, said when they change, on capabilities-changed.. - if got := l.saidOn("mesh.seat.node-engine.event.capabilities-changed.n1 "); len(got) != 1 || !strings.Contains(got[0], `"capabilities":["apt","bus-state-calls","ledger-calls","systemd"]`) { + if got := l.saidOn("mesh.seat.node-engine.event.capabilities-changed.n1 "); len(got) != 1 || !strings.Contains(got[0], `"capabilities":["apt","bus-state-calls","ledger-calls","systemd","want-parts"]`) { t.Errorf("capabilities-changed said %v, want once for the record's capabilities", got) } // A runner that does not answer states what it stated before; one answering without any states none (an older runner). @@ -362,7 +362,7 @@ func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) { } l.statedErr, l.stated = nil, map[string]string{} q.SayFacts(context.Background(), rep) - if got, _ = nodeengineclient.DecodeHaveNode(l.kv["n1.node"].value); l.kv["n1.node"].rev != at+1 || !slices.Equal(got.Record.Capabilities, []string{"apt", "systemd"}) { + if got, _ = nodeengineclient.DecodeHaveNode(l.kv["n1.node"].value); l.kv["n1.node"].rev != at+1 || !slices.Equal(got.Record.Capabilities, []string{"apt", "systemd", "want-parts"}) { t.Errorf("a runner stating no calls: revision %d (was %d), capabilities %v", l.kv["n1.node"].rev, at, got.Record.Capabilities) } at++ diff --git a/internal/link/record.go b/internal/link/record.go index 6ce89833..bd93ea82 100644 --- a/internal/link/record.go +++ b/internal/link/record.go @@ -37,6 +37,10 @@ type RecordKeys struct { // buildReads are the words and forms this engine reads of a manifest (novox/hq ADR 0614 §4): the reader capabilities of // the mesh-sdk it is built with, its seats left out. Computed once: they are the build's, and never change while it runs. +// CapabilityWantParts is the capability a node-engine states in its node record when it reads the want's version, parts and +// kept-on (design 47 G §525); the assigner writes them into a node's want only while the record states it. +const CapabilityWantParts = "want-parts" + var buildReads = sync.OnceValue(func() []string { return slices.DeleteFunc(catalogue.ManifestReads(), func(r string) bool { return strings.HasPrefix(r, catalogue.ReadSeat) }) }) @@ -125,6 +129,8 @@ func recordOf(node string, r Report, keys RecordKeys) *nodeengine.NodeRecord { rec.Capabilities = append(rec.Capabilities, c.Name) } } + // A node-engine of G states that it reads the want's new fields: version, parts and where a kept build is kept (design 47 G §525). + rec.Capabilities = append(rec.Capabilities, CapabilityWantParts) for _, c := range keys.Stated { if c != "" && !slices.Contains(rec.Capabilities, c) { rec.Capabilities = append(rec.Capabilities, c) -- 2.54.0 From 602ed830e9f20562004a4e85816bb61032f51661 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:05:43 +0100 Subject: [PATCH 09/59] H-15: the apply guard and the yields read the node-engine's module from the entry claiming its seat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The module is the one whose catalog entry claims node-engine, else the one this node heard run the build its own fingerprint names; no name is fixed in the guard or in the compose. Builds: design 47 G §364, §527, §428 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- cmd/mesh-host/nodeagent.go | 2 ++ internal/apply/placement_guard.go | 27 ++++++++++++++-- internal/apply/placement_guard_test.go | 20 ++++++++++++ internal/compose/frombus/frombus.go | 35 +++++++++++++++++++++ internal/compose/frombus/frombus_test.go | 14 +++++++++ internal/compose/resolution/placement.go | 13 ++++++-- internal/compose/resolution/runtime_test.go | 17 +++++++--- internal/compose/resolution/yields.go | 6 ++-- internal/selfcompose/selfcompose.go | 22 ++++++++++++- 9 files changed, 143 insertions(+), 13 deletions(-) diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 76873b20..5eecb947 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -670,6 +670,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone session.SetManifestOf(composer.ManifestOf) // the event running names the manifest of the build that runs go followRunning(run, session, composer, say) + apply.UseEngineModule(composer.EngineModule) // the guard of the engine's own trees fixes no name (§527) env.Composed, env.SelfComposed, env.Policy, env.ComposedToday = composer, true, composer.Policy, composer.ComposedToday env.WhyUnmade, env.Waiting = composer.WhyUnmade, composer.Waiting env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does @@ -683,6 +684,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus current.set(nil) composerNow.Store(nil) session.SetManifestOf(nil) + apply.UseEngineModule(nil) shadowNow.Store(nil) if ctx.Err() != nil { return diff --git a/internal/apply/placement_guard.go b/internal/apply/placement_guard.go index 864c4cb4..248dda54 100644 --- a/internal/apply/placement_guard.go +++ b/internal/apply/placement_guard.go @@ -54,6 +54,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "git.novox.be/novox/mesh-sdk/go/declaration" @@ -303,8 +304,28 @@ func splitQuoted(s string) []string { return out } -// engineModule is the module whose resources may place in the engine's own trees. -const engineModule = "mesh-host" +// engineModuleOf answers the module whose resources may place in the engine's own trees: the module whose entry of the +// catalog half claims the seat node-engine (design 47 G §527), asked of the node's composer; none until one is given. +var engineModuleOf atomic.Pointer[func() string] + +// UseEngineModule gives the apply guard the function that answers the node-engine's own module; nil takes it away. The +// guard fixes no name (§364, §527). +func UseEngineModule(f func() string) { + if f == nil { + engineModuleOf.Store(nil) + return + } + engineModuleOf.Store(&f) +} + +// isEngineModule is whether module is the node-engine's own. +func isEngineModule(module string) bool { + f := engineModuleOf.Load() + if f == nil || module == "" { + return false + } + return (*f)() == module +} // passwdFile is the user database homes are read from. A variable so a test names its own. var passwdFile = "/etc/passwd" @@ -531,7 +552,7 @@ func refuseOne(path string, kind placing, module, owner string, declared map[str return &PlacementRefusedError{Path: path, Why: "nothing is placed in or mounted from " + tree} } } - if module != engineModule { + if !isEngineModule(module) { for _, tree := range engineTrees { if atOrBelow(path, tree) { return &PlacementRefusedError{Path: path, Why: tree + " is the node-engine's own, placed in by " + diff --git a/internal/apply/placement_guard_test.go b/internal/apply/placement_guard_test.go index 156563c3..8bff411d 100644 --- a/internal/apply/placement_guard_test.go +++ b/internal/apply/placement_guard_test.go @@ -193,4 +193,24 @@ func TestAModulesOwnPlacesAreStillPlaced(t *testing.T) { if err := refusePath("/var/lib/notes/data", placingDirectory, "notes", "1001:2000"); err != nil { t.Errorf("an owner below /var/lib: %v", err) } + // Design 47 G §364, §527: the node-engine's own trees are placed in by the module the composer says claims its seat, + // whatever that module is called, and by no other; with none said, by nobody. + tree := engineTrees[0] + t.Cleanup(func() { UseEngineModule(nil) }) + for _, c := range []struct { + engine func() string + module string + placed bool + }{ + {func() string { return "the-engine" }, "the-engine", true}, + {func() string { return "the-engine" }, "notes", false}, + {func() string { return "the-engine" }, "", false}, + {func() string { return "" }, "", false}, + {nil, "the-engine", false}, + } { + UseEngineModule(c.engine) + if err := refusePath(tree, placingDirectory, c.module, ""); (err == nil) != c.placed { + t.Errorf("%s for module %q: %v, want placed %v", tree, c.module, err, c.placed) + } + } } diff --git a/internal/compose/frombus/frombus.go b/internal/compose/frombus/frombus.go index 7af21b84..5e090c09 100644 --- a/internal/compose/frombus/frombus.go +++ b/internal/compose/frombus/frombus.go @@ -160,6 +160,13 @@ type Scope struct { // composing node's own consumer (design 47 G §199): the passing of a seat reads it, and a module nothing was heard of // counts by the trunk's entry. Never read from a have key, a want or a call. Heard map[string]map[string]Heard + // Engine is told, at each composing of the node's input, the node-engine's own module: the one whose entry of the catalog half + // claims the seat node-engine, else the one of the node's heard modules that runs the build OwnFingerprint names (design 47 G + // §527); "" when neither shows it. Nil: told to nobody. + Engine func(module string) + // OwnFingerprint is the fingerprint this node-engine was started at (its MESH_FINGERPRINT); empty when it was started + // without one, as an engine of before G. + OwnFingerprint string // HeldToday is, in an answer, the modules the node keeps today as they run ("node" for its own composition), each with // how ("held", "failed", "runs a build of the old form"): a candidate that cannot compose one, or holds one only // behind one of them, keeps it as today, which is said under CandidateAnswer.Held and is no refusal (issue 1126). @@ -593,6 +600,31 @@ type Heard struct { Stopped bool } +// EngineModule is the node-engine's own module (design 47 G §527): the module whose catalog entry claims the seat node-engine, +// which the validator makes one per revision (§508). With none claiming it, which only a revision of before G shows, as at a +// revert of G, the module this node heard run the build its own fingerprint names; "" when neither shows it. +func EngineModule(in compose.Input, node, ownFingerprint string) string { + for _, name := range sortedKeys(in.Catalogue) { + if resolution.ClaimsEngine(in.Catalogue[name]) { + return name + } + } + if ownFingerprint == "" { + return "" + } + for i := range in.Nodes { + if in.Nodes[i].Name != node { + continue + } + for _, module := range sortedKeys(in.Nodes[i].Heard) { + if in.Nodes[i].Heard[module].Fingerprint == ownFingerprint { + return module + } + } + } + return "" +} + // keepHeard puts, as each node's Heard, what it said it runs of each module it is assigned, with the manifest of the build // read by the event's digest; a digest that cannot be read leaves the entry unread, and the passing says so (§459). func keepHeard(in *compose.Input, s Scope, read func(module, digest string) (catalogue.Manifest, error)) { @@ -643,6 +675,9 @@ func Compose(ctx context.Context, r Reader, mesh Mesh, s Scope, fps map[string]s if err != nil { return nil, nil, held, err } + if s.Engine != nil { + s.Engine(EngineModule(in, s.Node, s.OwnFingerprint)) + } if s.Holds != nil { holds := map[string]compose.Hold{} for name := range held { diff --git a/internal/compose/frombus/frombus_test.go b/internal/compose/frombus/frombus_test.go index c34bbd50..5ea996e9 100644 --- a/internal/compose/frombus/frombus_test.go +++ b/internal/compose/frombus/frombus_test.go @@ -19,6 +19,7 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "github.com/novox/mesh-host/internal/compositions" + "maps" "os" "reflect" "slices" @@ -592,6 +593,19 @@ func TestInput(t *testing.T) { t.Errorf("anchor's Heard holds %+v, want f2 unread", n.Heard) } } + // Design 47 G §527: the node-engine's own module is the one whose entry claims its seat; with none, the one this node heard + // run the build its own fingerprint names; with neither, none. + if got := EngineModule(in, "laptop", "f1"); got != "xterm" { + t.Errorf("by the own fingerprint: %q, want xterm", got) + } + if got := EngineModule(in, "laptop", "other"); got != "" { + t.Errorf("an unheard fingerprint names %q", got) + } + in.Catalogue = maps.Clone(in.Catalogue) + in.Catalogue["the-engine"] = catalogue.Manifest{Module: "the-engine", Claims: []catalogue.Claim{{Name: "node-engine"}}} + if got := EngineModule(in, "laptop", "f1"); got != "the-engine" { + t.Errorf("by the claim: %q, want the-engine", got) + } // Issue 1126, the seed's refusals. restic and the provisioner's client as the catalogue holds them (mesh-catalog // c05aa5e6), restic moved with an own secret the vault leaves out (it hands it to the client, mesh-vault ownnode.go // handed): composed from the indexer's manifest restic is not handed and stands in for that secret, as novox's, diff --git a/internal/compose/resolution/placement.go b/internal/compose/resolution/placement.go index 1c65ca49..d94f0b34 100644 --- a/internal/compose/resolution/placement.go +++ b/internal/compose/resolution/placement.go @@ -7,6 +7,7 @@ import ( "sort" "strings" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" mcat "git.novox.be/novox/mesh-sdk/go/catalogue" ) @@ -56,8 +57,16 @@ func systemPath(path string) string { return "" } -// engineModule is the node-engine's own module, the one that places in engineTrees. -const engineModule = "mesh-host" +// ClaimsEngine is whether a manifest claims the seat node-engine: the node-engine's own module is the one whose entry of +// the catalog half does (design 47 G §527), never a name fixed in code. +func ClaimsEngine(m mcat.Manifest) bool { + for _, c := range m.Claims { + if c.Name == nodeengine.Seat { + return true + } + } + return false +} // accessRef is how a module names one of its accesses: ${access:}. var accessRef = regexp.MustCompile(`\$\{access:([a-z0-9][a-z0-9-]*)\}`) diff --git a/internal/compose/resolution/runtime_test.go b/internal/compose/resolution/runtime_test.go index ba0932fd..da6ae5bc 100644 --- a/internal/compose/resolution/runtime_test.go +++ b/internal/compose/resolution/runtime_test.go @@ -310,16 +310,25 @@ func TestAModuleOfItsOwnAccountIsServedByARuntimeOfItsOwn(t *testing.T) { } gob, img := builder("gob", "node-build-go"), builder("img", "node-build-image") with.Needed["gob"], with.Needed["img"] = map[string]string{"broker": "own"}, map[string]string{"broker": "own"} + // The node-engine's module is the one whose entry claims its seat (design 47 G §527), whatever it is called; with none in + // the resolution its settings are not found and the default stands. + engine := mcat.Manifest{Module: "mesh-host", Claims: []mcat.Claim{{Name: "node-engine"}}} for _, c := range []struct { settings mcat.SettingsBy share string + noEngine bool }{ - {nil, "50"}, - {mcat.SettingsBy{"mesh-host": {{From: "anchor", Values: map[string]any{"build-cpu-share": float64(75)}}}}, "75"}, - {mcat.SettingsBy{"mesh-host": {{From: "anchor", Values: map[string]any{"build-cpu-share": float64(0)}}}}, "50"}, + {nil, "50", false}, + {mcat.SettingsBy{"mesh-host": {{From: "anchor", Values: map[string]any{"build-cpu-share": float64(75)}}}}, "75", false}, + {mcat.SettingsBy{"mesh-host": {{From: "anchor", Values: map[string]any{"build-cpu-share": float64(0)}}}}, "50", false}, + {mcat.SettingsBy{"mesh-host": {{From: "anchor", Values: map[string]any{"build-cpu-share": float64(75)}}}}, "50", true}, } { with.Settings = c.settings - runtimes, err := resolutionOwnRuntimes(mcat.Resolution{Node: "anchor", Modules: []mcat.Manifest{gob, img, telegram, goRuntime}}, with) + modules := []mcat.Manifest{gob, img, telegram, goRuntime, engine} + if c.noEngine { + modules = modules[:4] + } + runtimes, err := resolutionOwnRuntimes(mcat.Resolution{Node: "anchor", Modules: modules}, with) if err != nil { t.Fatal(err) } diff --git a/internal/compose/resolution/yields.go b/internal/compose/resolution/yields.go index 32f70cf8..41076f1d 100644 --- a/internal/compose/resolution/yields.go +++ b/internal/compose/resolution/yields.go @@ -21,14 +21,14 @@ func resolutionYieldsOf(r mcat.Resolution, m mcat.Manifest, with mcat.Rendering) if !yields(m) { return nil, nil } - engine := mcat.Manifest{Module: engineModule} + var engine mcat.Manifest // the module claiming the seat node-engine; none in the resolution leaves the default for _, e := range r.Modules { - if e.Module == engineModule { + if ClaimsEngine(e) { engine = e } } share := mcat.BuildCPUDefaultPercent - for _, s := range Effective(engine, with.Settings[engineModule]) { + for _, s := range Effective(engine, with.Settings[engine.Module]) { if s.Key == mcat.BuildCPUShareSetting { if n, ok := percentOf(s.Value); ok { share = n diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index cba3a1f2..d4c84594 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -33,6 +33,7 @@ import ( "github.com/novox/mesh-host/internal/compositions" engcomp "github.com/novox/mesh-host/internal/compositions" "maps" + "os" "reflect" "slices" "sort" @@ -180,6 +181,8 @@ type Composer struct { // node's own consumer (NoteRunning), under mu; heardRead is whether the first read of the stream ended (RunningRead). heard map[string]map[string]heardRun heardRead bool + // engine is the node-engine's own module as the last composing of the node's input found it (Scope.Engine), under mu. + engine string // rehand asks for a pass that hands again what the agent waits for (Waiting). rehand chan struct{} // sealedOwn is each own secret's sealed value as first handed, by module/name and generation: a value sealed again @@ -1002,7 +1005,7 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { var keptMu sync.Mutex var holds map[string]compose.Hold fromManifest := map[string]string{} // by module, why it is composed from the indexer's manifest and not a build - scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running, Heard: c.heardNow(), Kept: func(module, words string) { + scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running, Heard: c.heardNow(), Engine: c.noteEngine, OwnFingerprint: os.Getenv(engcomp.FingerprintVar), Kept: func(module, words string) { keptMu.Lock() defer keptMu.Unlock() if !slices.Contains(keptRuns[module], words) { @@ -2351,3 +2354,20 @@ func (c *Composer) heardNow() map[string]map[string]frombus.Heard { } return out } + +// noteEngine keeps the node-engine's own module a composing of the node's input found; one that finds none changes nothing. +func (c *Composer) noteEngine(module string) { + if module == "" { + return + } + c.mu.Lock() + c.engine = module + c.mu.Unlock() +} + +// EngineModule is the node-engine's own module (design 47 G §527), for the apply guard; "" until a composing found it. +func (c *Composer) EngineModule() string { + c.mu.Lock() + defer c.mu.Unlock() + return c.engine +} -- 2.54.0 From ce01ec1f681361b21fe94d8fade316fe30a23fe5 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:15:16 +0100 Subject: [PATCH 10/59] H-13: the node-engine's findings are written and said by mesh-sdk's finding writer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A finding is written in the node's state findings and followed by finding-written or finding-cleared on the engine's own stream mesh-host-findings; the engine states every finding it observes at its start (the seat list's finding only while the reader waits for the list, else not) and when the keeper asks, and answers an ask by a new statement. The node's grant in the golden file gains the stream's publish, the ask stream's consumer, and the whole of its state findings. Builds: design 47 G §383, §384, §390, §461, §485, §503 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- cmd/mesh-host/nodeagent.go | 9 +- cmd/mesh-host/nodeagent_integration_test.go | 38 +++++ cmd/mesh-host/seatlist.go | 167 ++++++++++++++++---- cmd/mesh-host/seatlist_unread_test.go | 6 +- internal/link/findings.go | 115 ++++++++++++++ internal/nodegrant/testdata/node.json | 23 +-- internal/seatwait/seatwait.go | 3 + 7 files changed, 319 insertions(+), 42 deletions(-) diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 5eecb947..ea3f3b4a 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -593,15 +593,18 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus own := &agentNow{} // And the seat list it reads through the seat mesh-catalog, from its bootstrapped definitions until it has: the // composer hands nothing new meanwhile, and the node says it has waited five minutes (novox/hq ADR 0636 §1). - seats, err := startSeatList(run, session, node, say) + findings := newFindingsOfTheNode(session, node) + seats, err := startSeatList(run, session, node, findings, say) if err != nil { // No reader is no gate that lets everything through: the node acts on nothing new, as one that waits does. say(err.Error() + "; the node composes nothing new until it is started again and reads the list (novox/hq ADR 0636 §1)") } // The marks of the haves run off the composition pass, each bounded (see marker). marks := newMarker(run) - findings := findingsOfTheNode{session} - heldFindings := &seatwait.HeldKeeper{Node: node, Observer: "node-engine", Writer: findings, + if werr := findings.Start(run, say); werr != nil { + say("the node-engine's findings are kept in its state and not said as events until it is started again: " + werr.Error()) + } + heldFindings := &seatwait.HeldKeeper{Node: node, Observer: findingsModule, Writer: findings, Left: func() ([]views.Finding, error) { return findings.left(node) }, Say: say} hooks := composerHooks{ seatList: seats, diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index 44c753ad..226c0e1b 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -11,6 +11,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" @@ -159,6 +160,13 @@ func newItBedAs(t *testing.T, perms nodegrant.Permissions) *itBed { if b.findings, err = b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: link.FindingsBucket, History: 1}); err != nil { t.Fatal(err) } + // The engine's own finding stream and the keeper's ask stream (design 47 G §383, §390), made as the bus module makes them. + for _, c := range []nats.StreamConfig{{Name: "MESH-HOST-FINDINGS", Subjects: []string{"mesh.stream.mesh-host.event.>"}}, + {Name: condseat.StreamFindingsAsked, Subjects: []string{condseat.SubjectFindingsAsked}}} { + if _, err = b.js.AddStream(&c); err != nil { + t.Fatal(err) + } + } b.standInSeat() b.opts = options{state: filepath.Join(b.dir, "state", "state.json"), timeout: 2 * time.Second} if err := os.MkdirAll(filepath.Dir(b.opts.state), 0o755); err != nil { @@ -1037,6 +1045,27 @@ func TestNatsANodeThatCannotReadTheSeatListActsOnNothingNew(t *testing.T) { if b.file("web") != "" { t.Fatalf("the node applied web with its finding written: %q", b.file("web")) } + // Design 47 G §383, §384, §390 (test 107): the key is followed by the event on the engine's own stream, the engine stated + // its findings at its start, and an ask of the keeper is answered by a new statement. + lastEvent := func(event string) (string, bool) { + m, err := b.js.GetLastMsg("MESH-HOST-FINDINGS", findingSubject(event, itNode)) + if err != nil { + return "", false + } + return string(m.Data), true + } + within(t, 5*time.Second, "the node did not say finding-written after its key", func() bool { + got, ok := lastEvent("finding-written") + return ok && strings.Contains(got, `"kind":"seat-list-unread"`) && strings.Contains(got, `"observer":"mesh-host"`) + }) + within(t, 5*time.Second, "the node did not state its findings at its start", func() bool { _, ok := lastEvent("findings-stated"); return ok }) + if _, err := b.js.Publish(condseat.SubjectFindingsAsked, []byte(`{"ask":"a1","at":"2026-11-11T12:00:00Z","by":"test"}`)); err != nil { + t.Fatal(err) + } + within(t, 5*time.Second, "the node did not answer the keeper's ask with a statement naming the finding", func() bool { + got, ok := lastEvent("findings-stated") + return ok && strings.Contains(got, `"asked":"a1"`) && strings.Contains(got, `"kind":"seat-list-unread"`) + }) b.seatSilent.Store(false) // The hand act of the operator and the healers (the verb seat-list-again): a read now, as the seat's event would make. @@ -1048,6 +1077,15 @@ func TestNatsANodeThatCannotReadTheSeatListActsOnNothingNew(t *testing.T) { _, err := b.findings.Get(finding) return errors.Is(err, nats.ErrKeyNotFound) }) + within(t, 5*time.Second, "the node did not say finding-cleared for the seat list", func() bool { + for i := uint64(1); i < 40; i++ { + if m, err := b.js.GetMsg("MESH-HOST-FINDINGS", i); err == nil && m.Subject == findingSubject("finding-cleared", itNode) && + strings.Contains(string(m.Data), `"kind":"seat-list-unread"`) { + return true + } + } + return false + }) // Started again with the seat refusing: nothing new is composed, and what ran is neither removed nor changed. stop() diff --git a/cmd/mesh-host/seatlist.go b/cmd/mesh-host/seatlist.go index dd07ddb6..c53d423c 100644 --- a/cmd/mesh-host/seatlist.go +++ b/cmd/mesh-host/seatlist.go @@ -5,7 +5,10 @@ import ( "encoding/json" "errors" "fmt" + "slices" + "sort" "strings" + "sync" "sync/atomic" "time" @@ -103,29 +106,80 @@ func (realClock) AfterFunc(d time.Duration, f func()) seatwait.Timer { return time.AfterFunc(d, f) } -// findingsOfTheNode writes the node's findings in its state `findings`, in the layout of the seat mesh-conditions. -type findingsOfTheNode struct{ session *link.Session } +// findingsModule is the module whose own stream the node-engine says its findings on, and whose name its findings carry as +// their observer (module.json's `module` and its stream `mesh-host-findings`, design 47 G §383). +const findingsModule = "mesh-host" -func (w findingsOfTheNode) encode(f views.Finding) (string, []byte, error) { - // The seat list's finding is of the machine; a held module's is of the module on a machine, `.`. - scope, machine, source := conditions.ScopeMachine, f.ID, "seat-list-unread" +// findingsOfTheNode keeps the node's findings through mesh-sdk's finding writer (design 47 G §383, §384, §461): each finding is +// written in the node's state `findings` and said as an event on the engine's own stream, the engine says every finding it +// observes at its start and when the keeper asks, and the keeper reads no state of the engine's. +type findingsOfTheNode struct { + session *link.Session + node string + + mu sync.Mutex + ctx context.Context + writer *views.Writer +} + +func newFindingsOfTheNode(session *link.Session, node string) *findingsOfTheNode { + return &findingsOfTheNode{session: session, node: node} +} + +// findingSubject is a stream event's subject for the observer's own node (design 47 G §383, ADR 0363 §2). +func findingSubject(event, node string) string { + return "mesh.stream." + findingsModule + ".event." + event + "." + node +} + +// Start makes the writer and states the findings now, until ctx ends. A writer that cannot start (the link has no stream +// yet) is said; the findings are then kept in the state alone, and said by the next start. +func (w *findingsOfTheNode) Start(ctx context.Context, say func(string)) error { + writer, err := views.NewWriter(views.WriterConfig{ + Module: findingsModule, + State: link.FindingsState{Session: w.session}, + Subjects: views.WriterSubjects{Written: findingSubject("finding-written", w.node), Cleared: findingSubject("finding-cleared", w.node), + Stated: findingSubject("findings-stated", w.node), Asked: condseat.SubjectFindingsAsked}, + Observe: w.observe, + Logf: func(format string, args ...any) { say(fmt.Sprintf(format, args...)) }, + Publish: w.session.PublishEvent, + Listen: func(_ context.Context, subject string, handle func([]byte)) (func(), error) { + return w.session.ListenFrom(condseat.StreamFindingsAsked, subject, handle) + }, + }) + if err != nil { + return err + } + w.mu.Lock() + w.ctx, w.writer = ctx, writer + w.mu.Unlock() + go func() { <-ctx.Done(); writer.Close() }() + return writer.Start(ctx) +} + +func (w *findingsOfTheNode) started() (context.Context, *views.Writer, error) { + w.mu.Lock() + defer w.mu.Unlock() + if w.writer == nil { + return nil, nil, errors.New("the node-engine's finding writer is not started") + } + return w.ctx, w.writer, nil +} + +// findingBody is the finding in the layout of the seat mesh-conditions (design 47 G §439). The seat list's finding is of the +// machine; a held module's is of the module on a machine, `.`. The writer names the observer. +func findingBody(f views.Finding) views.FindingBody { + scope, machine := conditions.ScopeMachine, f.ID if f.Kind == seatwait.HeldKind { - scope, source = conditions.ScopeModule, seatwait.HeldKind + scope = conditions.ScopeModule machine = f.ID[strings.LastIndex(f.ID, ".")+1:] } - key := conditions.Key(scope, f.ID, f.Kind) - v := condseat.FindingsByKeyValue{V: 1, Key: key, Kind: f.Kind, Observer: f.Observer, Severity: conditions.Warning, - Subject: condseat.Subject{Scope: scope, ID: f.ID, Machine: machine}, Summary: f.Summary, - Headline: f.Headline, Explanation: f.Explanation, Source: source, At: f.At.UTC()} - if err := conditions.CheckFinding(v); err != nil { - return "", nil, err - } - raw, err := json.Marshal(v) - return key, raw, err + return views.FindingBody{Kind: f.Kind, Scope: scope, ID: f.ID, At: f.At.UTC(), Severity: conditions.Warning, + Summary: f.Summary, Headline: f.Headline, Explanation: f.Explanation, Source: f.Kind, + Subject: views.FindingSubject{Scope: scope, ID: f.ID, Machine: machine}} } // left is the findings of held modules an earlier run of this engine left on node: its keys of this kind. -func (w findingsOfTheNode) left(node string) ([]views.Finding, error) { +func (w *findingsOfTheNode) left(node string) ([]views.Finding, error) { keys, err := w.session.FindingKeys() if err != nil { return nil, err @@ -139,37 +193,96 @@ func (w findingsOfTheNode) left(node string) ([]views.Finding, error) { if id, ok = strings.CutSuffix(id, "."+seatwait.HeldKind); !ok || !strings.HasSuffix(id, "."+node) { continue } - out = append(out, views.Finding{Observer: "node-engine", Kind: seatwait.HeldKind, ID: id}) + out = append(out, views.Finding{Observer: findingsModule, Kind: seatwait.HeldKind, ID: id}) } return out, nil } -func (w findingsOfTheNode) Put(f views.Finding) error { - key, raw, err := w.encode(f) +func (w *findingsOfTheNode) Put(f views.Finding) error { + ctx, writer, err := w.started() if err != nil { return err } - return w.session.PutFinding(key, raw) + return writer.Write(ctx, findingBody(f)) } // keyOf is the finding's key in the state; a read or a clear needs no more of the finding than Kind and ID. -func (w findingsOfTheNode) keyOf(f views.Finding) string { - return conditions.Key(conditions.ScopeMachine, f.ID, f.Kind) +func (w *findingsOfTheNode) keyOf(f views.Finding) string { + b := findingBody(f) + return b.Key() } -func (w findingsOfTheNode) Delete(f views.Finding) error { - return w.session.DeleteFinding(w.keyOf(f)) +func (w *findingsOfTheNode) Delete(f views.Finding) error { + ctx, writer, err := w.started() + if err != nil { + // Before the writer starts, which is at its start the one statement that says what this engine observes: the key goes, + // and that statement names no finding of it, so the keeper clears it (§384). + return w.session.DeleteFinding(w.keyOf(f)) + } + b := findingBody(f) + why := "the node-engine no longer observes it" + switch f.Kind { + case seatwait.Kind: + why = "the node-engine has read the whole seat list" + case seatwait.HeldKind: + why = "the module is composed again or is no longer wanted on the node" + } + return writer.Clear(ctx, b.Scope, b.ID, b.Kind, why) } // Has says whether the finding stands in the node's state `findings` now. -func (w findingsOfTheNode) Has(f views.Finding) (bool, error) { +func (w *findingsOfTheNode) Has(f views.Finding) (bool, error) { return w.session.HasFinding(w.keyOf(f)) } +// observe is the node-engine's own truth for a statement (§384): the findings that stand in its state and are still true +// (the seat list's only while the reader waits for it, so an engine started with the full list states none), and the seat +// list's finding as pending while its five minutes run and none stands yet. +func (w *findingsOfTheNode) observe(context.Context) (views.Observed, error) { + var out views.Observed + st := seatListNow.Load() + waits := st != nil && seatwait.Waits(st.reader.Status()) + keys, err := w.session.FindingKeys() + if err != nil { + return out, err + } + sort.Strings(keys) + for _, key := range keys { + scope, rest, ok := strings.Cut(key, ".") + i := strings.LastIndex(rest, ".") + if !ok || i < 0 { + continue + } + id, kind := rest[:i], rest[i+1:] + if kind == seatwait.Kind && !waits { + continue // read in full: it is no finding of this engine now, and its key goes with this statement + } + e, err := link.FindingsState{Session: w.session}.Get(key) + if err != nil || e == nil { + continue + } + var body views.FindingBody + if json.Unmarshal(e.Value, &body) != nil { + continue + } + body.Scope, body.ID, body.Kind = scope, id, kind + out.Findings = append(out.Findings, body) + } + if waits { + if stands, since := st.waiter.Waiting(); stands { + key := conditions.Key(conditions.ScopeMachine, w.node, seatwait.Kind) + if !slices.Contains(keys, key) { + out.Pending = append(out.Pending, views.PendingKey{Key: key, Since: since.UTC(), Bound: seatwait.Bound.String()}) + } + } + } + return out, nil +} + // startSeatList starts this run's reader: its bootstrapped seats are given to catalogue at once, and every change of what // it holds again, saying so to the composer and to the waiter that writes the five-minute finding, and stating again what // the engine reads (link.Session.RestateReads, ADR 0618 §4): the seats it states are those of the list it holds. -func startSeatList(ctx context.Context, session *link.Session, node string, say func(string)) (*seatListState, error) { +func startSeatList(ctx context.Context, session *link.Session, node string, findings seatwait.Findings, say func(string)) (*seatListState, error) { st := &seatListState{changed: make(chan struct{}, 1)} // Start returns at once and reads the seat on a goroutine, which may announce a change before the waiter exists. var announce atomic.Pointer[func()] @@ -187,7 +300,7 @@ func startSeatList(ctx context.Context, session *link.Session, node string, say return nil, fmt.Errorf("the node-engine cannot follow the seat list: %w", err) } st.reader = reader - st.waiter = &seatwait.Waiter{Node: node, Observer: "node-engine", Reader: reader, Findings: findingsOfTheNode{session}, + st.waiter = &seatwait.Waiter{Node: node, Observer: findingsModule, Reader: reader, Findings: findings, Clock: seatWaitClock, Say: say} told := func() { st.waiter.Changed() diff --git a/cmd/mesh-host/seatlist_unread_test.go b/cmd/mesh-host/seatlist_unread_test.go index 3e76ac4c..a9a8cb18 100644 --- a/cmd/mesh-host/seatlist_unread_test.go +++ b/cmd/mesh-host/seatlist_unread_test.go @@ -22,9 +22,9 @@ func TestAReaderThatCouldNotStartHoldsTheComposer(t *testing.T) { t.Errorf("a composer with no reader asked for gates by %v, want none (the shadow)", got) } // The finding of a held module is of the module on its machine, in a layout the condition keeper accepts (design 47, test 55). - key, raw, err := findingsOfTheNode{}.encode(views.Finding{Observer: "node-engine", Kind: seatwait.HeldKind, + body := findingBody(views.Finding{Observer: "node-engine", Kind: seatwait.HeldKind, ID: seatwait.HeldID("mesh-grants", "ace"), Summary: "mesh-grants is held on ace: x", Headline: "h", Explanation: "e"}) - if err != nil || key != "module.mesh-grants.ace.module-held" || len(raw) == 0 { - t.Errorf("a held module's finding encodes as %q, %v", key, err) + if body.Key() != "module.mesh-grants.ace.module-held" || body.Subject.Machine != "ace" || body.Observer != "" { + t.Errorf("a held module's finding is %+v with key %q", body, body.Key()) } } diff --git a/internal/link/findings.go b/internal/link/findings.go index a4a46390..ab636c1a 100644 --- a/internal/link/findings.go +++ b/internal/link/findings.go @@ -1,9 +1,13 @@ package link import ( + "context" + "encoding/json" "errors" "fmt" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" "github.com/nats-io/nats.go" ) @@ -89,3 +93,114 @@ func (o OverNATS) findingsKV() (nats.KeyValue, error) { } return kv, nil } + +// The node-engine's finding writer (mesh-sdk views.Writer, design 47 G §383, §384, §461) is made over the link open now: +// its state is the bucket above, its events are published on the engine's own stream awaiting the bus's acknowledgement, and +// its consumer of `findings-asked` is its own, from the next ask on. + +// FindingsState is the node-engine's state `findings` over the link open now, as the finding writer reads it +// (views.FindingState). +type FindingsState struct{ Session *Session } + +// Get is the key's entry, or none for a key the state does not hold. +func (s FindingsState) Get(key string) (*stdio.StateEntry, error) { + kv, err := s.Session.findings() + if err != nil { + return nil, err + } + e, err := kv.Get(key) + if errors.Is(err, nats.ErrKeyNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return &stdio.StateEntry{Key: key, Value: e.Value(), Revision: e.Revision()}, nil +} + +// Put writes the value as JSON under the key. +func (s FindingsState) Put(key string, value any) (uint64, error) { + raw, err := json.Marshal(value) + if err != nil { + return 0, err + } + kv, err := s.Session.findings() + if err != nil { + return 0, err + } + return kv.Put(key, raw) +} + +// Delete deletes the key; a key already gone is deleted. +func (s FindingsState) Delete(key string) error { return s.Session.DeleteFinding(key) } + +// Keys lists the keys the state holds. +func (s FindingsState) Keys() ([]string, error) { return s.Session.FindingKeys() } + +var _ views.FindingState = FindingsState{} + +// eventsOver is a link that publishes on a stream and makes a consumer of its own on another. +type eventsOver interface { + publishEvent(ctx context.Context, subject string, data []byte, id string) error + listenFrom(stream, subject string, handle func([]byte)) (func(), error) +} + +func publishEvent(js nats.JetStreamContext, ctx context.Context, subject string, data []byte, id string) error { + _, err := js.Publish(subject, data, nats.MsgId(id), nats.Context(ctx)) + return err +} + +func listenFrom(js nats.JetStreamContext, stream, subject string, handle func([]byte)) (func(), error) { + sub, err := js.Subscribe(subject, func(m *nats.Msg) { handle(m.Data) }, nats.BindStream(stream), nats.DeliverNew(), nats.AckNone()) + if err != nil { + return nil, fmt.Errorf("a consumer of %s: %w", subject, err) + } + return func() { _ = sub.Unsubscribe() }, nil +} + +func (l *natsLink) publishEvent(ctx context.Context, subject string, data []byte, id string) error { + return publishEvent(l.js, ctx, subject, data, id) +} + +func (l *natsLink) listenFrom(stream, subject string, handle func([]byte)) (func(), error) { + return listenFrom(l.js, stream, subject, handle) +} + +func (o OverNATS) publishEvent(ctx context.Context, subject string, data []byte, id string) error { + return publishEvent(o.JS, ctx, subject, data, id) +} + +func (o OverNATS) listenFrom(stream, subject string, handle func([]byte)) (func(), error) { + return listenFrom(o.JS, stream, subject, handle) +} + +func (q *Session) events() (eventsOver, error) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + e, ok := bus.(eventsOver) + if bus == nil || !ok { + return nil, ErrNoLink + } + return e, nil +} + +// PublishEvent publishes one message awaiting the bus's acknowledgement, over the link open now (views.WriterConfig.Publish). +func (q *Session) PublishEvent(ctx context.Context, subject string, data []byte, id string) error { + e, err := q.events() + if err != nil { + return err + } + return e.publishEvent(ctx, subject, data, id) +} + +// ListenFrom makes a consumer of its own on stream for subject that delivers only the messages published after it is +// made, over the link open now, and answers how to end it (views.WriterConfig.Listen). +func (q *Session) ListenFrom(stream, subject string, handle func([]byte)) (func(), error) { + e, err := q.events() + if err != nil { + return nil, err + } + return e.listenFrom(stream, subject, handle) +} diff --git a/internal/nodegrant/testdata/node.json b/internal/nodegrant/testdata/node.json index eded4963..6e82524d 100644 --- a/internal/nodegrant/testdata/node.json +++ b/internal/nodegrant/testdata/node.json @@ -3,8 +3,8 @@ "$JS.ACK.NODES.NODE.>", "$JS.API.CONSUMER.CREATE.KV_composed-NODE.>", "$JS.API.CONSUMER.CREATE.KV_mesh-config_resolved.*.$KV.mesh-config_resolved.NODE.>", - "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_candidates.>", "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_assignments.>", + "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_candidates.>", "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_wants.*.$KV.mesh-delivery-assigner_wants.want.NODE.>", "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-builder_builds.>", "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-indexer_candidates.>", @@ -13,17 +13,19 @@ "$JS.API.CONSUMER.CREATE.KV_mesh-delivery-indexer_verdicts.>", "$JS.API.CONSUMER.CREATE.KV_mesh-host_acts.*.$KV.mesh-host_acts.NODE.>", "$JS.API.CONSUMER.CREATE.KV_mesh-host_data.*.$KV.mesh-host_data.NODE.>", - "$JS.API.CONSUMER.CREATE.KV_mesh-host_findings.*.$KV.mesh-host_findings.machine.NODE.>", + "$JS.API.CONSUMER.CREATE.KV_mesh-host_findings.*.$KV.mesh-host_findings.>", "$JS.API.CONSUMER.CREATE.KV_mesh-host_health.*.$KV.mesh-host_health.NODE", "$JS.API.CONSUMER.CREATE.KV_mesh-host_packages.*.$KV.mesh-host_packages.NODE", "$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_provisions.*.$KV.mesh-provisioner_provisions.NODE.>", "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.*.node", "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.NODE.>", "$JS.API.CONSUMER.CREATE.OBJ_declaration-files-NODE.*.$O.declaration-files-NODE.C.*", + "$JS.API.CONSUMER.CREATE.SEAT_mesh-conditions_findings-asked", + "$JS.API.CONSUMER.CREATE.SEAT_mesh-conditions_findings-asked.>", "$JS.API.CONSUMER.DELETE.KV_composed-NODE.>", "$JS.API.CONSUMER.DELETE.KV_mesh-config_resolved.>", - "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-assigner_candidates.>", "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-assigner_assignments.>", + "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-assigner_candidates.>", "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-assigner_wants.>", "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-builder_builds.>", "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-indexer_candidates.>", @@ -38,11 +40,12 @@ "$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_provisions.>", "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.CONSUMER.DELETE.OBJ_declaration-files-NODE.*", + "$JS.API.CONSUMER.DELETE.SEAT_mesh-conditions_findings-asked.>", "$JS.API.CONSUMER.INFO.NODES.NODE", "$JS.API.DIRECT.GET.KV_composed-NODE.>", "$JS.API.DIRECT.GET.KV_mesh-config_resolved.$KV.mesh-config_resolved.NODE.>", - "$JS.API.DIRECT.GET.KV_mesh-delivery-assigner_candidates.>", "$JS.API.DIRECT.GET.KV_mesh-delivery-assigner_assignments.>", + "$JS.API.DIRECT.GET.KV_mesh-delivery-assigner_candidates.>", "$JS.API.DIRECT.GET.KV_mesh-delivery-assigner_wants.$KV.mesh-delivery-assigner_wants.want.NODE.>", "$JS.API.DIRECT.GET.KV_mesh-delivery-builder_builds.>", "$JS.API.DIRECT.GET.KV_mesh-delivery-indexer_candidates.>", @@ -51,7 +54,7 @@ "$JS.API.DIRECT.GET.KV_mesh-delivery-indexer_verdicts.>", "$JS.API.DIRECT.GET.KV_mesh-host_acts.$KV.mesh-host_acts.NODE.>", "$JS.API.DIRECT.GET.KV_mesh-host_data.$KV.mesh-host_data.NODE.>", - "$JS.API.DIRECT.GET.KV_mesh-host_findings.$KV.mesh-host_findings.machine.NODE.>", + "$JS.API.DIRECT.GET.KV_mesh-host_findings.$KV.mesh-host_findings.>", "$JS.API.DIRECT.GET.KV_mesh-host_health.$KV.mesh-host_health.NODE", "$JS.API.DIRECT.GET.KV_mesh-host_packages.$KV.mesh-host_packages.NODE", "$JS.API.DIRECT.GET.KV_mesh-provisioner_provisions.$KV.mesh-provisioner_provisions.NODE.>", @@ -60,8 +63,8 @@ "$JS.API.DIRECT.GET.OBJ_declaration-files-NODE.$O.declaration-files-NODE.M.*", "$JS.API.STREAM.INFO.KV_composed-NODE", "$JS.API.STREAM.INFO.KV_mesh-config_resolved", - "$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_candidates", "$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_assignments", + "$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_candidates", "$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_wants", "$JS.API.STREAM.INFO.KV_mesh-delivery-builder_builds", "$JS.API.STREAM.INFO.KV_mesh-delivery-indexer_candidates", @@ -75,10 +78,11 @@ "$JS.API.STREAM.INFO.KV_mesh-host_packages", "$JS.API.STREAM.INFO.KV_mesh-provisioner_provisions", "$JS.API.STREAM.INFO.KV_node-engine_have", + "$JS.API.STREAM.INFO.SEAT_mesh-conditions_findings-asked", "$JS.FC.KV_composed-NODE.>", "$JS.FC.KV_mesh-config_resolved.>", - "$JS.FC.KV_mesh-delivery-assigner_candidates.>", "$JS.FC.KV_mesh-delivery-assigner_assignments.>", + "$JS.FC.KV_mesh-delivery-assigner_candidates.>", "$JS.FC.KV_mesh-delivery-assigner_wants.>", "$JS.FC.KV_mesh-delivery-builder_builds.>", "$JS.FC.KV_mesh-delivery-indexer_candidates.>", @@ -96,7 +100,7 @@ "$KV.composed-NODE.>", "$KV.mesh-host_acts.NODE.>", "$KV.mesh-host_data.NODE.>", - "$KV.mesh-host_findings.machine.NODE.>", + "$KV.mesh-host_findings.>", "$KV.mesh-host_health.NODE", "$KV.mesh-host_packages.NODE", "$KV.node-engine_have.NODE.>", @@ -111,7 +115,8 @@ "mesh.seat.node-backup.call.now.NODE.node~NODE", "mesh.seat.node-backup.tool.backed-up.NODE", "mesh.seat.node-backup.tool.now.NODE", - "mesh.seat.node-engine.event.*.NODE" + "mesh.seat.node-engine.event.*.NODE", + "mesh.stream.mesh-host.event.*.NODE" ], "PublishDeny": null, "Subscribe": [ diff --git a/internal/seatwait/seatwait.go b/internal/seatwait/seatwait.go index db6c1c25..54a8f4f9 100644 --- a/internal/seatwait/seatwait.go +++ b/internal/seatwait/seatwait.go @@ -66,6 +66,9 @@ func (w *Waiter) key() views.Finding { func waiting(st seatlist.Status) bool { return !st.Heard || len(st.Waiting) > 0 } +// Waits is whether a reader of that status holds only the definitions it started with, or waits for some. +func Waits(st seatlist.Status) bool { return waiting(st) } + // Changed is told when the reader's held set or its status moved: it starts the bound when the reader waits and has no // bound running, stops it and clears the finding when the reader has read the whole list. func (w *Waiter) Changed() { -- 2.54.0 From 5d3b446988b7cb662b31dd723f522a9ba154055a Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 15:47:04 +0100 Subject: [PATCH 11/59] H-8: a want's parts are checked by the digests the composition fetches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A want with parts is applied only when every fetched archive, process and container digest is among the parts and every part is a fetched digest; otherwise the module keeps running what it has and its have is held with detail parts-differ naming the digests. A want with no parts is applied as before. Builds: design 47 G §524, §379 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- internal/agent/agent.go | 18 +++++++++ internal/agent/agent_test.go | 44 +++++++++++++++++++++ internal/agent/parts.go | 76 ++++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+) create mode 100644 internal/agent/parts.go diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 4577e493..9aa00adc 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -1601,6 +1601,24 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo ranBefore = ranBefore || a.prior[module] a.mu.Unlock() + // A want with parts is applied only when they and the digests its composition fetches agree (design 47 G §524): the + // module keeps running what it has, and the have says which digest differs. + if words := partsDifference(w.Want.Parts, comp.C.Resources); words != "" { + a.mu.Lock() + prev, runs := a.running[module] + a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev, Held: true, AnsweredRev: w.Rev} + a.mu.Unlock() + if !runs { + if h, found, _ := a.readHave(ctx, module); found { + prev = nodeengine.HaveByNodeModuleValue{Fp: h.FP, Composed: h.Composed} + } + } + a.env.Say(fmt.Sprintf("%s is held: %s", module, words)) + a.writeHave(ctx, Have{Module: module, FP: prev.Fp, Composed: prev.Composed, Phase: string(nodeengineclient.PhaseHeld), + Detail: words}) + return + } + // A data module is backed up before a new composition of it moves (ADR 0429 §14), unless the want says to go on // without one (ADR 0539 §16). if comp.C.Data && ranBefore && a.env.Backup != nil && w.Want.Backup != meshwants.WantsByWantNodeModuleValueBackupSkip { diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 082a3fe4..a609f558 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -322,6 +322,50 @@ func TestAWantAndItsCompositionApplyTheModuleAndTheHaveNamesBoth(t *testing.T) { if got := b.appliedList(); len(got) != 1 { t.Fatalf("a want naming another digest applied the composition: %v", got) } + + // Design 47 G section 524: a want with parts is applied only when its parts and the digests the composition fetches + // agree; a want with no parts is applied as before. + d1, d2, d3 := "sha256:"+strings.Repeat("1", 64), "sha256:"+strings.Repeat("2", 64), "sha256:"+strings.Repeat("3", 64) + archive := func(id, digest string) json.RawMessage { + raw, _ := json.Marshal(map[string]any{"id": "web." + id, "type": "archive", "path": "/tmp/r6/web/" + id, + "source": "https://store/" + id, "digest": digest, "module": "web"}) + return raw + } + for i, row := range []struct { + name string + parts map[string]string + applies bool + detail string + }{ + {"agree", map[string]string{"web:a": d1, "web:b": d2}, true, ""}, + {"no parts", nil, true, ""}, + {"fetched outside", map[string]string{"web:a": d1}, false, PartsDiffer + " fetched and not in the want's parts: " + d2}, + {"part not fetched", map[string]string{"web:a": d1, "web:b": d2, "web:c": d3}, false, + PartsDiffer + " in the want's parts and not fetched: " + d3}, + } { + before := len(b.appliedList()) + fpn := fpOf(string("5678"[i])) + ckn, crawn, dign := composition(t, "web", fpn, false, archive("a", d1), archive("b", d2)) + b.comps.put(ckn, crawn) + wkn := meshwants.WantsByWantNodeModuleKey(node, "web") + wrawn, err := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fpn, Composed: dign, Parts: row.parts}) + if err != nil { + t.Fatal(err) + } + b.wants.put(wkn, wrawn) + b.idle() + if got := len(b.appliedList()) - before; (got == 1) != row.applies { + t.Fatalf("parts %q: applied %d times, want applies=%v", row.name, got, row.applies) + } + if h := b.haves.last("web"); !row.applies && (h.Phase != string(nodeengineclient.PhaseHeld) || h.Detail != row.detail) { + t.Fatalf("parts %q: the have is %s %q, want held %q", row.name, h.Phase, h.Detail, row.detail) + } + } + // A container's digest is the one after its image's @. + container := json.RawMessage(`{"id":"web.c","type":"container","name":"web-c","image":"store/web@` + d3 + `","module":"web"}`) + if got := partsDifference(map[string]string{"web:c": d3}, []json.RawMessage{container}); got != "" { + t.Fatalf("a container whose digest is in the parts differs: %q", got) + } } // R6-A AC2: given a kept declaration and wants and compositions equal to it, when the agent starts, nothing is applied diff --git a/internal/agent/parts.go b/internal/agent/parts.go new file mode 100644 index 00000000..bb5a1f83 --- /dev/null +++ b/internal/agent/parts.go @@ -0,0 +1,76 @@ +package agent + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/declaration" +) + +// PartsDiffer is the start of the detail of a have held because a want's parts and the digests its composition fetches +// disagree (design 47 G section 524). +const PartsDiffer = "parts-differ:" + +// partsDifference checks a want's `parts` against the digests of the archive, process and container resources of the +// composition, which are the digests the apply fetches and checks (design 47 G section 524). A want with no `parts` (an +// assigner of before G, or a node whose record does not state `want-parts`) is applied as today: it returns "". With +// `parts`, the want is applied only when the digest of every resource fetched is among the values of `parts` and every +// value of `parts` is the digest of a resource fetched; otherwise it returns the detail naming each digest. +func partsDifference(parts map[string]string, resources []json.RawMessage) string { + if len(parts) == 0 { + return "" + } + fetched := map[string]bool{} + for _, raw := range resources { + r, err := declaration.ParseResource(raw) + if err != nil { + continue + } + switch res := r.(type) { + case *declaration.Archive: + if res.Source != "" { + fetched[res.Digest] = true + } + case *declaration.Process: + if res.Source != "" { + fetched[res.Digest] = true + } + case *declaration.Container: + if strings.HasPrefix(res.Image, "sha256:") { + fetched[res.Image] = true + } else if _, digest, ok := strings.Cut(res.Image, "@"); ok { + fetched[digest] = true + } + } + } + named := map[string]bool{} + for _, d := range parts { + named[d] = true + } + var outside, missing []string + for d := range fetched { + if !named[d] { + outside = append(outside, d) + } + } + for d := range named { + if !fetched[d] { + missing = append(missing, d) + } + } + if len(outside) == 0 && len(missing) == 0 { + return "" + } + sort.Strings(outside) + sort.Strings(missing) + var said []string + if len(outside) > 0 { + said = append(said, "fetched and not in the want's parts: "+strings.Join(outside, ", ")) + } + if len(missing) > 0 { + said = append(said, "in the want's parts and not fetched: "+strings.Join(missing, ", ")) + } + return fmt.Sprintf("%s %s", PartsDiffer, strings.Join(said, "; ")) +} -- 2.54.0 From 75f5c64646449c150f4b27c4481c9f3fb7d4dc8b Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 21:13:24 +0100 Subject: [PATCH 12/59] The node-engine says no capabilities-changed: manifest-reads-changed alone says a node's capabilities, as the contracts' K6 has the seat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The join with mesh-contracts' K6 (7d730a4a) removes the event and its body; H-3's second source of the same fact goes (haveState.capsSaid, capabilitiesToSay, sayCapabilities), the test asserts none is said. Builds: design 47 G §590 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- internal/link/have.go | 34 ++-------------------------------- internal/link/have_test.go | 8 ++++---- 2 files changed, 6 insertions(+), 36 deletions(-) diff --git a/internal/link/have.go b/internal/link/have.go index 202cac27..ed4f2f36 100644 --- a/internal/link/have.go +++ b/internal/link/have.go @@ -9,7 +9,6 @@ import ( nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" sdkkeys "git.novox.be/novox/mesh-sdk/go/keys" "github.com/novox/mesh-host/internal/compositions" - "slices" "sort" "strings" "sync" @@ -68,11 +67,8 @@ type haveState struct { userList map[string]nodeengine.UserListWritten // userListNoKey is each module whose user list was said to have no have key to ride on, so it is said once. userListNoKey map[string]bool - // runningSaid is, by module, the fingerprint and manifest its event running last said, so each is said once; and - // capsSaid the capabilities the event capabilities-changed last said (nil: none said yet this run). + // runningSaid is, by module, the fingerprint and manifest its event running last said, so each is said once. runningSaid map[string]string - capsSaid []string - capsKnown bool // unjudged is each module whose composition moved (an apply began, or the fact names another build) and that no look // has judged since it stopped moving: its health is not yet judged, never the last look's of what ran before. unjudged map[string]bool @@ -366,7 +362,7 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) { return } s := &q.haves - // What the pass says as the seat's events (running, capabilities-changed) is said once the haves' lock is let go: a + // What the pass says as the seat's events (running) is said once the haves' lock is let go: a // publish never holds it. var said []func() defer func() { @@ -467,9 +463,6 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) { }) { return } - if ev, ok := s.capabilitiesToSay(node, facts.Record); ok { - said = append(said, func() { q.sayCapabilities(ev) }) - } } } if err != nil { @@ -937,35 +930,12 @@ func (s *haveState) runningToSay(module string, h nodeengine.HaveByNodeModuleVal return nodeengine.Running{V: sdkkeys.Version, Module: module, Fingerprint: h.Fp, Manifest: digest}, true } -// capabilitiesToSay is the event capabilities-changed when the node's record says other capabilities than the event last -// said (or none was said this run). Under s.mu. -func (s *haveState) capabilitiesToSay(node string, rec *nodeengine.NodeRecord) (nodeengine.CapabilitiesChanged, bool) { - if rec == nil { - return nodeengine.CapabilitiesChanged{}, false - } - if s.capsKnown && slices.Equal(s.capsSaid, rec.Capabilities) { - return nodeengine.CapabilitiesChanged{}, false - } - s.capsKnown, s.capsSaid = true, slices.Clone(rec.Capabilities) - caps := slices.Clone(rec.Capabilities) - if caps == nil { - caps = []string{} - } - return nodeengine.CapabilitiesChanged{V: sdkkeys.Version, Node: node, Capabilities: caps}, true -} - // sayRunning says the event running, on its subject `running..`. func (q *Session) sayRunning(ev nodeengine.Running) { ev.Node, ev.At = q.Membership.Node, time.Now().UTC() q.sayEngineOn(nodeengine.SubjectRunning(ev.Node, ev.Module), "running", ev) } -// sayCapabilities says the event capabilities-changed, on its subject `capabilities-changed.`. -func (q *Session) sayCapabilities(ev nodeengine.CapabilitiesChanged) { - ev.At = time.Now().UTC() - q.sayEngineOn(nodeengine.SubjectCapabilitiesChanged(ev.Node), "capabilities-changed", ev) -} - // SetManifestOf gives the session what names the manifest of the build a module runs (the composer's picture of the // catalog half): its digest, or "" when the node does not know it. func (q *Session) SetManifestOf(f func(module, fp string) string) { diff --git a/internal/link/have_test.go b/internal/link/have_test.go index 041bc536..7a76f716 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -44,7 +44,7 @@ type haveLink struct { // sayFails fails that many of the node seat's says, as a publish past the bus's retry bound; said is every event said. sayFails int said []string - // onSaid are the events said on a subject of their own (running, capabilities-changed), by subject, in order. + // onSaid are the events said on a subject of their own (running), by subject, in order. onSaid []string } @@ -349,9 +349,9 @@ func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) { if l.kv["n1.node"].rev != at { t.Errorf("a report changing nothing wrote the node key again") } - // Design 47: the node's capabilities, said when they change, on capabilities-changed.. - if got := l.saidOn("mesh.seat.node-engine.event.capabilities-changed.n1 "); len(got) != 1 || !strings.Contains(got[0], `"capabilities":["apt","bus-state-calls","ledger-calls","systemd","want-parts"]`) { - t.Errorf("capabilities-changed said %v, want once for the record's capabilities", got) + // Design 47 G §590: the node's capabilities are said by the event manifest-reads-changed alone; no capabilities-changed. + if got := l.saidOn("mesh.seat.node-engine.event.capabilities-changed."); len(got) != 0 { + t.Errorf("capabilities-changed said %v, want none (G §590)", got) } // A runner that does not answer states what it stated before; one answering without any states none (an older runner). l.statedErr = fmt.Errorf("%w: nothing answered", ErrNoAnswer) -- 2.54.0 From 189ed10c36bf9004e0a0b92228e89c64386e5e1a Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 21:18:40 +0100 Subject: [PATCH 13/59] The node-engine's compositions use the catalog key's codec meshcatalogkey, as meshcatalogclient's wrappers go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §670 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- go.mod | 1 + internal/compositions/composed.go | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 870a6992..04fc37be 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogkey v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 diff --git a/internal/compositions/composed.go b/internal/compositions/composed.go index 16296023..c1cac11c 100644 --- a/internal/compositions/composed.go +++ b/internal/compositions/composed.go @@ -7,6 +7,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogkey" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/declaration" "git.novox.be/novox/mesh-sdk/go/keys" @@ -134,7 +135,7 @@ func Same(a, b Composed) bool { // Key is the key of a module's composition for a node at a fingerprint: `..`. func Key(node, module, fingerprint string) (string, error) { - part, err := meshcatalogclient.KeyPart(fingerprint) + part, err := meshcatalogkey.KeyPart(fingerprint) if err != nil { return "", err } -- 2.54.0 From 38e13ef0309b44fceb6796da80df0d6c38165748 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 22:37:21 +0100 Subject: [PATCH 14/59] H-13: the node-engine's seat-list and held-module findings go through mesh-sdk's held writer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The seat list's five-minute finding and the finding of a held module are raised as they happen, so they use the held writer (seeded at Start, Held() answers what stands) over the contract of the seat mesh-conditions; the keepers keep no set or sweep of their own, and the writer is started before the seat list reader writes or clears any. The engine's own finding stream stays its foundation stream mesh-host-findings. Builds: design 47 G §529, A-29, A-30, A-37, A-38, A-40 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- cmd/mesh-host/nodeagent.go | 19 +- cmd/mesh-host/nodeagent_integration_test.go | 5 + cmd/mesh-host/seatlist.go | 188 +++----------------- cmd/mesh-host/seatlist_unread_test.go | 8 - internal/link/findings.go | 15 -- internal/seatwait/held.go | 66 +++---- internal/seatwait/seatwait.go | 66 ++++--- internal/seatwait/seatwait_test.go | 85 +++++---- 8 files changed, 167 insertions(+), 285 deletions(-) diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index ea3f3b4a..2b5b8b9b 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -20,7 +20,6 @@ import ( "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay" "git.novox.be/novox/mesh-sdk/go/declaration" - "git.novox.be/novox/mesh-sdk/go/views" "github.com/novox/mesh-host/internal/agent" "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/compose/frombus" @@ -593,7 +592,17 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus own := &agentNow{} // And the seat list it reads through the seat mesh-catalog, from its bootstrapped definitions until it has: the // composer hands nothing new meanwhile, and the node says it has waited five minutes (novox/hq ADR 0636 §1). - findings := newFindingsOfTheNode(session, node) + // The finding writer is started first, so it holds what an earlier run left before the reader writes or clears any. + findings, ferr := newFindingsOfTheNode(session, node, say) + if ferr != nil { // a writer that cannot be made is a fault of its configuration: loud, never a node without findings + say("the node-engine cannot make its finding writer, so its node agent stops: " + ferr.Error()) + stopRun() + cancel() + return + } + if werr := findings.Start(run); werr != nil { + say("the node-engine's findings are kept in its state and not said as events until it is started again: " + werr.Error()) + } seats, err := startSeatList(run, session, node, findings, say) if err != nil { // No reader is no gate that lets everything through: the node acts on nothing new, as one that waits does. @@ -601,11 +610,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus } // The marks of the haves run off the composition pass, each bounded (see marker). marks := newMarker(run) - if werr := findings.Start(run, say); werr != nil { - say("the node-engine's findings are kept in its state and not said as events until it is started again: " + werr.Error()) - } - heldFindings := &seatwait.HeldKeeper{Node: node, Observer: findingsModule, Writer: findings, - Left: func() ([]views.Finding, error) { return findings.left(node) }, Say: say} + heldFindings := &seatwait.HeldKeeper{Node: node, Observer: findingsModule, Writer: findings, Ctx: run, Say: say} hooks := composerHooks{ seatList: seats, seatListFailed: err != nil, diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index 226c0e1b..fc86409e 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -992,6 +992,11 @@ func (c *fiveMinutes) pass() { } } +// findingSubject is the subject of one of the engine's finding events on node (views.FindingSubjects). +func findingSubject(event, node string) string { + return "mesh.stream." + findingsModule + ".event." + event + "." + node +} + // ADR 0636 §1, §8, whole against a real bus and the stand-in of the seat: while the seat refuses, the node composes and // applies nothing new, and keeps running what it ran; after the bound it writes one finding in its state findings; when // the seat names its list the module is applied and the finding is cleared; a node started again with the seat gone diff --git a/cmd/mesh-host/seatlist.go b/cmd/mesh-host/seatlist.go index c53d423c..c06f5a9f 100644 --- a/cmd/mesh-host/seatlist.go +++ b/cmd/mesh-host/seatlist.go @@ -5,14 +5,10 @@ import ( "encoding/json" "errors" "fmt" - "slices" - "sort" "strings" - "sync" "sync/atomic" "time" - conditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/seatdef" @@ -110,179 +106,49 @@ func (realClock) AfterFunc(d time.Duration, f func()) seatwait.Timer { // their observer (module.json's `module` and its stream `mesh-host-findings`, design 47 G §383). const findingsModule = "mesh-host" -// findingsOfTheNode keeps the node's findings through mesh-sdk's finding writer (design 47 G §383, §384, §461): each finding is -// written in the node's state `findings` and said as an event on the engine's own stream, the engine says every finding it -// observes at its start and when the keeper asks, and the keeper reads no state of the engine's. +// nodeFindings is the node-engine's finding writer: mesh-sdk's held writer, for the engine's findings are raised as they +// happen and not judged afresh (design 47 G A-29, A-40), with the contract of the seat mesh-conditions. +type nodeFindings = views.HeldWriter[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent] + +// findingsOfTheNode keeps the node's findings through mesh-sdk's held writer (design 47 G §383, §384, §461, A-29): each +// finding is written in the node's state `findings` and said as an event on the engine's own stream (a foundation part's +// `mesh-host-findings`, §529, A-30), the engine says what it holds at its start and when the keeper asks, and the keeper +// reads no state of the engine's. type findingsOfTheNode struct { - session *link.Session - node string - - mu sync.Mutex - ctx context.Context - writer *views.Writer + *nodeFindings } -func newFindingsOfTheNode(session *link.Session, node string) *findingsOfTheNode { - return &findingsOfTheNode{session: session, node: node} -} - -// findingSubject is a stream event's subject for the observer's own node (design 47 G §383, ADR 0363 §2). -func findingSubject(event, node string) string { - return "mesh.stream." + findingsModule + ".event." + event + "." + node -} - -// Start makes the writer and states the findings now, until ctx ends. A writer that cannot start (the link has no stream -// yet) is said; the findings are then kept in the state alone, and said by the next start. -func (w *findingsOfTheNode) Start(ctx context.Context, say func(string)) error { - writer, err := views.NewWriter(views.WriterConfig{ - Module: findingsModule, - State: link.FindingsState{Session: w.session}, - Subjects: views.WriterSubjects{Written: findingSubject("finding-written", w.node), Cleared: findingSubject("finding-cleared", w.node), - Stated: findingSubject("findings-stated", w.node), Asked: condseat.SubjectFindingsAsked}, - Observe: w.observe, +// newFindingsOfTheNode makes the writer at once; it reads the state when it is started (A-40). Its events are published on +// the engine's own stream awaiting the bus's acknowledgement and its consumer of `findings-asked` is its own. +func newFindingsOfTheNode(session *link.Session, node string, say func(string)) (*findingsOfTheNode, error) { + writer, err := views.NewHeldWriter(views.WriterConfig[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]{ + Module: findingsModule, + Node: node, + State: link.FindingsState{Session: session}, + Kinds: []string{seatwait.Kind, seatwait.HeldKind}, + Asked: condseat.SubjectFindingsAsked, Logf: func(format string, args ...any) { say(fmt.Sprintf(format, args...)) }, - Publish: w.session.PublishEvent, + Publish: session.PublishEvent, Listen: func(_ context.Context, subject string, handle func([]byte)) (func(), error) { - return w.session.ListenFrom(condseat.StreamFindingsAsked, subject, handle) + return session.ListenFrom(condseat.StreamFindingsAsked, subject, handle) }, }) - if err != nil { - return err - } - w.mu.Lock() - w.ctx, w.writer = ctx, writer - w.mu.Unlock() - go func() { <-ctx.Done(); writer.Close() }() - return writer.Start(ctx) -} - -func (w *findingsOfTheNode) started() (context.Context, *views.Writer, error) { - w.mu.Lock() - defer w.mu.Unlock() - if w.writer == nil { - return nil, nil, errors.New("the node-engine's finding writer is not started") - } - return w.ctx, w.writer, nil -} - -// findingBody is the finding in the layout of the seat mesh-conditions (design 47 G §439). The seat list's finding is of the -// machine; a held module's is of the module on a machine, `.`. The writer names the observer. -func findingBody(f views.Finding) views.FindingBody { - scope, machine := conditions.ScopeMachine, f.ID - if f.Kind == seatwait.HeldKind { - scope = conditions.ScopeModule - machine = f.ID[strings.LastIndex(f.ID, ".")+1:] - } - return views.FindingBody{Kind: f.Kind, Scope: scope, ID: f.ID, At: f.At.UTC(), Severity: conditions.Warning, - Summary: f.Summary, Headline: f.Headline, Explanation: f.Explanation, Source: f.Kind, - Subject: views.FindingSubject{Scope: scope, ID: f.ID, Machine: machine}} -} - -// left is the findings of held modules an earlier run of this engine left on node: its keys of this kind. -func (w *findingsOfTheNode) left(node string) ([]views.Finding, error) { - keys, err := w.session.FindingKeys() if err != nil { return nil, err } - var out []views.Finding - for _, k := range keys { - id, ok := strings.CutPrefix(k, conditions.ScopeModule+".") - if !ok { - continue - } - if id, ok = strings.CutSuffix(id, "."+seatwait.HeldKind); !ok || !strings.HasSuffix(id, "."+node) { - continue - } - out = append(out, views.Finding{Observer: findingsModule, Kind: seatwait.HeldKind, ID: id}) - } - return out, nil + return &findingsOfTheNode{writer}, nil } -func (w *findingsOfTheNode) Put(f views.Finding) error { - ctx, writer, err := w.started() - if err != nil { - return err - } - return writer.Write(ctx, findingBody(f)) -} - -// keyOf is the finding's key in the state; a read or a clear needs no more of the finding than Kind and ID. -func (w *findingsOfTheNode) keyOf(f views.Finding) string { - b := findingBody(f) - return b.Key() -} - -func (w *findingsOfTheNode) Delete(f views.Finding) error { - ctx, writer, err := w.started() - if err != nil { - // Before the writer starts, which is at its start the one statement that says what this engine observes: the key goes, - // and that statement names no finding of it, so the keeper clears it (§384). - return w.session.DeleteFinding(w.keyOf(f)) - } - b := findingBody(f) - why := "the node-engine no longer observes it" - switch f.Kind { - case seatwait.Kind: - why = "the node-engine has read the whole seat list" - case seatwait.HeldKind: - why = "the module is composed again or is no longer wanted on the node" - } - return writer.Clear(ctx, b.Scope, b.ID, b.Kind, why) -} - -// Has says whether the finding stands in the node's state `findings` now. -func (w *findingsOfTheNode) Has(f views.Finding) (bool, error) { - return w.session.HasFinding(w.keyOf(f)) -} - -// observe is the node-engine's own truth for a statement (§384): the findings that stand in its state and are still true -// (the seat list's only while the reader waits for it, so an engine started with the full list states none), and the seat -// list's finding as pending while its five minutes run and none stands yet. -func (w *findingsOfTheNode) observe(context.Context) (views.Observed, error) { - var out views.Observed - st := seatListNow.Load() - waits := st != nil && seatwait.Waits(st.reader.Status()) - keys, err := w.session.FindingKeys() - if err != nil { - return out, err - } - sort.Strings(keys) - for _, key := range keys { - scope, rest, ok := strings.Cut(key, ".") - i := strings.LastIndex(rest, ".") - if !ok || i < 0 { - continue - } - id, kind := rest[:i], rest[i+1:] - if kind == seatwait.Kind && !waits { - continue // read in full: it is no finding of this engine now, and its key goes with this statement - } - e, err := link.FindingsState{Session: w.session}.Get(key) - if err != nil || e == nil { - continue - } - var body views.FindingBody - if json.Unmarshal(e.Value, &body) != nil { - continue - } - body.Scope, body.ID, body.Kind = scope, id, kind - out.Findings = append(out.Findings, body) - } - if waits { - if stands, since := st.waiter.Waiting(); stands { - key := conditions.Key(conditions.ScopeMachine, w.node, seatwait.Kind) - if !slices.Contains(keys, key) { - out.Pending = append(out.Pending, views.PendingKey{Key: key, Since: since.UTC(), Bound: seatwait.Bound.String()}) - } - } - } - return out, nil +// Start seeds what the state holds, states the findings now and hears the keeper until ctx ends. +func (w *findingsOfTheNode) Start(ctx context.Context) error { + go func() { <-ctx.Done(); w.Close() }() + return w.nodeFindings.Start(ctx) } // startSeatList starts this run's reader: its bootstrapped seats are given to catalogue at once, and every change of what // it holds again, saying so to the composer and to the waiter that writes the five-minute finding, and stating again what // the engine reads (link.Session.RestateReads, ADR 0618 §4): the seats it states are those of the list it holds. -func startSeatList(ctx context.Context, session *link.Session, node string, findings seatwait.Findings, say func(string)) (*seatListState, error) { +func startSeatList(ctx context.Context, session *link.Session, node string, findings seatwait.Writer, say func(string)) (*seatListState, error) { st := &seatListState{changed: make(chan struct{}, 1)} // Start returns at once and reads the seat on a goroutine, which may announce a change before the waiter exists. var announce atomic.Pointer[func()] @@ -300,7 +166,7 @@ func startSeatList(ctx context.Context, session *link.Session, node string, find return nil, fmt.Errorf("the node-engine cannot follow the seat list: %w", err) } st.reader = reader - st.waiter = &seatwait.Waiter{Node: node, Observer: findingsModule, Reader: reader, Findings: findings, + st.waiter = &seatwait.Waiter{Node: node, Observer: findingsModule, Reader: reader, Writer: findings, Ctx: ctx, Clock: seatWaitClock, Say: say} told := func() { st.waiter.Changed() diff --git a/cmd/mesh-host/seatlist_unread_test.go b/cmd/mesh-host/seatlist_unread_test.go index a9a8cb18..ef6c1208 100644 --- a/cmd/mesh-host/seatlist_unread_test.go +++ b/cmd/mesh-host/seatlist_unread_test.go @@ -3,8 +3,6 @@ package main import ( "testing" - "git.novox.be/novox/mesh-sdk/go/views" - "github.com/novox/mesh-host/internal/seatwait" "github.com/novox/mesh-host/internal/selfcompose" ) @@ -21,10 +19,4 @@ func TestAReaderThatCouldNotStartHoldsTheComposer(t *testing.T) { if got := composerEnv(nil, "n1", func(string) {}, composerHooks{}).SeatList; got != nil { t.Errorf("a composer with no reader asked for gates by %v, want none (the shadow)", got) } - // The finding of a held module is of the module on its machine, in a layout the condition keeper accepts (design 47, test 55). - body := findingBody(views.Finding{Observer: "node-engine", Kind: seatwait.HeldKind, - ID: seatwait.HeldID("mesh-grants", "ace"), Summary: "mesh-grants is held on ace: x", Headline: "h", Explanation: "e"}) - if body.Key() != "module.mesh-grants.ace.module-held" || body.Subject.Machine != "ace" || body.Observer != "" { - t.Errorf("a held module's finding is %+v with key %q", body, body.Key()) - } } diff --git a/internal/link/findings.go b/internal/link/findings.go index ab636c1a..0a9a0d9f 100644 --- a/internal/link/findings.go +++ b/internal/link/findings.go @@ -46,21 +46,6 @@ func (q *Session) PutFinding(key string, value []byte) error { return err } -// HasFinding says whether one finding of this engine stands in its state now. -func (q *Session) HasFinding(key string) (bool, error) { - kv, err := q.findings() - if err != nil { - return false, err - } - if _, err = kv.Get(key); err != nil { - if errors.Is(err, nats.ErrKeyNotFound) { - return false, nil - } - return false, err - } - return true, nil -} - // FindingKeys lists the keys of this engine's findings over the link open now; none when the bucket holds none. func (q *Session) FindingKeys() ([]string, error) { kv, err := q.findings() diff --git a/internal/seatwait/held.go b/internal/seatwait/held.go index b2682d50..5ac18e8b 100644 --- a/internal/seatwait/held.go +++ b/internal/seatwait/held.go @@ -6,34 +6,32 @@ package seatwait import ( + "context" "fmt" "sort" "strings" - "sync" "time" - "git.novox.be/novox/mesh-sdk/go/views" + condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" ) // HeldKind is the finding of a held module; its id is `.`. const HeldKind = "module-held" -// HeldKeeper writes one finding for each module the node holds, and clears it when the module is no longer held. +// HeldKeeper writes one finding for each module the node holds, and clears it when the module is no longer held. What it +// has written is what its writer holds (design 47 G A-29, A-38, A-40): the findings an earlier run of the engine left are +// seeded into it at its Start, so a module held then and composed while the engine was away loses its condition at the +// first Reconcile, and the keeper keeps no set of its own. type HeldKeeper struct { Node string Observer string - Writer views.FindingWriter - // Left lists the findings of this kind an earlier run of the engine left, once, at the first Reconcile: a module held - // then and composed while the engine was away must not keep its condition. Nil leaves none. - Left func() ([]views.Finding, error) + Writer Writer + // Ctx is the run's context for the writer's acts; nil is the background. + Ctx context.Context // Now is the clock; nil is time.Now. Now func() time.Time // Say tells the node's journal; nil says nothing. Say func(string) - - mu sync.Mutex - swept bool - written map[string]views.Finding // by module } // HeldID is the finding's id for a module on a node. @@ -43,29 +41,21 @@ func HeldID(module, node string) string { return module + "." + node } // held for other words, and one is cleared for a module no longer held, composed or no longer wanted here. A write or a // clear that fails is said and tried again at the next Reconcile. func (k *HeldKeeper) Reconcile(held map[string]string) { - k.mu.Lock() - defer k.mu.Unlock() - if k.written == nil { - k.written = map[string]views.Finding{} + ctx := k.Ctx + if ctx == nil { + ctx = context.Background() } - if !k.swept && k.Left != nil { - left, err := k.Left() - if err != nil { - k.say(fmt.Sprintf("the findings an earlier run left of held modules cannot be read: %v", err)) - } else { - k.swept = true - for _, f := range left { - module := strings.TrimSuffix(f.ID, "."+k.Node) - if _, ok := k.written[module]; !ok { - k.written[module] = f - } - } + written := map[string]condseat.Finding{} // by module, what the writer holds of this node's held modules + for _, f := range k.Writer.Held() { + if f.Kind != HeldKind || f.Scope != scopeModule { + continue + } + if module, ok := strings.CutSuffix(f.ID, "."+k.Node); ok { + written[module] = f } - } else if k.Left == nil { - k.swept = true } - names := make([]string, 0, len(k.written)) - for m := range k.written { + names := make([]string, 0, len(written)) + for m := range written { names = append(names, m) } sort.Strings(names) @@ -73,11 +63,9 @@ func (k *HeldKeeper) Reconcile(held map[string]string) { if _, still := held[m]; still { continue } - if err := k.Writer.Delete(k.written[m]); err != nil { + if err := k.Writer.Clear(ctx, scopeModule, written[m].ID, HeldKind, "the module is composed again or is no longer wanted on the node"); err != nil { k.say(fmt.Sprintf("the finding that %s is held on %s cannot be cleared: %v", m, k.Node, err)) - continue } - delete(k.written, m) } names = names[:0] for m := range held { @@ -86,19 +74,21 @@ func (k *HeldKeeper) Reconcile(held map[string]string) { sort.Strings(names) for _, m := range names { why := held[m] - if was, ok := k.written[m]; ok && was.Summary != "" && strings.HasSuffix(was.Summary, ": "+why) { + if was, ok := written[m]; ok && strings.HasSuffix(was.Summary, ": "+why) { continue } - f := views.Finding{Observer: k.Observer, Kind: HeldKind, ID: HeldID(m, k.Node), + id := HeldID(m, k.Node) + f := condseat.Finding{Observer: k.Observer, Kind: HeldKind, Scope: scopeModule, ID: id, Severity: condseat.FindingSeverityWarning, + Source: HeldKind, + Subject: condseat.Subject{Scope: scopeModule, ID: id, Machine: k.Node}, Summary: fmt.Sprintf("%s is held on %s: %s", m, k.Node, why), Headline: fmt.Sprintf("%s is held on %s", m, k.Node), Explanation: "The node leaves this module out of its pass, keeps what runs of it and composes and applies every other module. The finding clears when the module is composed again or is no longer wanted on the node.", At: k.now()} - if err := k.Writer.Put(f); err != nil { + if err := k.Writer.Write(ctx, f); err != nil { k.say(fmt.Sprintf("the finding that %s is held on %s cannot be written: %v", m, k.Node, err)) continue } - k.written[m] = f k.say(f.Summary) } } diff --git a/internal/seatwait/seatwait.go b/internal/seatwait/seatwait.go index 54a8f4f9..7fc83a44 100644 --- a/internal/seatwait/seatwait.go +++ b/internal/seatwait/seatwait.go @@ -8,12 +8,13 @@ package seatwait import ( + "context" "fmt" "sync" "time" + condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-sdk/go/seatlist" - "git.novox.be/novox/mesh-sdk/go/views" ) // Bound is how long the reader may wait before the node says it (ADR 0636 §1). @@ -36,12 +37,28 @@ type Clock interface { AfterFunc(d time.Duration, f func()) Timer } -// Findings is the node's own state `findings`: the writer, and the read of whether a finding stands there. The bus state -// is the one truth of whether the finding exists (ADR 0636 §1, issue 1966): the Waiter keeps no copy, so a new process -// of the node-engine clears what an earlier one wrote. -type Findings interface { - views.FindingWriter - Has(f views.Finding) (bool, error) +// Writer is the node-engine's finding writer as the keepers use it: mesh-sdk's held writer over the node's state `findings` +// (views.HeldWriter, design 47 G A-29, A-38, A-40), which holds what stands there from its start, so the Waiter keeps no +// copy and a new process of the node-engine clears what an earlier one wrote (ADR 0636 §1, issue 1966). +type Writer interface { + Write(ctx context.Context, f condseat.Finding) error + Clear(ctx context.Context, scope, id, kind, why string) error + Held() []condseat.Finding +} + +const ( + scopeMachine = "machine" + scopeModule = "module" +) + +// stands is whether the writer holds the finding of that scope, id and kind. +func stands(w Writer, scope, id, kind string) bool { + for _, f := range w.Held() { + if f.Scope == scope && f.ID == id && f.Kind == kind { + return true + } + } + return false } // Waiter writes and clears the node's finding. @@ -49,8 +66,10 @@ type Waiter struct { Node string Observer string Reader Reader - Findings Findings + Writer Writer Clock Clock + // Ctx is the run's context for the writer's acts; nil is the background. + Ctx context.Context // Say tells the node's journal; nil says nothing. Say func(string) @@ -59,11 +78,15 @@ type Waiter struct { since time.Time } -// key is the finding this node's Waiter writes: Kind and ID name it, the rest is its text. -func (w *Waiter) key() views.Finding { - return views.Finding{Observer: w.Observer, Kind: Kind, ID: w.Node} +func (w *Waiter) ctx() context.Context { + if w.Ctx != nil { + return w.Ctx + } + return context.Background() } +func (w *Waiter) stands() bool { return stands(w.Writer, scopeMachine, w.Node, Kind) } + func waiting(st seatlist.Status) bool { return !st.Heard || len(st.Waiting) > 0 } // Waits is whether a reader of that status holds only the definitions it started with, or waits for some. @@ -81,13 +104,8 @@ func (w *Waiter) Changed() { w.timer = nil } w.since = time.Time{} - stands, err := w.Findings.Has(w.key()) - if err != nil { - w.say(fmt.Sprintf("the node's finding that its seat list is unread cannot be read: %v", err)) - return - } - if stands { - if err := w.Findings.Delete(w.key()); err != nil { + if w.stands() { + if err := w.Writer.Clear(w.ctx(), scopeMachine, w.Node, Kind, "the node-engine has read the whole seat list"); err != nil { w.say(fmt.Sprintf("the node's finding that its seat list is unread cannot be cleared: %v", err)) } } @@ -96,7 +114,7 @@ func (w *Waiter) Changed() { if w.timer != nil { return } - if stands, err := w.Findings.Has(w.key()); err == nil && stands { + if w.stands() { return // an earlier process wrote it and the list is still unread: it stands until the first full read } w.since = w.Clock.Now() @@ -110,8 +128,7 @@ func (w *Waiter) Waiting() (bool, time.Time) { if w.timer != nil { return true, w.since } - stands, err := w.Findings.Has(w.key()) - return err == nil && stands, w.since + return w.stands(), w.since } func (w *Waiter) expired() { @@ -123,15 +140,16 @@ func (w *Waiter) expired() { w.since = time.Time{} return } - if stands, err := w.Findings.Has(w.key()); err == nil && stands { + if w.stands() { return } - f := views.Finding{Observer: w.Observer, Kind: Kind, ID: w.Node, + f := condseat.Finding{Observer: w.Observer, Kind: Kind, Scope: scopeMachine, ID: w.Node, Severity: condseat.FindingSeverityWarning, + Source: Kind, Subject: condseat.Subject{Scope: scopeMachine, ID: w.Node, Machine: w.Node}, Summary: fmt.Sprintf("%s has read no complete seat list since %s: %s", w.Node, w.since.UTC().Format(time.RFC3339), st.Health()), Headline: fmt.Sprintf("%s waits for the seat list", w.Node), Explanation: "The node holds only the definitions it starts with, so it keeps running what it runs and starts, composes and grants nothing new. It reads the list again at the next change the seat mesh-catalog says, or when asked to.", At: w.Clock.Now()} - if err := w.Findings.Put(f); err != nil { + if err := w.Writer.Write(w.ctx(), f); err != nil { w.say(fmt.Sprintf("the node's finding that its seat list is unread cannot be written: %v", err)) return } diff --git a/internal/seatwait/seatwait_test.go b/internal/seatwait/seatwait_test.go index 3e945bd9..80f0a534 100644 --- a/internal/seatwait/seatwait_test.go +++ b/internal/seatwait/seatwait_test.go @@ -1,12 +1,13 @@ package seatwait import ( + "context" "errors" "testing" "time" + condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-sdk/go/seatlist" - "git.novox.be/novox/mesh-sdk/go/views" ) type fakeReader struct{ st seatlist.Status } @@ -45,25 +46,37 @@ func (c *fakeClock) advance(d time.Duration) { } } +// fakeFindings is a held writer: what it holds is what stands in the state, seeded by the test for an earlier process's. type fakeFindings struct { - put, deleted []views.Finding - err error - standing bool // the bus state holds the finding, written by this process or an earlier one + put []condseat.Finding + cleared []condseat.Finding // as held when cleared + why []string + err error + held []condseat.Finding } -func (f *fakeFindings) Has(views.Finding) (bool, error) { return f.standing, nil } +func (f *fakeFindings) Held() []condseat.Finding { return append([]condseat.Finding(nil), f.held...) } -func (f *fakeFindings) Put(x views.Finding) error { +func (f *fakeFindings) Write(_ context.Context, x condseat.Finding) error { if f.err != nil { return f.err } f.put = append(f.put, x) - f.standing = true + f.held = append(f.held, x) return nil } -func (f *fakeFindings) Delete(x views.Finding) error { - f.deleted = append(f.deleted, x) - f.standing = false + +func (f *fakeFindings) Clear(_ context.Context, scope, id, kind, why string) error { + for i, h := range f.held { + if h.Scope == scope && h.ID == id && h.Kind == kind { + f.held = append(f.held[:i:i], f.held[i+1:]...) + f.cleared = append(f.cleared, h) + f.why = append(f.why, why) + return nil + } + } + f.cleared = append(f.cleared, condseat.Finding{Scope: scope, ID: id, Kind: kind}) // cleared of nothing, said all the same + f.why = append(f.why, why) return nil } @@ -73,7 +86,7 @@ func caseTheNodeWritesOneFindingAfterFiveMinutesAndClearsItAtTheFirstFullRead(t clock := &fakeClock{now: time.Date(2026, 11, 5, 12, 0, 0, 0, time.UTC)} findings := &fakeFindings{} var said []string - w := &Waiter{Node: "ace", Observer: "node-engine", Reader: reader, Findings: findings, Clock: clock, Say: func(s string) { said = append(said, s) }} + w := &Waiter{Node: "ace", Observer: "node-engine", Reader: reader, Writer: findings, Clock: clock, Say: func(s string) { said = append(said, s) }} w.Changed() clock.advance(Bound - time.Second) @@ -92,8 +105,8 @@ func caseTheNodeWritesOneFindingAfterFiveMinutesAndClearsItAtTheFirstFullRead(t reader.st = seatlist.Status{Heard: true} w.Changed() - if len(findings.deleted) != 1 || findings.deleted[0].ID != "ace" { - t.Fatalf("the first full read left the finding: deleted %+v", findings.deleted) + if len(findings.cleared) != 1 || findings.cleared[0].ID != "ace" { + t.Fatalf("the first full read left the finding: deleted %+v", findings.cleared) } if on, _ := w.Waiting(); on { t.Error("the waiter still waits after the full read") @@ -105,14 +118,14 @@ func caseAReadWithinTheBoundWritesNothing(t *testing.T) { reader := &fakeReader{st: seatlist.Status{Heard: false}} clock := &fakeClock{now: time.Date(2026, 11, 5, 12, 0, 0, 0, time.UTC)} findings := &fakeFindings{} - w := &Waiter{Node: "ace", Observer: "node-engine", Reader: reader, Findings: findings, Clock: clock} + w := &Waiter{Node: "ace", Observer: "node-engine", Reader: reader, Writer: findings, Clock: clock} w.Changed() clock.advance(time.Minute) reader.st = seatlist.Status{Heard: true} w.Changed() clock.advance(time.Hour) - if len(findings.put) != 0 || len(findings.deleted) != 0 { - t.Fatalf("a list read in time wrote %+v and cleared %+v", findings.put, findings.deleted) + if len(findings.put) != 0 || len(findings.cleared) != 0 { + t.Fatalf("a list read in time wrote %+v and cleared %+v", findings.put, findings.cleared) } // The seat answered the list already held: no change was said, so the timer is the one to look again. @@ -131,7 +144,7 @@ func caseAFindingThatCannotBeWrittenIsSaidAndTriedAgain(t *testing.T) { clock := &fakeClock{now: time.Date(2026, 11, 5, 12, 0, 0, 0, time.UTC)} findings := &fakeFindings{err: errors.New("bus down")} var said []string - w := &Waiter{Node: "ace", Observer: "node-engine", Reader: reader, Findings: findings, Clock: clock, Say: func(s string) { said = append(said, s) }} + w := &Waiter{Node: "ace", Observer: "node-engine", Reader: reader, Writer: findings, Clock: clock, Say: func(s string) { said = append(said, s) }} w.Changed() clock.advance(Bound) if len(said) != 1 || len(findings.put) != 0 { @@ -150,8 +163,8 @@ func caseAFindingThatCannotBeWrittenIsSaidAndTriedAgain(t *testing.T) { func caseAFindingOfAnEarlierProcessIsClearedAtTheFirstFullRead(t *testing.T) { reader := &fakeReader{st: seatlist.Status{Heard: false}} clock := &fakeClock{now: time.Date(2026, 11, 11, 12, 0, 0, 0, time.UTC)} - findings := &fakeFindings{standing: true} - w := &Waiter{Node: "ace", Observer: "node-engine", Reader: reader, Findings: findings, Clock: clock} + findings := &fakeFindings{held: []condseat.Finding{{Scope: "machine", ID: "ace", Kind: Kind}}} + w := &Waiter{Node: "ace", Observer: "node-engine", Reader: reader, Writer: findings, Clock: clock} w.Changed() clock.advance(Bound) if len(findings.put) != 0 || len(clock.timers) != 0 { @@ -162,34 +175,42 @@ func caseAFindingOfAnEarlierProcessIsClearedAtTheFirstFullRead(t *testing.T) { } reader.st = seatlist.Status{Heard: true} w.Changed() - if len(findings.deleted) != 1 || findings.deleted[0].ID != "ace" || len(findings.put) != 0 { - t.Fatalf("the first full read left the earlier process's finding: deleted %+v, put %+v", findings.deleted, findings.put) + if len(findings.cleared) != 1 || findings.cleared[0].ID != "ace" || len(findings.put) != 0 { + t.Fatalf("the first full read left the earlier process's finding: deleted %+v, put %+v", findings.cleared, findings.put) } } // Design 47, test 55: a module held for its claim is a finding that names the node, the module and why, written once, // and cleared when the module is composed again or no longer wanted, also one an earlier run of the engine left. func caseAHeldModuleIsAFindingClearedWhenItIsComposedOrNoLongerWanted(t *testing.T) { - findings := &fakeFindings{} - k := &HeldKeeper{Node: "ace", Observer: "node-engine", Writer: findings, - Left: func() ([]views.Finding, error) { - return []views.Finding{{Kind: HeldKind, ID: HeldID("gone", "ace")}}, nil - }} + // the writer holds what an earlier run of the engine left (seeded at its Start), and what another node's keeper wrote + findings := &fakeFindings{held: []condseat.Finding{ + {Scope: "module", ID: HeldID("gone", "ace"), Kind: HeldKind}, + {Scope: "module", ID: HeldID("elsewhere", "g14"), Kind: HeldKind}}} + k := &HeldKeeper{Node: "ace", Observer: "node-engine", Writer: findings} k.Reconcile(map[string]string{"mesh-grants": "its seat is refused", "other": "no claim"}) - if len(findings.put) != 2 || findings.put[0].ID != "mesh-grants.ace" || len(findings.deleted) != 1 || findings.deleted[0].ID != "gone.ace" { - t.Fatalf("wrote %v, cleared %v", findings.put, findings.deleted) + if len(findings.put) != 2 || findings.put[0].ID != "mesh-grants.ace" || len(findings.cleared) != 1 || findings.cleared[0].ID != "gone.ace" { + t.Fatalf("wrote %v, cleared %v", findings.put, findings.cleared) + } + // the finding is of the module on its machine, in the contract's layout (design 47, test 55) + if f := findings.put[0]; f.Scope != "module" || f.Subject.Scope != "module" || f.Subject.Machine != "ace" || f.Subject.ID != "mesh-grants.ace" || + f.Severity != condseat.FindingSeverityWarning || f.Source != HeldKind || f.Observer != "node-engine" { + t.Fatalf("a held module's finding is %+v", f) + } + if got := findings.held[0]; got.ID != "elsewhere.g14" { + t.Fatalf("another node's finding was touched: %+v", findings.held) } k.Reconcile(map[string]string{"mesh-grants": "its seat is refused", "other": "no claim"}) if len(findings.put) != 2 { t.Fatalf("a module held for the same words was written again: %v", findings.put) } k.Reconcile(map[string]string{"mesh-grants": "its seat is refused"}) - if len(findings.deleted) != 2 || findings.deleted[1].ID != "other.ace" { - t.Fatalf("a module composed again kept its finding: %v", findings.deleted) + if len(findings.cleared) != 2 || findings.cleared[1].ID != "other.ace" { + t.Fatalf("a module composed again kept its finding: %v", findings.cleared) } k.Reconcile(nil) - if len(findings.deleted) != 3 { - t.Fatalf("a module no longer wanted kept its finding: %v", findings.deleted) + if len(findings.cleared) != 3 { + t.Fatalf("a module no longer wanted kept its finding: %v", findings.cleared) } } -- 2.54.0 From 7703162c95c3d56df0c5cad426a133fcd3eab011 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:27:28 +0100 Subject: [PATCH 15/59] =?UTF-8?q?G=20=C2=A7421,=20=C2=A7683:=20the=20node-?= =?UTF-8?q?engine=20reads=20the=20entry=20its=20want=20names=20by=20finger?= =?UTF-8?q?print=20and=20keeps=20its=20entries=20and=20wants?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The node's own composer reads the catalog half through meshcatalogclient's Entries cache over its own calls of mesh-catalog's verb entry, and no longer watches or reads the indexer's modules, manifests and verdicts or the builder's builds: a module's manifest is its entry's, each artifact's reference is made from the digest its want's part names (artifact-store:////blobs/, A-50). A want whose entry is asked or not found is held, its have the phase held with a detail opening entry-asked: or entry-missing: naming the fingerprint. The entries and wants it composes from are kept in composed- (entry. and want.), written before the composition that uses them, read at start before anything is asked, a kept want standing until the wants' live read answers, and dropped when nothing names them. The shadow keeps its reads (G §628). The node grant gains the verb entry and the running stream's consumer calls; OverNATS hears the running stream as the link's session does. Builds: design 47 G §421, §683, §614, §628, A-47, A-50 --- cmd/mesh-host/nodeagent.go | 3 + cmd/mesh-host/nodeagent_integration_test.go | 89 +++- internal/agent/agent.go | 11 +- internal/agent/agent_test.go | 6 + internal/agent/parts.go | 13 + internal/link/running.go | 12 + internal/nodegrant/testdata/node.json | 6 + internal/selfcompose/composedstate.go | 5 + internal/selfcompose/entries.go | 516 ++++++++++++++++++++ internal/selfcompose/entries_test.go | 145 ++++++ internal/selfcompose/selfcompose.go | 55 +++ internal/selfcompose/selfcompose_test.go | 3 +- module.json | 1 + 13 files changed, 849 insertions(+), 16 deletions(-) create mode 100644 internal/selfcompose/entries.go create mode 100644 internal/selfcompose/entries_test.go diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 2b5b8b9b..ef053d56 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -1101,6 +1101,9 @@ func selfComposer(session *link.Session, node string, hooks composerHooks, say f env.ComposedState = link.EngineComposed{Session: session, Node: node} // And each composition it writes there said as the seat's event composed, which the assigner waits on. env.SayComposed = session.SayComposed + // And the catalog half read by the fingerprints its wants and runnings name, over its own calls of the seat mesh-catalog's + // verb entry on its link session (design 47 G §421): neither the indexer's nor the builder's states are read. + env.Entry = selfcompose.EntryCall(session.AskMeshSeat) return selfcompose.New(env) } diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index fc86409e..4d3f4021 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -79,6 +79,55 @@ type itBed struct { // holder re-serves (issue 1966). seatAway atomic.Bool findings nats.KeyValue + // entries are the entries the stand-in of the seat mesh-catalog answers its verb entry from, by fingerprint (design 47 G §421). + entries map[string]meshcatalog.CatalogEntry +} + +// registerEntry makes the stand-in answer the entry of fp: module's manifest and its artifacts (refs `:`). +func (b *itBed) registerEntry(module, fp, manifest string, artifacts ...string) { + b.t.Helper() + var m map[string]any + if err := json.Unmarshal([]byte(manifest), &m); err != nil { + b.t.Fatal(err) + } + e := meshcatalog.CatalogEntry{Module: module, Fp: fp, Manifest: m, ManifestDigest: meshcatalogclient.ManifestDigest([]byte(manifest)), + Repository: "novox/" + module, Commit: strings.Repeat("1", 40)} + for _, a := range artifacts { + e.Artifacts = append(e.Artifacts, meshcatalog.Artifact{Ref: module + ":" + a, Kind: "archive", Language: "go"}) + } + b.mu.Lock() + b.entries[fp] = e + b.mu.Unlock() +} + +// running says the seat node-engine's event running of module on node at fp, as that node's engine does. +func (b *itBed) running(node, module, fp, manifestDigest string) { + b.t.Helper() + raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: node, Module: module, Fingerprint: fp, Manifest: manifestDigest, At: time.Now().UTC()}) + if _, err := b.js.Publish(nodeengine.SubjectRunning(node, module), raw); err != nil { + b.t.Fatal(err) + } +} + +// standInEntry answers the seat mesh-catalog's verb entry from the registered entries, and as the holder refuses an unknown one. +func (b *itBed) standInEntry() { + sub, err := b.conn.Subscribe("mesh.seat.mesh-catalog.call.entry.*", func(m *nats.Msg) { + var in meshcatalog.EntryInput + _ = json.Unmarshal(m.Data, &in) + b.mu.Lock() + e, ok := b.entries[in.Fingerprint] + b.mu.Unlock() + if !ok { + _ = m.Respond([]byte(`{"error":"no entry of fingerprint ` + in.Fingerprint + ` is kept"}`)) + return + } + out, _ := json.Marshal(map[string]any{"result": meshcatalog.EntryAnswer{Entry: e}}) + _ = m.Respond(out) + }) + if err != nil { + b.t.Fatal(err) + } + b.t.Cleanup(func() { _ = sub.Unsubscribe() }) } // aServer is an embedded bus with JetStream and two users: the test's, unrestricted, and the node's, granted exactly what @@ -131,7 +180,7 @@ func newItBedAs(t *testing.T, perms nodegrant.Permissions) *itBed { builtFor = "arch" t.Cleanup(func() { builtFor = was }) - b := &itBed{t: t, dir: t.TempDir()} + b := &itBed{t: t, dir: t.TempDir(), entries: map[string]meshcatalog.CatalogEntry{}} b.conn, b.nodeConn = aServer(t, perms, func(err error) { b.mu.Lock() b.refusals = append(b.refusals, err.Error()) @@ -167,7 +216,13 @@ func newItBedAs(t *testing.T, perms nodegrant.Permissions) *itBed { t.Fatal(err) } } + // The stream the seat node-engine's event running is kept in, the last of each subject (design 47 G §199). + if _, err = b.js.AddStream(&nats.StreamConfig{Name: nodeengine.StreamRunning, Subjects: []string{nodeengine.SubjectRunningAll}, + MaxMsgsPerSubject: 1}); err != nil { + t.Fatal(err) + } b.standInSeat() + b.standInEntry() b.opts = options{state: filepath.Join(b.dir, "state", "state.json"), timeout: 2 * time.Second} if err := os.MkdirAll(filepath.Dir(b.opts.state), 0o755); err != nil { t.Fatal(err) @@ -435,8 +490,7 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { path := filepath.Join(b.dir, "web.conf") man := `{"module":"web","version":"1","config":{"greeting":{"type":"string","description":"what the file says","default":"hello"}},` + `"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0644","content":"greeting=${config:greeting}\n"},` + - `{"id":"tools","type":"package","package":"web-tools"}],` + - `"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}` + `{"id":"tools","type":"package","package":"web-tools"}]}` digest := meshcatalogclient.ManifestDigest([]byte(man)) mraw, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "web", Manifest: man}) mkey, _ := meshcatalogclient.ManifestsKey(digest) @@ -464,9 +518,6 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { done, _ := builderbuilds.EncodeBuilderBuild(builderbuilds.BuilderBuild{State: builderbuilds.BuildDone, Manifest: digest, Artifacts: []builderbuilds.BuiltArtifact{{Name: "files", Kind: "archive", Reference: "artifact-store://web/files/blobs/" + fpDigest("f"), Digest: fpDigest("f")}}}) firstBuild := done - if checkBuild { - firstBuild = []byte(`{"v":1,"state":"no such state"}`) - } for _, w := range []struct { kv nats.KeyValue key string @@ -479,6 +530,13 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { t.Fatal(err) } } + // Design 47 G §421: the node reads the entry its want names, through the seat mesh-catalog's verb entry, and the part's digest + // from the want; what it hears another node run (the mailer on it2) is read by the same fingerprint. + if !checkBuild { + b.registerEntry("web", fp, man) + } + b.registerEntry("mailer", fpIt("b"), mailer) + b.running("it2", "mailer", fpIt("b"), mdigest) want, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fp, Self: true}) wkey := meshwants.WantsByWantNodeModuleKey(itNode, "web") if _, err := b.wants.Put(wkey, want); err != nil { @@ -499,17 +557,19 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { t.Fatal(err) } if checkBuild { - // The build of the placed module is read through the node's session: one that does not decode is said, and nothing - // is applied; a done one, then a config event, and the node composes. - within(t, 5*time.Second, "the node did not read web's build key through its session", func() bool { - b.mu.Lock() - defer b.mu.Unlock() - return strings.Contains(strings.Join(b.said, "\n"), "the builder's "+buildKey) + _ = buildKey + // The entry of the build its want names is asked of the catalog half through the node's session: one the half holds none + // of holds the want (entry-missing, naming the fingerprint) and nothing is applied; once the half holds it and it is + // asked again by hand, then a config event, and the node composes. + within(t, 5*time.Second, "the have does not say the entry of web's build is missing", func() bool { + h := b.have("web") + return h.Phase == nec.PhaseHeld && strings.HasPrefix(h.Detail, agent.EntryMissing) && strings.Contains(h.Detail, fp) }) if b.file("web") != "" { - t.Fatal("composed with a build key that does not decode") + t.Fatal("composed with an entry the catalog half does not hold") } - if _, err := builds.Put(buildKey, done); err != nil { + b.registerEntry("web", fp, man) + if err := composerNow.Load().EntryAgain(context.Background(), fp); err != nil { t.Fatal(err) } if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"bonjour","source":"module"}`)); err != nil { @@ -799,6 +859,7 @@ func TestNatsTheBusServerIsTakenOverAndReplacedOnlyWhenItsBuildChangesIt(t *test mraw, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "nats", Manifest: man}) mkey, _ := meshcatalogclient.ManifestsKey(digest) modv, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("3", 40)}) + b.registerEntry("nats", fp, man) // the node reads the entry its want names (design 47 G §421) if _, err := manifests.Put(mkey, mraw); err != nil { t.Fatal(err) } diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 9aa00adc..93def60b 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -504,6 +504,9 @@ const NotComposedDetail = "the node cannot compose it, and keeps running what it // composerMark says h is a mark NotComposed wrote: failed, its detail opening with NotComposedDetail. func composerMark(h Have) bool { + if h.Phase == string(nodeengineclient.PhaseHeld) && heldForEntry(h.Detail) { + return true + } return h.Phase == string(nodeengineclient.PhaseFailed) && strings.HasPrefix(h.Detail, NotComposedDetail) } @@ -647,8 +650,14 @@ func (a *Agent) NotComposedEach(ctx context.Context, why string, held map[string a.mu.Lock() a.notComposed[module] = mark a.mu.Unlock() + phase, detail := string(nodeengineclient.PhaseFailed), NotComposedDetail+why + if heldForEntry(why) { + // The entry its want names is not held: the phase held, its detail opening entry-asked: or entry-missing: and + // naming the fingerprint (design 47 G section 421). + phase, detail = string(nodeengineclient.PhaseHeld), why + } a.writeHave(ctx, Have{Module: module, FP: mark.prior.FP, Composed: mark.prior.Composed, Sealed: mark.prior.Sealed, - Phase: string(nodeengineclient.PhaseFailed), Detail: NotComposedDetail + why}) + Phase: phase, Detail: detail}) } } diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index a609f558..34268265 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -1772,6 +1772,12 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes( if h := b.haves.last("web"); h.Phase != string(nodeengineclient.PhaseRunning) || h.Composed != dig { t.Errorf("web, which composes, is marked beside idle: %+v", h) } + // Design 47 G §421: a module whose want's entry the catalog half holds none of is the phase held, its detail opening + // entry-missing: and naming the fingerprint; composed again, its have is what it runs. + b.ag.NotComposedEach(context.Background(), "", map[string]string{"idle": EntryMissing + " the catalog half holds no entry of the build src2:x"}) + if h := b.haves.last("idle"); h.Phase != string(nodeengineclient.PhaseHeld) || h.Composed != idleDig || h.Detail != EntryMissing+" the catalog half holds no entry of the build src2:x" { + t.Errorf("idle's have while its entry is missing: %+v", h) + } b.ag.NotComposedEach(context.Background(), "", nil) if h := b.haves.last("idle"); h.Phase != string(nodeengineclient.PhaseRunning) || h.Composed != idleDig { t.Errorf("idle's have once it composes again: %+v", h) diff --git a/internal/agent/parts.go b/internal/agent/parts.go index bb5a1f83..b324b9a7 100644 --- a/internal/agent/parts.go +++ b/internal/agent/parts.go @@ -13,6 +13,19 @@ import ( // disagree (design 47 G section 524). const PartsDiffer = "parts-differ:" +// EntryAsked and EntryMissing open the detail of a have held for the entry the want names (design 47 G section 421 and +// section 614): the entry is asked of the catalog half and not yet answered, or the half answered that it holds none. Both +// are the phase held, and each names the fingerprint. +const ( + EntryAsked = "entry-asked:" + EntryMissing = "entry-missing:" +) + +// heldForEntry says whether why is a hold for the entry a want names. +func heldForEntry(why string) bool { + return strings.HasPrefix(why, EntryAsked) || strings.HasPrefix(why, EntryMissing) +} + // partsDifference checks a want's `parts` against the digests of the archive, process and container resources of the // composition, which are the digests the apply fetches and checks (design 47 G section 524). A want with no `parts` (an // assigner of before G, or a node whose record does not state `want-parts`) is applied as today: it returns "". With diff --git a/internal/link/running.go b/internal/link/running.go index 722ce19a..fa24b0d1 100644 --- a/internal/link/running.go +++ b/internal/link/running.go @@ -93,3 +93,15 @@ func (q *Session) FollowRunning(ctx context.Context, heard func(nodeengine.Runni <-ctx.Done() return nil } + +// OverNATS reads the stream as the link's session does, over its connection. +func (b OverNATS) FirstRunning(ctx context.Context, first func([]byte)) error { + return (&natsLink{js: b.JS}).FirstRunning(ctx, first) +} + +// HearRunning follows the stream over the connection (see natsLink.HearRunning). +func (b OverNATS) HearRunning(consumer string, heard func([]byte)) (func(), error) { + return (&natsLink{js: b.JS}).HearRunning(consumer, heard) +} + +var _ RunningBus = OverNATS{} diff --git a/internal/nodegrant/testdata/node.json b/internal/nodegrant/testdata/node.json index 6e82524d..0270797c 100644 --- a/internal/nodegrant/testdata/node.json +++ b/internal/nodegrant/testdata/node.json @@ -21,7 +21,9 @@ "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.NODE.>", "$JS.API.CONSUMER.CREATE.OBJ_declaration-files-NODE.*.$O.declaration-files-NODE.C.*", "$JS.API.CONSUMER.CREATE.SEAT_mesh-conditions_findings-asked", + "$JS.API.CONSUMER.CREATE.SEAT_node-engine_running", "$JS.API.CONSUMER.CREATE.SEAT_mesh-conditions_findings-asked.>", + "$JS.API.CONSUMER.CREATE.SEAT_node-engine_running.>", "$JS.API.CONSUMER.DELETE.KV_composed-NODE.>", "$JS.API.CONSUMER.DELETE.KV_mesh-config_resolved.>", "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-assigner_assignments.>", @@ -41,6 +43,8 @@ "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.CONSUMER.DELETE.OBJ_declaration-files-NODE.*", "$JS.API.CONSUMER.DELETE.SEAT_mesh-conditions_findings-asked.>", + "$JS.API.CONSUMER.DELETE.SEAT_node-engine_running.>", + "$JS.API.CONSUMER.INFO.SEAT_node-engine_running.>", "$JS.API.CONSUMER.INFO.NODES.NODE", "$JS.API.DIRECT.GET.KV_composed-NODE.>", "$JS.API.DIRECT.GET.KV_mesh-config_resolved.$KV.mesh-config_resolved.NODE.>", @@ -79,6 +83,7 @@ "$JS.API.STREAM.INFO.KV_mesh-provisioner_provisions", "$JS.API.STREAM.INFO.KV_node-engine_have", "$JS.API.STREAM.INFO.SEAT_mesh-conditions_findings-asked", + "$JS.API.STREAM.INFO.SEAT_node-engine_running", "$JS.FC.KV_composed-NODE.>", "$JS.FC.KV_mesh-config_resolved.>", "$JS.FC.KV_mesh-delivery-assigner_assignments.>", @@ -110,6 +115,7 @@ "$SRV.PING.>", "$SRV.PING.node-tools.NODE", "mesh.control.NODE.>", + "mesh.seat.mesh-catalog.call.entry.node~NODE", "mesh.seat.mesh-catalog.call.seat-definitions.node~NODE", "mesh.seat.node-backup.call.backed-up.NODE.node~NODE", "mesh.seat.node-backup.call.now.NODE.node~NODE", diff --git a/internal/selfcompose/composedstate.go b/internal/selfcompose/composedstate.go index aef6dfbc..34c0f2c9 100644 --- a/internal/selfcompose/composedstate.go +++ b/internal/selfcompose/composedstate.go @@ -86,6 +86,11 @@ func (c *Composer) publishComposed(ctx context.Context) { // the indexer's, where the builder holds a done build of it; else the want's; else what runs of it. A module with none // is left out. func (c *Composer) newestFPs(ctx context.Context, at *published) map[string]string { + if c.byEntries() { + // The newest build is the indexer's (and the builder's done key says it is built): neither is read, and the node + // follows no branch (design 47 G §421, §628), so the state holds the wanted builds alone. + return maps.Clone(at.fps) + } out := map[string]string{} r := reader{c: c} for _, module := range at.placed { diff --git a/internal/selfcompose/entries.go b/internal/selfcompose/entries.go new file mode 100644 index 00000000..6b00c5b1 --- /dev/null +++ b/internal/selfcompose/entries.go @@ -0,0 +1,516 @@ +package selfcompose + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "reflect" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + + "github.com/novox/mesh-host/internal/agent" +) + +// **The catalog half, read by fingerprint** (design 47 G §421, §683, §698; the lead's answers A-47 and A-50). The node-engine +// follows no branch: for each build its wants and the runnings it heard name, it holds the catalog graph's entry of that +// fingerprint, asked of the seat mesh-catalog's verb `entry` once (meshcatalogclient.Entries) and kept in its composition +// state `composed-` beside the compositions (`entry.`, the entry as `entry` answers it), with each want it +// composes from (`want.`), written whole before the composition that uses them and read at its start before it asks +// anything. A module's manifest is the entry's, and each artifact's reference is made from the digest its want's parts name +// (referenceOf), so it reads neither the indexer's states nor the builder's. + +// The keys of the kept entries and wants in the composition state. +const ( + keptEntryPrefix = "entry." + keptWantPrefix = "want." + // keptWithin bounds one write of the composition state from an answer of the catalog half. + keptWithin = 15 * time.Second +) + +// keptWant is a want as the composition state keeps it (§683): the want beside when this node first kept it. +type keptWant struct { + V int `json:"v"` + Since time.Time `json:"since"` + Want meshwants.WantsByWantNodeModuleValue `json:"want"` +} + +// keptEntry is an entry held from the composition state or an answer, with when it was kept. +type keptEntry struct { + entry meshcatalog.CatalogEntry + since time.Time +} + +// EntryRow is one entry the node-engine keeps, as the seat's verb first-views lists it (§698). +type EntryRow struct { + Fingerprint string `json:"fingerprint"` + State string `json:"state"` + Since time.Time `json:"since"` +} + +// WantRow is one want the node-engine keeps, as the seat's verb first-views lists it (§698). +type WantRow struct { + Module string `json:"module"` + Fingerprint string `json:"fingerprint"` + KeptSince time.Time `json:"kept-since"` +} + +// byEntries says the composer reads the catalog half by fingerprint (the node's own composer), as the shadow, which keeps +// its reads of the indexer until it goes with the checker (§628), does not. +func (c *Composer) byEntries() bool { return c.entries != nil } + +// newEntries makes the composer's cache of entries over env's call of `entry`. +func (c *Composer) newEntries() { + if c.env.Entry == nil { + return + } + c.entries = meshcatalogclient.NewEntries(c.env.Entry) + c.entries.Now = c.now + c.entries.Changed = c.entryChanged + c.keptEnt = map[string]keptEntry{} + c.keptWants = map[string]bool{} + c.keptWantVal = map[string]keptWant{} +} + +// entryKey is an entry's key in the composition state, wantKey a want's. +// +// The fingerprint is keyed by its digits, as the compositions' keys are (engcomp.Key): a key holds no colon. +func entryKey(fp string) string { + return keptEntryPrefix + strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2) +} + +func wantStateKey(module string) string { return keptWantPrefix + module } + +// loadKept reads the entries and wants the composition state keeps, once at the start, before anything is asked of the catalog +// half (§683). A want read stands only until the live read of the wants answers (endKeptWants). A state that cannot be read +// is said, and what it would have kept is asked of `entry` and read from the wants as they come. +func (c *Composer) loadKept(ctx context.Context) { + if !c.byEntries() || c.env.ComposedState == nil { + return + } + kept, err := c.env.ComposedState.Entries(ctx) + if err != nil { + c.sayOnceEach("the entries and wants this node kept cannot be read, so each entry its wants name is asked of the catalog half: " + err.Error()) + return + } + now := c.now() + c.mu.Lock() + defer c.mu.Unlock() + for key, e := range kept { + switch { + case strings.HasPrefix(key, keptEntryPrefix): + var a meshcatalog.EntryAnswer + if json.Unmarshal(e.Value, &a) != nil || a.Entry.Module == "" { + c.env.Say("the kept entry " + key + " cannot be read, and is asked of the catalog half again") + continue + } + c.keptEnt[meshcatalogclient.SourcePrefix2+strings.TrimPrefix(key, keptEntryPrefix)] = keptEntry{entry: a.Entry, since: now} + case strings.HasPrefix(key, keptWantPrefix): + var w keptWant + module := strings.TrimPrefix(key, keptWantPrefix) + if json.Unmarshal(e.Value, &w) != nil || w.Want.Fp == "" { + c.env.Say("the kept want " + key + " cannot be read, so it stands until the wants are read") + continue + } + if _, live := c.wants[module]; !live { + c.wants[module] = w.Want + c.keptWants[module] = true + } + c.keptWantVal[module] = w + } + } +} + +// endKeptWants ends the standing of the kept wants once the live read of the wants answered: from then the wants the seat +// says are the ones the node composes from, never a want older than its stream's (§683). A kept want the read did not +// announce is gone. +func (c *Composer) endKeptWants() { + if !c.byEntries() { + return + } + c.mu.Lock() + defer c.mu.Unlock() + for module := range c.keptWants { + delete(c.wants, module) + delete(c.keptWants, module) + } +} + +// entryChanged is told of each entry answered or found missing: an answer is kept in the composition state before the pass it +// asks (so the composition that uses it finds it kept), and a pass is asked. +func (c *Composer) entryChanged(row meshcatalogclient.EntryRow) { + if row.State == meshcatalogclient.EntryAnswered && row.Entry != nil { + c.mu.Lock() + c.keptEnt[row.Fingerprint] = keptEntry{entry: *row.Entry, since: row.Since} + c.mu.Unlock() + c.keep(entryKey(row.Fingerprint), meshcatalog.EntryAnswer{Entry: *row.Entry}) + } + c.askRehand() +} + +// keep writes one value into the composition state; a state that cannot be written is said, and the value is kept again by +// the next pass. +func (c *Composer) keep(key string, v any) { + if c.env.ComposedState == nil { + return + } + raw, err := json.Marshal(v) + if err != nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), keptWithin) + defer cancel() + c.keptWrite.Lock() + defer c.keptWrite.Unlock() + if _, err := c.env.ComposedState.Put(ctx, key, raw); err != nil { + c.sayOnceEach("the node's composition state cannot keep " + key + ": " + err.Error()) + } +} + +// forget deletes one key from the composition state. +func (c *Composer) forget(key string) { + if c.env.ComposedState == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), keptWithin) + defer cancel() + c.keptWrite.Lock() + defer c.keptWrite.Unlock() + if err := c.env.ComposedState.Delete(ctx, key); err != nil { + c.sayOnceEach("the node's composition state cannot drop " + key + ": " + err.Error()) + } +} + +// entryOf is the entry of fp as the composer holds it: kept, or asked of the catalog half once (Entries.Need). +func (c *Composer) entryOf(fp string) (meshcatalog.CatalogEntry, meshcatalogclient.EntryState) { + c.mu.Lock() + k, ok := c.keptEnt[fp] + c.mu.Unlock() + if ok { + return k.entry, meshcatalogclient.EntryAnswered + } + return c.entries.Need(context.Background(), fp) +} + +// wantedBuilds is each fingerprint a want of this node or a running heard names, of the new form, with whether a want names it. +func (c *Composer) wantedBuilds() map[string]bool { + out := map[string]bool{} + c.mu.Lock() + defer c.mu.Unlock() + for module, w := range c.wants { + if module != agent.NodeModule && strings.HasPrefix(w.Fp, meshcatalogclient.SourcePrefix2) { + out[w.Fp] = true + } + } + for _, mods := range c.heard { + for _, run := range mods { + if !run.stopped && strings.HasPrefix(run.fingerprint, meshcatalogclient.SourcePrefix2) { + if _, named := out[run.fingerprint]; !named { + out[run.fingerprint] = false + } + } + } + } + return out +} + +// syncEntries asks the entry of each build a want or a running names, drops the one nothing names any more, keeps each want it +// composes from, and lays the picture frombus reads (c.modules, c.manifests) from the entries it holds. It runs at the start of +// each pass. +func (c *Composer) syncEntries(ctx context.Context) { + if !c.byEntries() { + return + } + need := c.wantedBuilds() + for fp := range need { + c.entryOf(fp) + } + for _, row := range c.entries.Rows() { + if _, named := need[row.Fingerprint]; !named { + c.entries.Release(row.Fingerprint) + } + } + c.mu.Lock() + var drop []string + for fp := range c.keptEnt { + if _, named := need[fp]; !named { + delete(c.keptEnt, fp) + drop = append(drop, entryKey(fp)) + } + } + type pending struct { + module string + kw keptWant + } + var put []pending + var gone []string + now := c.now() + for module, w := range c.wants { + if module == agent.NodeModule || c.keptWants[module] { + continue + } + had, ok := c.keptWantVal[module] + if ok && reflect.DeepEqual(had.Want, w) { + continue + } + since := now + if ok && had.Want.Fp == w.Fp { + since = had.Since + } + kw := keptWant{V: 1, Since: since, Want: w} + c.keptWantVal[module] = kw + put = append(put, pending{module, kw}) + } + for module := range c.keptWantVal { + if _, held := c.wants[module]; !held { + delete(c.keptWantVal, module) + gone = append(gone, wantStateKey(module)) + } + } + c.mu.Unlock() + // Written whole before the composition that uses them (§683). + for _, p := range put { + c.keep(wantStateKey(p.module), p.kw) + } + for _, key := range append(drop, gone...) { + c.forget(key) + } + c.layModules() +} + +// layModules lays c.modules and c.manifests from the entries held: what each node was heard to run first, then each module this +// node's want names, at the want's fingerprint (a want wins). +func (c *Composer) layModules() { + c.mu.Lock() + defer c.mu.Unlock() + modules := map[string]meshcatalog.ModulesByModuleValue{} + lay := func(module, fp string) { + k, ok := c.keptEnt[fp] + if !ok || k.entry.Module == "" { + return + } + e := k.entry + digest, raw, err := entryManifest(e) + if err != nil { + return + } + m := meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: e.Repository, Commit: e.Commit, Path: e.Path, + Needs: e.Needs, Packages: e.Packages, Verdict: e.Verdict} + if e.Inputs != nil { + m.Problems, m.Warnings, m.Unknown = e.Inputs.Problems, e.Inputs.Warnings, e.Inputs.Unknown + } + modules[module] = m + c.manifests[digest] = raw + } + nodes := make([]string, 0, len(c.heard)) + for node := range c.heard { + nodes = append(nodes, node) + } + sort.Strings(nodes) + for _, node := range nodes { + for module, run := range c.heard[node] { + if !run.stopped && run.fingerprint != "" { + lay(module, run.fingerprint) + } + } + } + for module, w := range c.wants { + if module != agent.NodeModule && w.Fp != "" { + lay(module, w.Fp) + } + } + // A module an assignment places whose entry is not held (not wanted here, not heard running, or its entry asked and not + // answered) stands in the catalogue as its name alone, so the placement still finds it: a module held for its entry offers + // nothing to the rest until its entry is held (§421). + for _, p := range c.assignments { + for _, pm := range p.Modules { + if _, held := modules[pm.Module]; held || pm.Module == "" { + continue + } + raw, _ := json.Marshal(map[string]any{"module": pm.Module, "version": "0"}) + sum := sha256.Sum256(raw) + digest := "sha256:" + hex.EncodeToString(sum[:]) + // At the fingerprint its want names, so it is no build behind the catalog's and its hold (entryHolds) is what says it. + modules[pm.Module] = meshcatalog.ModulesByModuleValue{Manifest: digest, Fp2: c.wants[pm.Module].Fp} + c.manifests[digest] = raw + } + } + c.modules = modules +} + +// entryManifest is an entry's manifest as a reader takes it, with its digest: the one the indexer set at take-in, or, where it set +// none, the digest of the manifest's bytes. +func entryManifest(e meshcatalog.CatalogEntry) (digest string, raw []byte, err error) { + if raw, err = json.Marshal(e.Manifest); err != nil { + return "", nil, err + } + digest = e.ManifestDigest + if digest == "" { + sum := sha256.Sum256(raw) + digest = "sha256:" + hex.EncodeToString(sum[:]) + } + return digest, raw, nil +} + +// entryHolds is, of each module this node's want names, why its entry is not held: asked and not answered, or answered not +// found. It is the phase held with a detail opening entry-asked: or entry-missing: and naming the fingerprint (§421, §614). +func (c *Composer) entryHolds() map[string]string { + if !c.byEntries() { + return nil + } + c.mu.Lock() + wants := map[string]meshwants.WantsByWantNodeModuleValue{} + for module, w := range c.wants { + if module != agent.NodeModule { + wants[module] = w + } + } + c.mu.Unlock() + out := map[string]string{} + for module, w := range wants { + if !strings.HasPrefix(w.Fp, meshcatalogclient.SourcePrefix2) { + continue + } + switch e, state := c.entryOf(w.Fp); state { + case meshcatalogclient.EntryAsked: + out[module] = fmt.Sprintf("%s the entry of the build %s is asked of the catalog half and not yet answered, so the node composes nothing new of %s", + agent.EntryAsked, w.Fp, module) + case meshcatalogclient.EntryNotFound: + out[module] = fmt.Sprintf("%s the catalog half holds no entry of the build %s, so the node composes nothing new of %s", + agent.EntryMissing, w.Fp, module) + default: + if missing := partsMissing(e, w); missing != "" { + out[module] = fmt.Sprintf("the want of %s names no digest for %s, so the node composes nothing new of it", module, missing) + } + } + } + return out +} + +// partsMissing names the artifacts of e whose digest w's parts do not give; "" when each has one. +func partsMissing(e meshcatalog.CatalogEntry, w meshwants.WantsByWantNodeModuleValue) string { + var out []string + for _, a := range e.Artifacts { + if w.Parts[a.Ref] == "" { + out = append(out, a.Ref) + } + } + sort.Strings(out) + return strings.Join(out, ", ") +} + +// referenceOf is an artifact's reference made from the digest its want's part names, in the form the artifact store is +// addressed by (the lead's answer A-50): artifact-store:////blobs/. +func referenceOf(module, artifact, digest string) string { + return "artifact-store://" + module + "/" + artifact + "/blobs/" + digest +} + +// buildOf is the build a want names, made from its entry and its parts alone (the builder's done key is not read): done, with +// each artifact of the entry at the digest its part names and the reference made from it. Nil where the entry is not held, the +// want names another fingerprint, or a part is missing. +func (c *Composer) buildOf(module, fp string) *builds.BuilderBuild { + c.mu.Lock() + w, wanted := c.wants[module] + k, ok := c.keptEnt[fp] + c.mu.Unlock() + if !wanted || w.Fp != fp || !ok || partsMissing(k.entry, w) != "" { + return nil + } + digest, raw, err := entryManifest(k.entry) + if err != nil { + return nil + } + c.mu.Lock() + c.manifests[digest] = raw + c.mu.Unlock() + b := builds.BuilderBuild{State: builds.BuildDone, Manifest: digest} + for _, a := range k.entry.Artifacts { + _, name, _ := strings.Cut(a.Ref, ":") + digest := w.Parts[a.Ref] + b.Artifacts = append(b.Artifacts, builds.BuiltArtifact{Name: name, Kind: a.Kind, Digest: digest, + Reference: referenceOf(module, name, digest)}) + } + return &b +} + +// FirstViews is the node-engine's first views as the seat's verb first-views lists them (§698): the views it follows (none: it +// follows no branch), the entries it keeps and the wants it composes from. +func (c *Composer) FirstViews() (views []map[string]any, entries []EntryRow, wants []WantRow) { + views, entries, wants = []map[string]any{}, []EntryRow{}, []WantRow{} + if !c.byEntries() { + return + } + seen := map[string]bool{} + for _, r := range c.entries.Rows() { + seen[r.Fingerprint] = true + entries = append(entries, EntryRow{Fingerprint: r.Fingerprint, State: string(r.State), Since: r.Since}) + } + c.mu.Lock() + for fp, k := range c.keptEnt { + if !seen[fp] { + entries = append(entries, EntryRow{Fingerprint: fp, State: string(meshcatalogclient.EntryAnswered), Since: k.since}) + } + } + for module, w := range c.wants { + if module == agent.NodeModule || w.Fp == "" { + continue + } + wants = append(wants, WantRow{Module: module, Fingerprint: w.Fp, KeptSince: c.keptWantVal[module].Since}) + } + c.mu.Unlock() + sort.Slice(entries, func(i, j int) bool { return entries[i].Fingerprint < entries[j].Fingerprint }) + sort.Slice(wants, func(i, j int) bool { return wants[i].Module < wants[j].Module }) + return +} + +// EntryAgain asks fp's entry of the catalog half again now (the seat's verb entry-again, §698): one the composer keeps, whatever +// its state. The entry kept of it is dropped, so a pass holds the module until the answer. +func (c *Composer) EntryAgain(ctx context.Context, fp string) error { + if !c.byEntries() { + return fmt.Errorf("this engine reads no entry by fingerprint") + } + c.mu.Lock() + _, kept := c.keptEnt[fp] + delete(c.keptEnt, fp) + c.mu.Unlock() + held := kept + for _, r := range c.entries.Rows() { + held = held || r.Fingerprint == fp + } + if !held { + return fmt.Errorf("no entry of %s is kept", fp) + } + if !kept { + return c.entries.Again(ctx, fp) + } + c.entries.Need(ctx, fp) // makes the row and asks once + c.askRehand() + return nil +} + +// EntryCall is env.Entry over the engine's link session: the seat mesh-catalog's verb `entry`, read in the engine's own name +// (the manifest's invokes names it). A refusal in the verb's own words that it keeps no entry of the build is the holder's +// answer that it holds none, and is told so; any other failure is a call that did not reach the holder, and is told so too. +func EntryCall(ask func(ctx context.Context, seat, verb string, args any) (json.RawMessage, error)) meshcatalogclient.Call { + return func(key string, body any) (json.RawMessage, error) { + seatName, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".") + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + raw, err := ask(ctx, seatName, verb, body) + if err != nil && strings.Contains(err.Error(), noEntryWords) { + out, _ := json.Marshal(map[string]string{"error": err.Error()}) + return out, nil + } + return raw, err + } +} + +// noEntryWords are the words the seat's holder refuses an unknown fingerprint in. +const noEntryWords = "no entry of fingerprint" diff --git a/internal/selfcompose/entries_test.go b/internal/selfcompose/entries_test.go new file mode 100644 index 00000000..7136b509 --- /dev/null +++ b/internal/selfcompose/entries_test.go @@ -0,0 +1,145 @@ +package selfcompose + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + + "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" + "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" + nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "github.com/novox/mesh-host/internal/agent" + "github.com/novox/mesh-host/internal/buckets" + "github.com/novox/mesh-host/internal/buckets/bucketstest" + "github.com/novox/mesh-host/internal/compositions" +) + +// Design 47 G §421, §683, §698 (the lead's answers A-47, A-50): the node-engine reads the entry its want names by the +// fingerprint, through its own calls of the seat mesh-catalog's verb entry, and reads neither the indexer's states nor the +// builder's; a want whose entry the catalog half holds none of is held, its have saying entry-missing and naming the +// fingerprint; the entries and wants it composes from are kept in its composition state, written before the composition that +// uses them and read at its start before anything is asked; a want removed takes its kept want and entry away. +func composeFromEntries(t *testing.T) { + b := newFakeBus() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) + ghost := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64) + var manifest map[string]any + if err := json.Unmarshal([]byte(lemurs), &manifest); err != nil { + t.Fatal(err) + } + entry := meshcatalog.CatalogEntry{Module: "lemurs", Fp: fp, Manifest: manifest, Repository: "novox/x", Commit: "c", + ManifestDigest: meshcatalogclient.ManifestDigest([]byte(lemurs))} + var mu sync.Mutex + var asked []string + ask := func(_ context.Context, seat, verb string, args any) (json.RawMessage, error) { + in := args.(meshcatalog.EntryInput) + mu.Lock() + asked = append(asked, in.Fingerprint) + mu.Unlock() + if seat != "mesh-catalog" || verb != "entry" { + t.Errorf("asked %s.%s", seat, verb) + } + if in.Fingerprint != fp { + return nil, errors.New("no entry of fingerprint " + in.Fingerprint + " is kept") + } + return json.Marshal(meshcatalog.EntryAnswer{Entry: entry}) + } + state := &fakeComposed{kv: map[string]ComposedEntry{}} + var held map[string]string + env := b.env() + env.Entry, env.ComposedState = EntryCall(ask), state + env.Refused = func(_ string, h map[string]string) { mu.Lock(); held = h; mu.Unlock() } + c := New(env) + out, err := c.Watch(ctx, "n1.>") + if err != nil { + t.Fatal(err) + } + place := assignedValues(t, meshmoduleassigner.Assignments{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"}, + Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}, {Module: "ghost"}}}) + have, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}}) + nodeWant, _ := bucketstest.NodeWant(meshwants.WantsByWantNodeNodeValue{Self: true}) + b.putAssigned(t, place) + b.put(t, buckets.Have, "n1.node", have) + b.put(t, buckets.Wants, "want.n1.node", nodeWant) + for module, w := range map[string]meshwants.WantsByWantNodeModuleValue{"lemurs": {Fp: fp, Self: true}, "ghost": {Fp: ghost, Self: true}} { + raw, _ := bucketstest.Want(w) + b.put(t, buckets.Wants, "want.n1."+module, raw) + } + for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, ResolvedBucket} { + b.send(bucket, agent.Update{Caught: true}) + } + lemursKey, _ := compositions.Key("n1", "lemurs", fp) + handed := map[string]bool{} + for !handed[lemursKey] { + handed[next(t, out).Key] = true + } + // Neither the indexer's states nor the builder's are watched or read. + b.mu.Lock() + for _, r := range b.reads { + if r[0] == buckets.Modules || r[0] == buckets.Manifests || r[0] == buckets.Verdicts || r[0] == builds.BuilderBuildsBucket { + t.Errorf("the node-engine read %s of %s", r[1], r[0]) + } + } + b.mu.Unlock() + // A want whose entry the half holds none of is held, saying entry-missing and the fingerprint; the rest is composed. + mu.Lock() + why := held["ghost"] + mu.Unlock() + if !strings.HasPrefix(why, agent.EntryMissing) || !strings.Contains(why, ghost) || held["lemurs"] != "" { + t.Errorf("held %q, want ghost held with %s naming %s", held, agent.EntryMissing, ghost) + } + // Each entry and want it composes from is kept in the composition state before the composition that uses them. + state.mu.Lock() + _, hasEntry := state.kv[entryKey(fp)] + _, hasWant := state.kv["want.lemurs"] + state.mu.Unlock() + if !hasEntry || !hasWant { + t.Errorf("the composition state holds entry %v and want %v of lemurs: %v", hasEntry, hasWant, state.written()) + } + // The seat's first-views lists them. + _, entries, wants := c.FirstViews() + if len(entries) < 2 || len(wants) != 2 { + t.Errorf("first-views lists %v and %v", entries, wants) + } + // A restart reads what is kept before it asks anything: lemurs's entry is not asked again. + mu.Lock() + asked = nil + mu.Unlock() + env2 := b.env() + env2.Entry, env2.ComposedState = EntryCall(ask), state + c2 := New(env2) + if _, err := c2.Watch(ctx, "n1.>"); err != nil { + t.Fatal(err) + } + c2.mu.Lock() + kept, keptWant := c2.keptEnt[fp], c2.wants["lemurs"].Fp + c2.mu.Unlock() + if kept.entry.Module != "lemurs" || keptWant != fp { + t.Errorf("a restart read entry %q and want %q from the state", kept.entry.Module, keptWant) + } + // A want removed takes its kept want and entry away. + raw, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fp, Removed: true}) + b.put(t, buckets.Wants, "want.n1.lemurs", raw) + for range 20 { + state.mu.Lock() + _, hasWant = state.kv["want.lemurs"] + _, hasEntry = state.kv[entryKey(fp)] + state.mu.Unlock() + if !hasWant && !hasEntry { + break + } + next(t, out) + } + if hasWant || hasEntry { + t.Errorf("a removed want left its kept want %v and entry %v", hasWant, hasEntry) + } +} diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index d4c84594..91753a13 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -126,6 +126,10 @@ type Env struct { // SeatListChanged says the reader's held set changed: the node composes again, so what waited for the list is // handed at the read and not on a timer. Nil hears nothing. SeatListChanged <-chan struct{} + // Entry asks the seat mesh-catalog's verb `entry` of a fingerprint (meshcatalogclient.EntryVerb), in the engine's own name. + // Set, the composer reads the catalog half by the fingerprints its wants and the runnings it heard name and nothing else of + // the indexer's or the builder's states (design 47 G §421, §628); nil reads the indexer's states, as the shadow does. + Entry meshcatalogclient.Call } // ManifestOf is the digest of the manifest of the build a module runs at fp, from the composer's picture of the catalog half; @@ -175,6 +179,14 @@ type Composer struct { notTaken map[string]string // unmadeSaid is, by module, the stable part of the reason its last journal line said (without where the picture stands). unmadeSaid map[string]string + // entries is the cache of entries asked by fingerprint, nil where the composer reads the indexer's states; keptEnt the entries + // held from the composition state or an answer, keptWantVal the wants it keeps there and keptWants the modules whose want + // stands from that state alone until the live read of the wants answers (entries.go). Under mu; keptWrite orders the writes. + entries *meshcatalogclient.Entries + keptEnt map[string]keptEntry + keptWantVal map[string]keptWant + keptWants map[string]bool + keptWrite sync.Mutex // modulesRev is the revision of the last update of the indexer's modules taken: where the picture of them stands. modulesRev uint64 // heard is, by node and module, what each node's event running last said it runs, kept from the seat's stream by this @@ -374,6 +386,12 @@ func New(env Env) *Composer { if env.Say == nil { env.Say = func(string) {} } + c := newComposer(env) + c.newEntries() + return c +} + +func newComposer(env Env) *Composer { return &Composer{env: env, wants: map[string]meshwants.WantsByWantNodeModuleValue{}, assignments: map[string]meshmoduleassigner.Assignments{}, assignmentKeys: meshmoduleassignerclient.NewKeys(), records: map[string]nodeengine.NodeRecord{}, inOwn: map[string]bool{}, modules: map[string]meshcatalog.ModulesByModuleValue{}, values: map[string]map[string]any{}, manifests: map[string][]byte{}, archives: map[string][]byte{}, builds: map[string]builds.BuilderBuild{}, handed: map[string]string{}, notTaken: map[string]string{}, unmadeSaid: map[string]string{}, rehand: make(chan struct{}, 1), sealedOwn: map[string]ownSealed{}, @@ -437,6 +455,21 @@ type input struct { } func (c *Composer) inputs() []input { + ins := c.allInputs() + if !c.byEntries() { + return ins + } + // Reading the catalog half by fingerprint, it watches neither the indexer's modules nor the builder's builds (§421, §628). + out := ins[:0:0] + for _, in := range ins { + if in.bucket != buckets.Modules && in.bucket != builds.BuilderBuildsBucket { + out = append(out, in) + } + } + return out +} + +func (c *Composer) allInputs() []input { node := c.env.Node return []input{ {buckets.Wants, "want." + node + ".>", (*Composer).takeWant, nil}, @@ -491,6 +524,8 @@ func (c *Composer) takeBuildDone(u agent.Update) bool { // input's watch ends. func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, error) { ins := c.inputs() + // What the node kept of the entries and the wants it composes from is read before anything is asked (§683). + c.loadKept(ctx) type tagged struct { in int u agent.Update @@ -598,6 +633,9 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er dirty = true // a secret changed: its new generation is asked of the vault at the compose case t.u.Caught: caught[t.in] = true + if ins[t.in].bucket == buckets.Wants { + c.endKeptWants() // the wants the seat says now are the ones composed from (§683) + } case ins[t.in].changed != nil: dirty = ins[t.in].changed(c, t.u) || dirty default: @@ -759,6 +797,7 @@ func (c *Composer) takeWantLocked(u agent.Update) string { return "" } delete(c.notTaken, module) + delete(c.keptWants, module) w, err := meshwants.DecodeWantsByWantNodeModuleValue(u.Value) if u.Deleted || err != nil || w.Removed || w.Fp == "" { delete(c.wants, module) @@ -960,6 +999,7 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { } } }() + c.syncEntries(ctx) c.mu.Lock() fps := map[string]string{} kept := map[string]string{} @@ -1024,6 +1064,15 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { // (ADR 0636 §1): the module is held as it runs before anything of its build is read, so no take-in or artifact // reference of it (the artifact store's seat is not bootstrapped) can refuse the pass (issue 1966). scope.Held = c.heldUntilListed(ctx, r, own) + // A want whose entry the catalog half has not answered, or holds none of, is held with what runs of it (§421). + if holds := c.entryHolds(); len(holds) > 0 { + if scope.Held == nil { + scope.Held = map[string]string{} + } + for module, why := range holds { + scope.Held[module] = why + } + } missed := c.missed.Load() compositions, skipped, held, err := frombus.Compose(ctx, r, c.env.Mesh, scope, fps) err = c.missedIn(missed, err) @@ -1533,6 +1582,9 @@ func (r reader) Manifest(ctx context.Context, digest string) ([]byte, error) { if ok { return raw, nil } + if r.c.byEntries() { + return nil, fmt.Errorf("the catalog half's entries hold no manifest %s", digest) + } key, err := meshcatalogclient.ManifestsKey(digest) if err != nil { return nil, err @@ -1554,6 +1606,9 @@ func (r reader) Manifest(ctx context.Context, digest string) ([]byte, error) { } func (r reader) Build(ctx context.Context, module, fp string) (*builds.BuilderBuild, error) { + if r.c.byEntries() { + return r.c.buildOf(module, fp), nil + } key, err := builds.BuilderBuildKey(module, fp) if err != nil { return nil, err diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index f6d1f386..af655ddb 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -138,7 +138,7 @@ func next(t *testing.T, out <-chan agent.Update) agent.Update { case u := <-out: return u case <-time.After(5 * time.Second): - t.Fatal("no update from the composer") + t.Fatalf("no update from the composer") return agent.Update{} } } @@ -156,6 +156,7 @@ func quiet(t *testing.T, out <-chan agent.Update) { // every input was read, hands each, then Caught; an input that changes nothing composed hands nothing; a module no want // names any more is taken back; a node that cannot compose says so once and hands nothing. func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { + t.Run("it reads the catalog half by the fingerprints its wants name", composeFromEntries) b := newFakeBus() fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) digest := meshcatalogclient.ManifestDigest([]byte(lemurs)) diff --git a/module.json b/module.json index 52f97f17..2fef2be6 100644 --- a/module.json +++ b/module.json @@ -21,6 +21,7 @@ } ], "invokes": [ + "seat:mesh-catalog.entry", "seat:mesh-catalog.seat-definitions" ], "consumes": [ -- 2.54.0 From 97d3df3e59ce9ac02cb0d6138247b8356eb26e48 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:27:28 +0100 Subject: [PATCH 16/59] =?UTF-8?q?G=20=C2=A7422,=20=C2=A7698:=20the=20node-?= =?UTF-8?q?engine=20serves=20first-views,=20first-view-again=20and=20entry?= =?UTF-8?q?-again,=20and=20its=20manifest=20names=20the=20verbs=20it=20ser?= =?UTF-8?q?ves?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit first-views answers {first-views, entries, wants}, entry-again asks one entry again as a recorded hand act and answers {asked, read}; first-view-again names no view the engine follows (it follows no branch) and refuses in words. module.json's claim of node-engine lists every verb the engine serves. Builds: design 47 G §422, §698 --- cmd/mesh-host/engineseat.go | 24 ++++++++++++++ cmd/mesh-host/main.go | 1 + internal/link/handacts.go | 59 ++++++++++++++++++++++++++++++++++ internal/link/handover_test.go | 19 +++++++++-- module.json | 21 +++++++++++- 5 files changed, 121 insertions(+), 3 deletions(-) diff --git a/cmd/mesh-host/engineseat.go b/cmd/mesh-host/engineseat.go index 31c7731c..2b14c308 100644 --- a/cmd/mesh-host/engineseat.go +++ b/cmd/mesh-host/engineseat.go @@ -390,3 +390,27 @@ func composeCandidateFrom(node string, shadow *atomic.Pointer[selfcompose.Shadow return json.RawMessage(raw), nil } } + +// firstViewsFrom answers the seat's verb first-views from the composer of this run (design 47 G §698): the views the node-engine +// follows (none), the entries it keeps and the wants it composes from. +func firstViewsFrom(composer *atomic.Pointer[selfcompose.Composer]) func() (any, error) { + return func() (any, error) { + c := composer.Load() + if c == nil { + return nil, errors.New("this node composes none of its own modules now (it applies the controller's compositions, or its node agent is starting), so it keeps no entry") + } + views, entries, wants := c.FirstViews() + return map[string]any{"first-views": views, "entries": entries, "wants": wants}, nil + } +} + +// entryAgainFrom asks one entry of the catalog half again, for the seat's verb entry-again (design 47 G §698). +func entryAgainFrom(composer *atomic.Pointer[selfcompose.Composer]) func(ctx context.Context, fingerprint string) error { + return func(ctx context.Context, fingerprint string) error { + c := composer.Load() + if c == nil { + return errors.New("this node composes none of its own modules now, so it keeps no entry to ask again") + } + return c.EntryAgain(ctx, fingerprint) + } +} diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index cf5c9e84..193efcc6 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1019,6 +1019,7 @@ func runLink(ctx context.Context, opts options) error { queue.Acts.Composition = compositionFrom(current) queue.Acts.Compose = composeFrom(current, &composerNow) queue.Acts.Packages = packagesFrom(mine.Node, queue, &composerNow) + queue.Acts.FirstViews, queue.Acts.EntryAgain = firstViewsFrom(&composerNow), entryAgainFrom(&composerNow) queue.Acts.SeatList, queue.Acts.SeatListAgain = seatListRead, seatListAgain queue.Acts.ProcessBuilds = func(module string) (any, error) { d, err := declaredNow(opts.state, nil) diff --git a/internal/link/handacts.go b/internal/link/handacts.go index 28ad5861..83ae1106 100644 --- a/internal/link/handacts.go +++ b/internal/link/handacts.go @@ -1,6 +1,7 @@ package link import ( + "context" "crypto/rand" "crypto/sha256" "encoding/hex" @@ -55,6 +56,12 @@ const ( // (novox/hq ADR 0636 §7). The node-engine seat's definition carries both verbs. ActSeatList = nodeengine.VerbSeatList ActSeatListAgain = nodeengine.VerbSeatListAgain + // ActFirstViews reads the node-engine's first views: the views it follows, the entries it keeps of the builds its wants + // and runnings name, and the wants it composes from; ActFirstViewAgain asks a view again and ActEntryAgain an entry + // (design 47 G §422, §698). + ActFirstViews = nodeengine.VerbFirstViews + ActFirstViewAgain = nodeengine.VerbFirstViewAgain + ActEntryAgain = nodeengine.VerbEntryAgain // ActProcessBuilds reads the build each process on this node was started with (design 47, *The fingerprint handover*); // it reads only; who may call it is the seat definition's (design 47 G §440, §447). ActProcessBuilds = nodeengine.VerbProcessBuilds @@ -159,6 +166,10 @@ type HandActs struct { // SeatList answers the seat list the engine holds; SeatListAgain makes it read the list now and answers what it holds. SeatList func() (any, error) SeatListAgain func() (any, error) + // FirstViews answers the verb first-views; EntryAgain asks one entry of the catalog half again, refusing in words one the + // engine keeps none of. + FirstViews func() (any, error) + EntryAgain func(ctx context.Context, fingerprint string) error // ProcessBuilds answers the build each process of the node was started with, narrowed to a module when one is given. ProcessBuilds func(module string) (any, error) // ComposeCandidate composes a check's candidate (the indexer's candidate key) on this node in shadow and answers per @@ -236,6 +247,21 @@ func (h *HandActs) Call(c EngineCall) (any, error) { return nil, errors.New("this engine reads no seat list yet: its node agent has not started") } return h.SeatList() + case ActFirstViews: + if h.FirstViews == nil { + return nil, errors.New("this engine keeps no first views yet: its node agent has not started") + } + return h.FirstViews() + case ActFirstViewAgain: + if arg("owner") == "" || arg("state") == "" || arg("why") == "" { + return nil, errors.New("first-view-again names the owner, the state and why") + } + // The node-engine follows no branch and no view of another part's state: the views it holds are its entries (entry-again) + // and its seat list (seat-list-again). + return nil, fmt.Errorf("this engine follows no first view of %s's %s: its entries are asked again by entry-again, and its seat list by seat-list-again", + arg("owner"), arg("state")) + case ActEntryAgain: + return h.AskEntryAgain(c.Caller, arg("fingerprint"), arg("why")) case ActProcessBuilds: if h.ProcessBuilds == nil { return nil, errors.New("this engine reads no process yet: its node agent has not started") @@ -284,6 +310,39 @@ func (h *HandActs) Call(c EngineCall) (any, error) { return nil, fmt.Errorf("the node seat %s serves no verb %q", EngineSeat, c.Verb) } +// AskEntryAgain asks fingerprint's entry of the catalog half again by hand: a hand act recorded with who asked and why, then +// done. The answer is the ask and the verb that reads where it stands (design 47 G §698). +func (h *HandActs) AskEntryAgain(caller, fingerprint, why string) (any, error) { + if h.EntryAgain == nil { + return nil, errors.New("this engine reads no entry by fingerprint yet: its node agent has not started") + } + if fingerprint == "" || why == "" { + return nil, errors.New("entry-again names the build's fingerprint and why") + } + ctx, cancel := context.WithTimeout(context.Background(), VaultWithin) + defer cancel() + a := HandAct{ID: h.id(ActEntryAgain), Act: ActEntryAgain, Subject: fingerprint, Caller: caller, Why: why, At: h.now().UTC(), State: HandDone} + err := h.EntryAgain(ctx, fingerprint) + if err != nil { + a.State, a.Said = HandRefused, err.Error() + } + // A done ask is kept as an act; a refused one, which any caller may cause, rolls with the judgements. + record := h.record + if err != nil { + record = h.roll + } + if rerr := record(a); rerr != nil { + return nil, rerr + } + if err != nil { + return nil, err + } + return map[string]any{ + "asked": map[string]any{"v": sdkkeys.Version, "fingerprint": fingerprint, "why": why, "by": caller, "at": a.At}, + "read": ActFirstViews, + }, nil +} + // ---- the record -------------------------------------------------------------------------------------------------- // ActStore is the engine's own state, this node's keys alone (BusState prefixes them with the node): each act one key, diff --git a/internal/link/handover_test.go b/internal/link/handover_test.go index 3fb5491d..563baf0c 100644 --- a/internal/link/handover_test.go +++ b/internal/link/handover_test.go @@ -271,6 +271,16 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) { ManifestReads: func() ([]string, time.Time) { return []string{"form:a", "seat:b"}, time.Date(2026, 11, 4, 10, 0, 0, 0, time.UTC) }, + // Design 47 G §422, §698: the first views and the entries asked again. + FirstViews: func() (any, error) { + return map[string]any{"first-views": []any{}, "entries": []any{}, "wants": []any{}}, nil + }, + EntryAgain: func(_ context.Context, fingerprint string) error { + if fingerprint == "src2:gone" { + return errors.New("no entry of src2:gone is kept") + } + return nil + }, // Design 47: the build each process was started with, narrowed by module. ProcessBuilds: func(module string) (any, error) { return map[string]any{"processes": []any{}, "module": module}, nil @@ -315,6 +325,11 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) { {"seat-list-again", "mesh.seat.node-engine.call.seat-list-again.n1.ace~mesh-healers", `"read":true`}, {"seat-list-again", "mesh.seat.node-engine.call.seat-list-again.n1.ace~messenger", `"read":true`}, + {"first-views", "mesh.seat.node-engine.call.first-views.n1.person~jochen", `"entries":[],"first-views":[],"wants":[]`}, + {"first-view-again", "mesh.seat.node-engine.call.first-view-again.n1.person~jochen " + `{"owner":"mesh-catalog","state":"modules","why":"doubt"}`, `follows no first view of mesh-catalog's modules`}, + {"entry-again", "mesh.seat.node-engine.call.entry-again.n1.person~jochen " + `{"fingerprint":"src2:a","why":"doubt"}`, `"fingerprint":"src2:a","v":1,"why":"doubt"},"read":"first-views"`}, + {"entry-again", "mesh.seat.node-engine.call.entry-again.n1.person~jochen " + `{"fingerprint":"src2:gone","why":"doubt"}`, `no entry of src2:gone is kept`}, + {"entry-again", "mesh.seat.node-engine.call.entry-again.n1.person~jochen " + `{"fingerprint":"src2:a"}`, `entry-again names the build's fingerprint and why`}, {"process-builds", "mesh.seat.node-engine.call.process-builds.n1.person~jochen " + `{"module":"web"}`, `"module":"web","processes":[]`}, {"process-builds", "mesh.seat.node-engine.call.process-builds.n1.ace~messenger", `"processes":[]`}, {"unserved", "mesh.seat.node-engine.call.unserved.n1.x", `"error":"the node seat node-engine serves no verb \"unserved\""`}, @@ -329,8 +344,8 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) { } } acts, _ := h.Acts(0) - if got := acts.(map[string]any)["acts"].([]HandAct); len(got) != 3 || got[0].Caller != "unproven" || - got[2].Caller != "shanks.node-tools" || got[2].Act != ActJudgeDirectories { + if got := acts.(map[string]any)["acts"].([]HandAct); len(got) != 5 || got[2].Caller != "unproven" || + got[4].Caller != "shanks.node-tools" || got[4].Act != ActJudgeDirectories || got[0].Act != ActEntryAgain || got[0].State != HandRefused { t.Errorf("the judgements are recorded as %+v", got) } }) diff --git a/module.json b/module.json index 2fef2be6..f6f38d1f 100644 --- a/module.json +++ b/module.json @@ -17,7 +17,26 @@ "claims": [ { "name": "node-engine", - "scope": "node" + "scope": "node", + "serves": [ + "hand-over", + "access-act", + "judge-directories", + "setuid-search", + "root-free", + "acts", + "packages", + "composition", + "compose", + "compose-candidate", + "manifest-reads", + "seat-list", + "seat-list-again", + "first-views", + "first-view-again", + "entry-again", + "process-builds" + ] } ], "invokes": [ -- 2.54.0 From 5173f2a31cec54d90f8b548def17f4fc87b8eb0b Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:39:36 +0100 Subject: [PATCH 17/59] W13.8: the node-engine fetches a kept build through the build node's kept-artifact, the node taken from the want's kept-on MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A want that names a node in kept-on, for an artifact its parts name, is fetched through kept-artifact of that node's build seat and read as an object on the bus, checked against the digest; with no kept-on it is fetched from the store, and the builder's builds are read no more (the apply hook, the fetch and the object read from bc8fbc7, its node and fingerprint from the want). Builds: design 47 G §271, §526, §421 (A-50) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- cmd/mesh-host/keptartifact.go | 71 +++++++++++++++++++++++++++ cmd/mesh-host/keptartifact_test.go | 73 ++++++++++++++++++++++++++++ cmd/mesh-host/nodeagent.go | 6 +++ go.mod | 3 ++ internal/apply/fetch_first.go | 17 +++++-- internal/apply/kept_artifact.go | 29 +++++++++++ internal/apply/replace_image_test.go | 41 ++++++++++++++++ internal/link/files.go | 43 ++++++++++++++++ internal/link/session.go | 8 ++- internal/selfcompose/kept.go | 31 ++++++++++++ internal/selfcompose/kept_test.go | 35 +++++++++++++ 11 files changed, 353 insertions(+), 4 deletions(-) create mode 100644 cmd/mesh-host/keptartifact.go create mode 100644 cmd/mesh-host/keptartifact_test.go create mode 100644 internal/apply/kept_artifact.go create mode 100644 internal/selfcompose/kept.go create mode 100644 internal/selfcompose/kept_test.go diff --git a/cmd/mesh-host/keptartifact.go b/cmd/mesh-host/keptartifact.go new file mode 100644 index 00000000..0e7dcdb0 --- /dev/null +++ b/cmd/mesh-host/keptartifact.go @@ -0,0 +1,71 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildimage" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildnode" + "github.com/novox/mesh-host/internal/apply" +) + +// The artifact store's holder takes its own new build from the build node that kept it (design 47): its done build names the +// node and the digest, the node's build seat answers where the bytes are as an object on the bus, and the node-engine +// reads them and checks them against the digest. Never another module's files: the seat's verb is the way. + +// keptAskWithin bounds one ask of a build seat; keptObjectWithin one read of the object. +var ( + keptAskWithin = 30 * time.Second + keptObjectWithin = 10 * time.Minute +) + +// keptBuildSeats are the three build seats, each of which serves the verb kept-artifact: the node that kept the build +// holds the seat of the language it was built in, and the others do not answer. +var keptBuildSeats = []string{nodebuildgo.Seat, nodebuildnode.Seat, nodebuildimage.Seat} + +// keptArtifactFetch is the apply's fetch of what a module's done build kept on a build node. +func keptArtifactFetch(buildOf func(context.Context, string, string) (string, string, bool, error), + ask func(ctx context.Context, node, seat, verb string, args any, within time.Duration) (json.RawMessage, error), + object func(ctx context.Context, store, name string, limit int64) ([]byte, error)) apply.KeptFetch { + return func(ctx context.Context, module, digest string) ([]byte, bool, error) { + node, fp, kept, err := buildOf(ctx, module, digest) + if err != nil || !kept { + return nil, false, err + } + hex := strings.TrimPrefix(digest, "sha256:") + var notHeld []string + for _, seat := range keptBuildSeats { + raw, err := ask(ctx, node, seat, nodebuildgo.VerbKeptArtifact, + nodebuildgo.KeptArtifactInput{Module: module, Fp: fp, Digest: hex}, keptAskWithin) + if err != nil { + notHeld = append(notHeld, seat+": "+err.Error()) + continue + } + var got nodebuildgo.KeptArtifactAnswer + if json.Unmarshal(raw, &got) != nil { + return nil, true, fmt.Errorf("%s of %s answered with something that is not an answer of kept-artifact", seat, node) + } + switch { + case got.Error != "": + return nil, true, fmt.Errorf("%s of %s could not give the kept build of %s: %s", seat, node, module, got.Error) + case got.Refused != "": + return nil, true, fmt.Errorf("%s of %s refuses the kept build of %s: %s %s", seat, node, module, got.Refused, got.Words) + case got.Store == "" || got.Object == "" || got.Digest != hex: + return nil, true, fmt.Errorf("%s of %s answered for %s without the store and the object of the digest %s", seat, node, module, hex) + } + octx, cancel := context.WithTimeout(ctx, keptObjectWithin) + body, err := object(octx, got.Store, got.Object, got.Size+1) + cancel() + if err != nil { + return nil, true, fmt.Errorf("the build of %s kept on %s could not be read from %s: %w", module, node, got.Store, err) + } + return body, true, nil + } + return nil, true, fmt.Errorf("%s keeps the build of %s for the artifact store and none of its build seats answers (%s)", + node, module, strings.Join(notHeld, "; ")) + } +} diff --git a/cmd/mesh-host/keptartifact_test.go b/cmd/mesh-host/keptartifact_test.go new file mode 100644 index 00000000..b662f01b --- /dev/null +++ b/cmd/mesh-host/keptartifact_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" +) + +// A module's done build that kept its artifact on a build node is fetched through that node's build seat (design 47). +func TestAKeptArtifactIsFetchedThroughTheBuildSeatOfItsNode(t *testing.T) { + hex := strings.Repeat("a", 64) + stored := func(string) (json.RawMessage, error) { + return json.RawMessage(`{"module":"distribution","fp":"f1","digest":"` + hex + `","size":5,"store":"kept-builds","object":"` + hex + `"}`), nil + } + for _, row := range []struct { + name string + kept bool + answer func(seat string) (json.RawMessage, error) + object error + wantKept bool + wantErr string + wantSeat string + }{ + {"a build kept nothing is not asked", false, nil, nil, false, "", ""}, + {"the seat of the language answers", true, func(seat string) (json.RawMessage, error) { + if seat != "node-build-node" { + return nil, errors.New("no responder") + } + return stored(seat) + }, nil, true, "", "node-build-node"}, + {"no build seat of the node answers", true, func(string) (json.RawMessage, error) { return nil, errors.New("no responder") }, + nil, true, "novox keeps the build of distribution", ""}, + {"the seat refuses it", true, func(string) (json.RawMessage, error) { + return json.RawMessage(`{"module":"distribution","fp":"f1","refused":"not kept","words":"swept"}`), nil + }, nil, true, "refuses the kept build of distribution: not kept", ""}, + {"the object cannot be read", true, stored, errors.New("no object"), true, "could not be read from kept-builds", "node-build-go"}, + } { + t.Run(row.name, func(t *testing.T) { + var seats []string + fetch := keptArtifactFetch( + func(context.Context, string, string) (string, string, bool, error) { return "novox", "f1", row.kept, nil }, + func(_ context.Context, node, seat, verb string, _ any, _ time.Duration) (json.RawMessage, error) { + if node != "novox" || verb != "kept-artifact" { + t.Errorf("asked %s of %s on %s", verb, seat, node) + } + seats = append(seats, seat) + return row.answer(seat) + }, + func(_ context.Context, store, name string, limit int64) ([]byte, error) { + if store != "kept-builds" || name != hex || limit < 5 { + t.Errorf("read %s/%s up to %d", store, name, limit) + } + return []byte("bytes"), row.object + }) + body, kept, err := fetch(context.Background(), "distribution", "sha256:"+hex) + if kept != row.wantKept { + t.Fatalf("kept = %v", kept) + } + if row.wantErr == "" && (err != nil || (kept && string(body) != "bytes")) { + t.Fatalf("got %q, %v", body, err) + } + if row.wantErr != "" && (err == nil || !strings.Contains(err.Error(), row.wantErr)) { + t.Fatalf("the error is %v, want %q", err, row.wantErr) + } + if !row.kept && len(seats) > 0 { + t.Errorf("a build that kept nothing asked %v", seats) + } + }) + } +} diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 2b5b8b9b..45d0ba7d 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -448,6 +448,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus } began := time.Now() node := session.Membership.Node + var keptFetch atomic.Pointer[apply.KeptFetch] // the composer's, set once it exists composedBucket, err := compositions.BucketFor(node) if err != nil { cancel() @@ -550,6 +551,9 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus Backoff: link.Backoff, Apply: func(ctx context.Context, c compositions.Composed, others []compositions.Composed) agent.Result { actx, done := applyContext(session.WithLink(ctx)) + if kf := keptFetch.Load(); kf != nil { + actx = apply.WithKept(actx, *kf) + } // While the grants part's list is followed, the controller's list in the bus module's composition is not // written: the apply keeps the list on the machine (ADR 0547 clause 8, never both). held, herr := users.Hold(c) @@ -681,6 +685,8 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus apply.UseEngineModule(composer.EngineModule) // the guard of the engine's own trees fixes no name (§527) env.Composed, env.SelfComposed, env.Policy, env.ComposedToday = composer, true, composer.Policy, composer.ComposedToday env.WhyUnmade, env.Waiting = composer.WhyUnmade, composer.Waiting + keptFetchFn := keptArtifactFetch(composer.KeptBuildOf, session.AskSeatOn, session.KeptObject) + keptFetch.Store(&keptFetchFn) env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does session.NoteComposes(linked, nodeengineclient.ComposesOwn, "it composes its own modules, and applies no composition made elsewhere") ag = agent.New(env) diff --git a/go.mod b/go.mod index 04fc37be..a5028435 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,9 @@ require ( git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildimage v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildnode v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate v0.0.0-00010101000000-000000000000 diff --git a/internal/apply/fetch_first.go b/internal/apply/fetch_first.go index ad91ec02..e4645f80 100644 --- a/internal/apply/fetch_first.go +++ b/internal/apply/fetch_first.go @@ -60,10 +60,21 @@ func fetchFirst(ctx context.Context, ordered []declaration.Resource, d *declarat continue } } - got, seen := bySource[source] - if !seen { + // What the module's done build kept on a build node is fetched from there, not from the store (design 47). + cacheKey, kept := source, false + if kf := keptFetchFrom(ctx); kf != nil && ours { + cacheKey = "kept " + module + " " + digest + if _, seen := bySource[cacheKey]; seen { + kept = true + } else if body, isKept, err := kf(ctx, module, digest); isKept { + kept = true + bySource[cacheKey] = fetchedArtifact{body: body, err: err} + } + } + got, seen := bySource[cacheKey] + if !seen && !kept { got.body, got.err = fetchMinding(ctx, source, in.away) - bySource[source] = got + bySource[cacheKey] = got } if got.err == nil { sum := sha256.Sum256(got.body) diff --git a/internal/apply/kept_artifact.go b/internal/apply/kept_artifact.go new file mode 100644 index 00000000..3585470a --- /dev/null +++ b/internal/apply/kept_artifact.go @@ -0,0 +1,29 @@ +package apply + +import "context" + +// The build the artifact store's holder keeps on a build node (novox/hq design 47, *The artifact store's holder takes its +// own new build even when its seat is the one broken*): the holder's program cannot be fetched from the store it serves, so +// its done build names the node that keeps it and the node-engine fetches it from there, through the build seat's verb. +// How is the caller's (a hook on the apply's context, as the files' link is); what apply does is use the bytes in place of +// the source's fetch, check them against the declared digest as every fetch is, and gate the module when they cannot be had. + +// KeptFetch gives the bytes of the artifact a module's done build kept on a build node, by the artifact's digest +// ("sha256:" and 64 hex digits, as a declaration names it). kept is false when the module's build kept no such artifact: +// the artifact is then fetched from its source, as before. An error says the artifact is kept and could not be had. +type KeptFetch func(ctx context.Context, module, digest string) (body []byte, kept bool, err error) + +type keptFetchKey struct{} + +// WithKept is ctx carrying how a kept artifact is fetched. +func WithKept(ctx context.Context, f KeptFetch) context.Context { + if f == nil { + return ctx + } + return context.WithValue(ctx, keptFetchKey{}, f) +} + +func keptFetchFrom(ctx context.Context) KeptFetch { + f, _ := ctx.Value(keptFetchKey{}).(KeptFetch) + return f +} diff --git a/internal/apply/replace_image_test.go b/internal/apply/replace_image_test.go index fa48c838..a7fd05d5 100644 --- a/internal/apply/replace_image_test.go +++ b/internal/apply/replace_image_test.go @@ -164,6 +164,47 @@ func TestARunningContainerIsKeptWhenItsReplacementImageCannotBeObtained(t *testi t.Errorf("the container's outcome is %+v, want skipped", o) } }) + + // Design 47 (W13.8): the holder's own program is fetched from the build node that kept it, not from the store; when it + // cannot be had, or is not the declared digest, nothing of the module is replaced and the failure says so. + for _, row := range []struct { + name string + fetch KeptFetch + wantErr string + }{ + {"the node that keeps it is away", func(context.Context, string, string) ([]byte, bool, error) { + return nil, true, errors.New("novox keeps it and does not answer") + }, "novox keeps it and does not answer"}, + {"what the node kept is not the declared digest", func(context.Context, string, string) ([]byte, bool, error) { + return []byte("other"), true, nil + }, "what arrived is"}, + } { + t.Run("a kept artifact: "+row.name, func(t *testing.T) { + gone := httptest.NewServer(http.NotFoundHandler()) + source := gone.URL + "/program.tar.gz" + gone.Close() + was := storeAwayAtMost + storeAwayAtMost = 0 + t.Cleanup(func() { storeAwayAtMost = was }) + d := parseTrusted(t, `{"declaration":1,"resources":[ + {"id":"distribution.program","type":"process","name":"program","source":"`+source+`", + "digest":"sha256:`+strings.Repeat("0", 64)+`","run":["program"]} + ]}`) + var asked []string + fetch := func(ctx context.Context, module, digest string) ([]byte, bool, error) { + asked = append(asked, module+" "+digest) + return row.fetch(ctx, module, digest) + } + run := func(ctx context.Context, name string, args ...string) (string, error) { return "", nil } + _, _, err := Apply(WithKept(context.Background(), fetch), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) + if err == nil || !strings.Contains(err.Error(), row.wantErr) { + t.Errorf("the failure is %v, want it to say %q", err, row.wantErr) + } + if len(asked) != 1 || asked[0] != "distribution sha256:"+strings.Repeat("0", 64) { + t.Errorf("the kept fetch was asked %q", asked) + } + }) + } } func TestAReplacementImageIsPresentBeforeTheRunningContainerIsRemoved(t *testing.T) { diff --git a/internal/link/files.go b/internal/link/files.go index 44d02ddf..23f24eaa 100644 --- a/internal/link/files.go +++ b/internal/link/files.go @@ -172,6 +172,49 @@ func (l *natsLink) GrantsObject(ctx context.Context, name string, limit int64) ( return OverNATS{Conn: l.conn, JS: l.js, Node: l.node}.GrantsObject(ctx, name, limit) } +// KeptObject reads one object of a build module's store of kept builds by its name (design 47: the build the artifact store's +// holder keeps on a build node; the build seat's verb kept-artifact names the store and the object), with the same +// subjects as FileObject, verified against its digest. +func (b OverNATS) KeptObject(ctx context.Context, store, name string, limit int64) ([]byte, error) { + js := b.JS + if b.Conn != nil { + wait := 10 * time.Second + if at, ok := ctx.Deadline(); ok { + if wait = time.Until(at); wait <= 0 { + return nil, context.DeadlineExceeded + } + } + bounded, err := b.Conn.JetStream(nats.MaxWait(wait)) + if err != nil { + return nil, err + } + js = bounded + } + return objectByName(ctx, js, store, declaration.FileStreamPrefix+store, name, limit) +} + +func (l *natsLink) KeptObject(ctx context.Context, store, name string, limit int64) ([]byte, error) { + return OverNATS{Conn: l.conn, JS: l.js, Node: l.node}.KeptObject(ctx, store, name, limit) +} + +// KeptObjects is a link that can read an object of a build module's store of kept builds. +type KeptObjects interface { + KeptObject(ctx context.Context, store, name string, limit int64) ([]byte, error) +} + +// KeptObject reads an object of a build module's store of kept builds on the link open now. +func (q *Session) KeptObject(ctx context.Context, store, name string, limit int64) ([]byte, error) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + k, ok := bus.(KeptObjects) + if bus == nil || !ok { + return nil, ErrNoLink + } + return k.KeptObject(ctx, store, name, limit) +} + // GrantsObjects is a link that can read an object of the grants part's bucket of sealed user lists. type GrantsObjects interface { GrantsObject(ctx context.Context, name string, limit int64) ([]byte, error) diff --git a/internal/link/session.go b/internal/link/session.go index 2d74daf2..59bfb062 100644 --- a/internal/link/session.go +++ b/internal/link/session.go @@ -348,6 +348,12 @@ func SeatToolSubject(seat, verb, node string) string { // AskSeat asks the holder of a seat on this node a verb, over the link open now, and answers its result, or its refusal // as an error. The one call the node agent makes of another part: the backup before a data module moves (ADR 0429 §14). func (q *Session) AskSeat(ctx context.Context, seat, verb string, args any, within time.Duration) (json.RawMessage, error) { + return q.AskSeatOn(ctx, q.Membership.Node, seat, verb, args, within) +} + +// AskSeatOn asks the holder of a node seat on another node a verb: the build seat of the node a want's `kept-on` names +// gives the kept build (design 47 G section 526). +func (q *Session) AskSeatOn(ctx context.Context, node, seat, verb string, args any, within time.Duration) (json.RawMessage, error) { q.init() q.mu.Lock() bus := q.bus @@ -365,7 +371,7 @@ func (q *Session) AskSeat(ctx context.Context, seat, verb string, args any, with } asking, cancel := context.WithTimeout(ctx, within) defer cancel() - reply, err := asker.Ask(asking, SeatToolSubject(seat, verb, q.Membership.Node), body) + reply, err := asker.Ask(asking, SeatToolSubject(seat, verb, node), body) if err != nil { return nil, fmt.Errorf("%s.%s could not be asked: %w", seat, verb, err) } diff --git a/internal/selfcompose/kept.go b/internal/selfcompose/kept.go new file mode 100644 index 00000000..52abee8c --- /dev/null +++ b/internal/selfcompose/kept.go @@ -0,0 +1,31 @@ +package selfcompose + +import ( + "context" + "strings" +) + +// KeptBuildOf is the node that keeps the build a module is wanted at, and its fingerprint, for the artifact store's +// holder (design 47 G section 271, 526, A-50): the want's `kept-on` and nothing else names it, the builder's builds being +// no read of the node-engine's (section 421). kept is false when the module has no want, its want names no keeping node, +// or its `parts` do not name the digest ("sha256:" and 64 hex digits, or bare): the artifact is then fetched from the store. +func (c *Composer) KeptBuildOf(_ context.Context, module, digest string) (node, fp string, kept bool, err error) { + c.mu.Lock() + w, ok := c.wants[module] + c.mu.Unlock() + if !ok || w.Fp == "" || w.KeptOn == "" { + return "", "", false, nil + } + if len(w.Parts) > 0 { + named := false + for _, d := range w.Parts { + if strings.TrimPrefix(d, "sha256:") == strings.TrimPrefix(digest, "sha256:") { + named = true + } + } + if !named { + return "", "", false, nil + } + } + return w.KeptOn, w.Fp, true, nil +} diff --git a/internal/selfcompose/kept_test.go b/internal/selfcompose/kept_test.go new file mode 100644 index 00000000..c2b7f305 --- /dev/null +++ b/internal/selfcompose/kept_test.go @@ -0,0 +1,35 @@ +package selfcompose + +import ( + "context" + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" +) + +// The node that keeps a kept build is the want's kept-on alone, for an artifact its parts name (design 47 G 526, A-50). +func TestTheKeepingNodeIsTheWantsKeptOn(t *testing.T) { + kd := strings.Repeat("6", 64) + for _, row := range []struct { + name, digest string + want *meshwants.WantsByWantNodeModuleValue + node string + kept bool + }{ + {"kept, by its bare digest", kd, &meshwants.WantsByWantNodeModuleValue{Fp: "f1", KeptOn: "n3", Parts: map[string]string{"db:p": "sha256:" + kd}}, "n3", true}, + {"kept, by the declared digest", "sha256:" + kd, &meshwants.WantsByWantNodeModuleValue{Fp: "f1", KeptOn: "n3", Parts: map[string]string{"db:p": "sha256:" + kd}}, "n3", true}, + {"a digest the parts do not name", "sha256:" + strings.Repeat("7", 64), &meshwants.WantsByWantNodeModuleValue{Fp: "f1", KeptOn: "n3", Parts: map[string]string{"db:p": "sha256:" + kd}}, "", false}, + {"a want naming no keeping node", kd, &meshwants.WantsByWantNodeModuleValue{Fp: "f1", Parts: map[string]string{"db:p": "sha256:" + kd}}, "", false}, + {"no want of the module", kd, nil, "", false}, + } { + c := &Composer{wants: map[string]meshwants.WantsByWantNodeModuleValue{}} + if row.want != nil { + c.wants["db"] = *row.want + } + node, fp, kept, err := c.KeptBuildOf(context.Background(), "db", row.digest) + if err != nil || node != row.node || kept != row.kept || (kept && fp != "f1") { + t.Errorf("%s: node %q fp %q kept %v err %v", row.name, node, fp, kept, err) + } + } +} -- 2.54.0 From 7c7b5e2d6829001acd7f71e750844ab7978477e3 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:48:16 +0100 Subject: [PATCH 18/59] G A-92: the node-engine's manifest declares the state bounds, a lifetime per key, in its old section MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §560, §557, §627, A-77, A-92 (the seat definition: mesh-contracts 2ae0c679304b828a3973dd23ecd19861577d141d) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- module.json | 5 +++++ packaging/bounds_state_test.go | 30 ++++++++++++++++++++++++++++++ packaging/operator_keys_test.go | 5 +++-- 3 files changed, 38 insertions(+), 2 deletions(-) create mode 100644 packaging/bounds_state_test.go diff --git a/module.json b/module.json index 52f97f17..82b00dec 100644 --- a/module.json +++ b/module.json @@ -51,6 +51,11 @@ }, { "name": "packages" + }, + { + "name": "bounds", + "history": 1, + "key-ttl": true } ], "config": { diff --git a/packaging/bounds_state_test.go b/packaging/bounds_state_test.go new file mode 100644 index 00000000..d92beab9 --- /dev/null +++ b/packaging/bounds_state_test.go @@ -0,0 +1,30 @@ +package packaging_test + +import ( + "os" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// The node-engine's waits run against bound keys with their own time to live, in its old section as its other states are +// (design 47 G A-92, §560, §627): the manifest declares the state bounds with a lifetime per key. +func TestTheManifestDeclaresTheBoundsStateWithAKeyLifetime(t *testing.T) { + raw, err := os.ReadFile("../module.json") + if err != nil { + t.Fatal(err) + } + m, err := catalogue.Decode(raw) + if err != nil { + t.Fatalf("the manifest no longer parses: %v", err) + } + for _, s := range m.State { + if s.Name == "bounds" { + if !s.KeyTTL || s.History != 1 { + t.Fatalf("the state bounds is %+v: one value a key, each key with its own lifetime", s) + } + return + } + } + t.Fatal("the manifest declares no state bounds") +} diff --git a/packaging/operator_keys_test.go b/packaging/operator_keys_test.go index d620eef7..2c6ca2ca 100644 --- a/packaging/operator_keys_test.go +++ b/packaging/operator_keys_test.go @@ -91,10 +91,11 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) { // ADR 0565 clause 6: and the state packages, one key per node, its compositions' declared packages, read by pacman. // ADR 0564 §2, ADR 0636 §1: and the state findings, the node's finding that it waits for the seat list (no TTL: the // finding stays until the first full read deletes its key). - if len(parsed.State) != 4 || parsed.State[0].Name != "health" || parsed.State[0].TTLSeconds != 0 || + // Design 47 G A-92: and the state bounds, each key a wait's bound with its own lifetime (packaging/bounds_state_test.go). + if len(parsed.State) != 5 || parsed.State[0].Name != "health" || parsed.State[0].TTLSeconds != 0 || parsed.State[1].Name != "findings" || parsed.State[1].TTLSeconds != 0 || parsed.State[2].Name != "data" || parsed.State[2].TTLSeconds != 0 || parsed.State[3].Name != "packages" || - parsed.State[3].TTLSeconds != 0 { + parsed.State[3].TTLSeconds != 0 || parsed.State[4].Name != "bounds" { t.Fatalf("the manifest declares the state %+v", parsed.State) } } -- 2.54.0 From 2b916ac04366631b4a80f904f2832a42dad35644 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:51:09 +0100 Subject: [PATCH 19/59] G A-98: the node agent gains Env.Stop, which stops a module's units and containers and removes nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §297, A-98 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- cmd/mesh-host/nodeagent.go | 28 +++++++++++++++ cmd/mesh-host/stop_test.go | 40 +++++++++++++++++++++ internal/agent/agent.go | 4 +++ internal/apply/stop.go | 69 +++++++++++++++++++++++++++++++++++++ internal/apply/stop_test.go | 45 ++++++++++++++++++++++++ 5 files changed, 186 insertions(+) create mode 100644 cmd/mesh-host/stop_test.go create mode 100644 internal/apply/stop.go create mode 100644 internal/apply/stop_test.go diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 45d0ba7d..098c8753 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -28,6 +28,7 @@ import ( "github.com/novox/mesh-host/internal/seatwait" "github.com/novox/mesh-host/internal/selfcompose" "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" ) // The node agent's wiring (novox/hq ADR 0429 §4, ADR 0539 §9): the buckets it watches, the haves it writes, the backup @@ -578,6 +579,9 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus sayFacts(ctx, ag) return res }, + Stop: func(ctx context.Context, module string) agent.Result { + return stopComposition(ctx, opts, module) + }, Remove: func(ctx context.Context, module string, others []compositions.Composed) agent.Result { actx, done := applyContext(session.WithLink(ctx)) res := removeComposition(actx, opts, module, others, ag.Placed(module), sched, say) @@ -857,6 +861,30 @@ func removeComposition(ctx context.Context, opts options, module string, others return agent.Result{} } +// stopComposition stops a module's units and containers from the node's record and removes nothing (design 47 G A-98, +// §297): what the module kept stays, so a revert restores it before the reverted build starts. +func stopComposition(ctx context.Context, opts options, module string) agent.Result { + applying.Lock() + defer applying.Unlock() + unlock, err := store.Lock(opts.state, nil) + if err != nil { + return agent.Result{Failed: true, Detail: shorten(err.Error())} + } + defer unlock() + known, err := store.Load(opts.state) + if err != nil { + return agent.Result{Failed: true, Detail: shorten(err.Error())} + } + built, err := system.For(builtFor) + if err != nil { + return agent.Result{Failed: true, Detail: shorten(err.Error())} + } + if _, err := apply.StopModule(ctx, built, known, module, opts.run()); err != nil { + return agent.Result{Failed: true, Detail: shorten(err.Error())} + } + return agent.Result{} +} + // busyModules are the modules whose last apply left a container running because it said busy. func busyModules() []string { busyMu.Lock() diff --git a/cmd/mesh-host/stop_test.go b/cmd/mesh-host/stop_test.go new file mode 100644 index 00000000..4ca264a6 --- /dev/null +++ b/cmd/mesh-host/stop_test.go @@ -0,0 +1,40 @@ +package main + +import ( + "context" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/store" +) + +// The node agent's Stop stops a module's container from the node's record, asks the machine nothing else and leaves the +// record, so the module's kept data and its record survive (design 47 G A-98). +func TestTheNodeAgentsStopStopsAModuleAndRemovesNothing(t *testing.T) { + was := builtFor + builtFor = "arch" + t.Cleanup(func() { builtFor = was }) + path := filepath.Join(t.TempDir(), "state.json") + var s store.State + s.Record(store.Applied{ID: "app.box", Type: "container", Target: "app-box"}) + s.Record(store.Applied{ID: "other.box", Type: "container", Target: "other-box"}) + if err := store.Save(path, s); err != nil { + t.Fatal(err) + } + var asked []string + opts := options{state: path, runner: func(_ context.Context, name string, args ...string) (string, error) { + asked = append(asked, name+" "+strings.Join(args, " ")) + return "", nil + }} + if res := stopComposition(context.Background(), opts, "app"); res.Failed { + t.Fatalf("stop failed: %s", res.Detail) + } + if !strings.Contains(strings.Join(asked, "\n"), "stop app-box") || strings.Contains(strings.Join(asked, "\n"), "other-box") || + strings.Contains(strings.Join(asked, "\n"), " rm ") { + t.Fatalf("the machine was asked %v", asked) + } + if after, err := store.Load(path); err != nil || len(after.Resources) != 2 { + t.Fatalf("the record changed: %v, %v", after.Resources, err) + } +} diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 9aa00adc..2b40d698 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -136,6 +136,10 @@ type Env struct { Apply func(ctx context.Context, c compositions.Composed, others []compositions.Composed) Result // Remove removes the module's own resources and nothing of another's. Remove func(ctx context.Context, module string, others []compositions.Composed) Result + // Stop stops the module's units and containers and removes nothing, so what it kept survives: a revert stops the newer + // build with it, restores the module's kept data, and then applies the reverted build (design 47 G A-98, §297). Nil + // stops nothing. + Stop func(ctx context.Context, module string) Result // Backup backs a module up before a new composition of it is applied; nil backs up nothing. Backup func(ctx context.Context, module string) Backup // Kept is the resources the node last applied, by module, from the declaration it kept before the switch (ADR 0539 diff --git a/internal/apply/stop.go b/internal/apply/stop.go new file mode 100644 index 00000000..8a8b553e --- /dev/null +++ b/internal/apply/stop.go @@ -0,0 +1,69 @@ +package apply + +import ( + "context" + "fmt" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/declaration" + "github.com/novox/mesh-host/internal/store" + "github.com/novox/mesh-host/internal/system" +) + +// StopModule stops the units and containers the node's record holds for a module and removes nothing: no unit, file, +// container, data or record, so what the module kept survives and a later Apply starts the build it is given (design 47 +// G A-98, §297). Units come first, a timer before the service it starts, then containers; a unit the mesh never held +// (Stateless) is left as it is. It answers what it stopped, and the first failure. +func StopModule(ctx context.Context, sys system.System, known store.State, module string, run Runner) ([]string, error) { + var timers, services, containers []store.Applied + for _, r := range known.Resources { + if store.OwnerOf(r, declaration.ModuleGuess) != module { + continue + } + switch r.Type { + case string(declaration.TypeService): + if r.Stateless { + continue + } + if strings.HasSuffix(r.Target, ".timer") { + timers = append(timers, r) + } else { + services = append(services, r) + } + case string(declaration.TypeContainer): + containers = append(containers, r) + } + } + var stopped []string + for _, r := range append(timers, services...) { + manager := managerFor(r.Scope, r.User, run) + if state, err := sys.ServiceState(ctx, manager, r.Target); err != nil { + if strings.Contains(err.Error(), "does not exist on this machine") { + continue + } + return stopped, fmt.Errorf("stopping %s of %s: %w", r.Target, module, err) + } else if state == "stopped" { + continue + } + if err := sys.SetServiceState(ctx, manager, r.Target, "stopped"); err != nil { + return stopped, fmt.Errorf("stopping %s of %s: %w", r.Target, module, err) + } + stopped = append(stopped, r.Target) + } + if len(containers) == 0 { + return stopped, nil + } + cri, err := containerRuntime(ctx, run) + if err != nil { + return stopped, err + } + sort.Slice(containers, func(i, j int) bool { return containers[i].Target < containers[j].Target }) + for _, r := range containers { + if _, err := run(ctx, cri, "stop", r.Target); err != nil { + return stopped, fmt.Errorf("stopping the container %s of %s: %w", r.Target, module, err) + } + stopped = append(stopped, r.Target) + } + return stopped, nil +} diff --git a/internal/apply/stop_test.go b/internal/apply/stop_test.go new file mode 100644 index 00000000..2febfb5d --- /dev/null +++ b/internal/apply/stop_test.go @@ -0,0 +1,45 @@ +package apply + +import ( + "context" + "reflect" + "testing" + + "github.com/novox/mesh-host/internal/store" +) + +// Stop stops a module's timer, then its service, then its container, and leaves every record, and another module's +// units, as they are (design 47 G A-98). +func TestStopModuleStopsAModulesUnitsAndContainersAndRemovesNothing(t *testing.T) { + m := unitsMachine(map[string]*fakeUnit{ + "app.service": {active: "active", enabled: "enabled"}, + "app.timer": {active: "active", enabled: "enabled"}, + "other.service": {active: "active", enabled: "enabled"}, + }) + state := store.State{} + for _, r := range []store.Applied{ + {ID: "app.run", Type: "service", Target: "app.service"}, + {ID: "app.tick", Type: "service", Target: "app.timer"}, + {ID: "app.c", Type: "container", Target: "app-box"}, + {ID: "other.run", Type: "service", Target: "other.service"}, + {ID: "other.c", Type: "container", Target: "other-box"}, + } { + state.Record(r) + } + got, err := StopModule(context.Background(), archHost(t), state, "app", m.run) + if err != nil { + t.Fatal(err) + } + if want := []string{"app.timer", "app.service", "app-box"}; !reflect.DeepEqual(got, want) { + t.Fatalf("stopped %v, want %v, the timer first", got, want) + } + if m.units["app.service"].active != "inactive" || m.units["app.timer"].active != "inactive" || m.units["other.service"].active != "active" { + t.Fatalf("units are %+v", m.units) + } + if !m.did("docker stop app-box") || m.did("docker stop other-box") || m.did("docker rm") { + t.Fatalf("the machine was asked %v", m.asked) + } + if len(state.Resources) != 5 { + t.Fatalf("a record was removed: %d left", len(state.Resources)) + } +} -- 2.54.0 From cf378f9a2524bbf30c40ee8a2409a77d51687cfd Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 23:27:28 +0100 Subject: [PATCH 20/59] =?UTF-8?q?G=20=C2=A7421,=20=C2=A7683:=20the=20node-?= =?UTF-8?q?engine=20reads=20the=20entry=20its=20want=20names=20by=20finger?= =?UTF-8?q?print=20and=20keeps=20its=20entries=20and=20wants?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The node's own composer reads the catalog half through meshcatalogclient's Entries cache over its own calls of mesh-catalog's verb entry, and no longer watches or reads the indexer's modules, manifests and verdicts or the builder's builds: a module's manifest is its entry's, each artifact's reference is made from the digest its want's part names (artifact-store:////blobs/, A-50). A want whose entry is asked or not found is held, its have the phase held with a detail opening entry-asked: or entry-missing: naming the fingerprint. The entries and wants it composes from are kept in composed- (entry. and want.), written before the composition that uses them, read at start before anything is asked, a kept want standing until the wants' live read answers, and dropped when nothing names them. The shadow keeps its reads (G §628). The node grant gains the verb entry and the running stream's consumer calls; OverNATS hears the running stream as the link's session does. Builds: design 47 G §421, §683, §614, §628, A-47, A-50 --- cmd/mesh-host/nodeagent.go | 3 + cmd/mesh-host/nodeagent_integration_test.go | 89 +++- internal/agent/agent.go | 11 +- internal/agent/agent_test.go | 6 + internal/agent/parts.go | 13 + internal/link/running.go | 12 + internal/nodegrant/testdata/node.json | 6 + internal/selfcompose/composedstate.go | 5 + internal/selfcompose/entries.go | 516 ++++++++++++++++++++ internal/selfcompose/entries_test.go | 145 ++++++ internal/selfcompose/selfcompose.go | 55 +++ internal/selfcompose/selfcompose_test.go | 3 +- module.json | 1 + 13 files changed, 849 insertions(+), 16 deletions(-) create mode 100644 internal/selfcompose/entries.go create mode 100644 internal/selfcompose/entries_test.go diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 098c8753..9a10734f 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -1135,6 +1135,9 @@ func selfComposer(session *link.Session, node string, hooks composerHooks, say f env.ComposedState = link.EngineComposed{Session: session, Node: node} // And each composition it writes there said as the seat's event composed, which the assigner waits on. env.SayComposed = session.SayComposed + // And the catalog half read by the fingerprints its wants and runnings name, over its own calls of the seat mesh-catalog's + // verb entry on its link session (design 47 G §421): neither the indexer's nor the builder's states are read. + env.Entry = selfcompose.EntryCall(session.AskMeshSeat) return selfcompose.New(env) } diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index fc86409e..4d3f4021 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -79,6 +79,55 @@ type itBed struct { // holder re-serves (issue 1966). seatAway atomic.Bool findings nats.KeyValue + // entries are the entries the stand-in of the seat mesh-catalog answers its verb entry from, by fingerprint (design 47 G §421). + entries map[string]meshcatalog.CatalogEntry +} + +// registerEntry makes the stand-in answer the entry of fp: module's manifest and its artifacts (refs `:`). +func (b *itBed) registerEntry(module, fp, manifest string, artifacts ...string) { + b.t.Helper() + var m map[string]any + if err := json.Unmarshal([]byte(manifest), &m); err != nil { + b.t.Fatal(err) + } + e := meshcatalog.CatalogEntry{Module: module, Fp: fp, Manifest: m, ManifestDigest: meshcatalogclient.ManifestDigest([]byte(manifest)), + Repository: "novox/" + module, Commit: strings.Repeat("1", 40)} + for _, a := range artifacts { + e.Artifacts = append(e.Artifacts, meshcatalog.Artifact{Ref: module + ":" + a, Kind: "archive", Language: "go"}) + } + b.mu.Lock() + b.entries[fp] = e + b.mu.Unlock() +} + +// running says the seat node-engine's event running of module on node at fp, as that node's engine does. +func (b *itBed) running(node, module, fp, manifestDigest string) { + b.t.Helper() + raw, _ := json.Marshal(nodeengine.Running{V: 1, Node: node, Module: module, Fingerprint: fp, Manifest: manifestDigest, At: time.Now().UTC()}) + if _, err := b.js.Publish(nodeengine.SubjectRunning(node, module), raw); err != nil { + b.t.Fatal(err) + } +} + +// standInEntry answers the seat mesh-catalog's verb entry from the registered entries, and as the holder refuses an unknown one. +func (b *itBed) standInEntry() { + sub, err := b.conn.Subscribe("mesh.seat.mesh-catalog.call.entry.*", func(m *nats.Msg) { + var in meshcatalog.EntryInput + _ = json.Unmarshal(m.Data, &in) + b.mu.Lock() + e, ok := b.entries[in.Fingerprint] + b.mu.Unlock() + if !ok { + _ = m.Respond([]byte(`{"error":"no entry of fingerprint ` + in.Fingerprint + ` is kept"}`)) + return + } + out, _ := json.Marshal(map[string]any{"result": meshcatalog.EntryAnswer{Entry: e}}) + _ = m.Respond(out) + }) + if err != nil { + b.t.Fatal(err) + } + b.t.Cleanup(func() { _ = sub.Unsubscribe() }) } // aServer is an embedded bus with JetStream and two users: the test's, unrestricted, and the node's, granted exactly what @@ -131,7 +180,7 @@ func newItBedAs(t *testing.T, perms nodegrant.Permissions) *itBed { builtFor = "arch" t.Cleanup(func() { builtFor = was }) - b := &itBed{t: t, dir: t.TempDir()} + b := &itBed{t: t, dir: t.TempDir(), entries: map[string]meshcatalog.CatalogEntry{}} b.conn, b.nodeConn = aServer(t, perms, func(err error) { b.mu.Lock() b.refusals = append(b.refusals, err.Error()) @@ -167,7 +216,13 @@ func newItBedAs(t *testing.T, perms nodegrant.Permissions) *itBed { t.Fatal(err) } } + // The stream the seat node-engine's event running is kept in, the last of each subject (design 47 G §199). + if _, err = b.js.AddStream(&nats.StreamConfig{Name: nodeengine.StreamRunning, Subjects: []string{nodeengine.SubjectRunningAll}, + MaxMsgsPerSubject: 1}); err != nil { + t.Fatal(err) + } b.standInSeat() + b.standInEntry() b.opts = options{state: filepath.Join(b.dir, "state", "state.json"), timeout: 2 * time.Second} if err := os.MkdirAll(filepath.Dir(b.opts.state), 0o755); err != nil { t.Fatal(err) @@ -435,8 +490,7 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { path := filepath.Join(b.dir, "web.conf") man := `{"module":"web","version":"1","config":{"greeting":{"type":"string","description":"what the file says","default":"hello"}},` + `"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0644","content":"greeting=${config:greeting}\n"},` + - `{"id":"tools","type":"package","package":"web-tools"}],` + - `"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}` + `{"id":"tools","type":"package","package":"web-tools"}]}` digest := meshcatalogclient.ManifestDigest([]byte(man)) mraw, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "web", Manifest: man}) mkey, _ := meshcatalogclient.ManifestsKey(digest) @@ -464,9 +518,6 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { done, _ := builderbuilds.EncodeBuilderBuild(builderbuilds.BuilderBuild{State: builderbuilds.BuildDone, Manifest: digest, Artifacts: []builderbuilds.BuiltArtifact{{Name: "files", Kind: "archive", Reference: "artifact-store://web/files/blobs/" + fpDigest("f"), Digest: fpDigest("f")}}}) firstBuild := done - if checkBuild { - firstBuild = []byte(`{"v":1,"state":"no such state"}`) - } for _, w := range []struct { kv nats.KeyValue key string @@ -479,6 +530,13 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { t.Fatal(err) } } + // Design 47 G §421: the node reads the entry its want names, through the seat mesh-catalog's verb entry, and the part's digest + // from the want; what it hears another node run (the mailer on it2) is read by the same fingerprint. + if !checkBuild { + b.registerEntry("web", fp, man) + } + b.registerEntry("mailer", fpIt("b"), mailer) + b.running("it2", "mailer", fpIt("b"), mdigest) want, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fp, Self: true}) wkey := meshwants.WantsByWantNodeModuleKey(itNode, "web") if _, err := b.wants.Put(wkey, want); err != nil { @@ -499,17 +557,19 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { t.Fatal(err) } if checkBuild { - // The build of the placed module is read through the node's session: one that does not decode is said, and nothing - // is applied; a done one, then a config event, and the node composes. - within(t, 5*time.Second, "the node did not read web's build key through its session", func() bool { - b.mu.Lock() - defer b.mu.Unlock() - return strings.Contains(strings.Join(b.said, "\n"), "the builder's "+buildKey) + _ = buildKey + // The entry of the build its want names is asked of the catalog half through the node's session: one the half holds none + // of holds the want (entry-missing, naming the fingerprint) and nothing is applied; once the half holds it and it is + // asked again by hand, then a config event, and the node composes. + within(t, 5*time.Second, "the have does not say the entry of web's build is missing", func() bool { + h := b.have("web") + return h.Phase == nec.PhaseHeld && strings.HasPrefix(h.Detail, agent.EntryMissing) && strings.Contains(h.Detail, fp) }) if b.file("web") != "" { - t.Fatal("composed with a build key that does not decode") + t.Fatal("composed with an entry the catalog half does not hold") } - if _, err := builds.Put(buildKey, done); err != nil { + b.registerEntry("web", fp, man) + if err := composerNow.Load().EntryAgain(context.Background(), fp); err != nil { t.Fatal(err) } if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"bonjour","source":"module"}`)); err != nil { @@ -799,6 +859,7 @@ func TestNatsTheBusServerIsTakenOverAndReplacedOnlyWhenItsBuildChangesIt(t *test mraw, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "nats", Manifest: man}) mkey, _ := meshcatalogclient.ManifestsKey(digest) modv, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("3", 40)}) + b.registerEntry("nats", fp, man) // the node reads the entry its want names (design 47 G §421) if _, err := manifests.Put(mkey, mraw); err != nil { t.Fatal(err) } diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 2b40d698..4e288c90 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -508,6 +508,9 @@ const NotComposedDetail = "the node cannot compose it, and keeps running what it // composerMark says h is a mark NotComposed wrote: failed, its detail opening with NotComposedDetail. func composerMark(h Have) bool { + if h.Phase == string(nodeengineclient.PhaseHeld) && heldForEntry(h.Detail) { + return true + } return h.Phase == string(nodeengineclient.PhaseFailed) && strings.HasPrefix(h.Detail, NotComposedDetail) } @@ -651,8 +654,14 @@ func (a *Agent) NotComposedEach(ctx context.Context, why string, held map[string a.mu.Lock() a.notComposed[module] = mark a.mu.Unlock() + phase, detail := string(nodeengineclient.PhaseFailed), NotComposedDetail+why + if heldForEntry(why) { + // The entry its want names is not held: the phase held, its detail opening entry-asked: or entry-missing: and + // naming the fingerprint (design 47 G section 421). + phase, detail = string(nodeengineclient.PhaseHeld), why + } a.writeHave(ctx, Have{Module: module, FP: mark.prior.FP, Composed: mark.prior.Composed, Sealed: mark.prior.Sealed, - Phase: string(nodeengineclient.PhaseFailed), Detail: NotComposedDetail + why}) + Phase: phase, Detail: detail}) } } diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index a609f558..34268265 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -1772,6 +1772,12 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes( if h := b.haves.last("web"); h.Phase != string(nodeengineclient.PhaseRunning) || h.Composed != dig { t.Errorf("web, which composes, is marked beside idle: %+v", h) } + // Design 47 G §421: a module whose want's entry the catalog half holds none of is the phase held, its detail opening + // entry-missing: and naming the fingerprint; composed again, its have is what it runs. + b.ag.NotComposedEach(context.Background(), "", map[string]string{"idle": EntryMissing + " the catalog half holds no entry of the build src2:x"}) + if h := b.haves.last("idle"); h.Phase != string(nodeengineclient.PhaseHeld) || h.Composed != idleDig || h.Detail != EntryMissing+" the catalog half holds no entry of the build src2:x" { + t.Errorf("idle's have while its entry is missing: %+v", h) + } b.ag.NotComposedEach(context.Background(), "", nil) if h := b.haves.last("idle"); h.Phase != string(nodeengineclient.PhaseRunning) || h.Composed != idleDig { t.Errorf("idle's have once it composes again: %+v", h) diff --git a/internal/agent/parts.go b/internal/agent/parts.go index bb5a1f83..b324b9a7 100644 --- a/internal/agent/parts.go +++ b/internal/agent/parts.go @@ -13,6 +13,19 @@ import ( // disagree (design 47 G section 524). const PartsDiffer = "parts-differ:" +// EntryAsked and EntryMissing open the detail of a have held for the entry the want names (design 47 G section 421 and +// section 614): the entry is asked of the catalog half and not yet answered, or the half answered that it holds none. Both +// are the phase held, and each names the fingerprint. +const ( + EntryAsked = "entry-asked:" + EntryMissing = "entry-missing:" +) + +// heldForEntry says whether why is a hold for the entry a want names. +func heldForEntry(why string) bool { + return strings.HasPrefix(why, EntryAsked) || strings.HasPrefix(why, EntryMissing) +} + // partsDifference checks a want's `parts` against the digests of the archive, process and container resources of the // composition, which are the digests the apply fetches and checks (design 47 G section 524). A want with no `parts` (an // assigner of before G, or a node whose record does not state `want-parts`) is applied as today: it returns "". With diff --git a/internal/link/running.go b/internal/link/running.go index 722ce19a..fa24b0d1 100644 --- a/internal/link/running.go +++ b/internal/link/running.go @@ -93,3 +93,15 @@ func (q *Session) FollowRunning(ctx context.Context, heard func(nodeengine.Runni <-ctx.Done() return nil } + +// OverNATS reads the stream as the link's session does, over its connection. +func (b OverNATS) FirstRunning(ctx context.Context, first func([]byte)) error { + return (&natsLink{js: b.JS}).FirstRunning(ctx, first) +} + +// HearRunning follows the stream over the connection (see natsLink.HearRunning). +func (b OverNATS) HearRunning(consumer string, heard func([]byte)) (func(), error) { + return (&natsLink{js: b.JS}).HearRunning(consumer, heard) +} + +var _ RunningBus = OverNATS{} diff --git a/internal/nodegrant/testdata/node.json b/internal/nodegrant/testdata/node.json index 6e82524d..0270797c 100644 --- a/internal/nodegrant/testdata/node.json +++ b/internal/nodegrant/testdata/node.json @@ -21,7 +21,9 @@ "$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.NODE.>", "$JS.API.CONSUMER.CREATE.OBJ_declaration-files-NODE.*.$O.declaration-files-NODE.C.*", "$JS.API.CONSUMER.CREATE.SEAT_mesh-conditions_findings-asked", + "$JS.API.CONSUMER.CREATE.SEAT_node-engine_running", "$JS.API.CONSUMER.CREATE.SEAT_mesh-conditions_findings-asked.>", + "$JS.API.CONSUMER.CREATE.SEAT_node-engine_running.>", "$JS.API.CONSUMER.DELETE.KV_composed-NODE.>", "$JS.API.CONSUMER.DELETE.KV_mesh-config_resolved.>", "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-assigner_assignments.>", @@ -41,6 +43,8 @@ "$JS.API.CONSUMER.DELETE.KV_node-engine_have.>", "$JS.API.CONSUMER.DELETE.OBJ_declaration-files-NODE.*", "$JS.API.CONSUMER.DELETE.SEAT_mesh-conditions_findings-asked.>", + "$JS.API.CONSUMER.DELETE.SEAT_node-engine_running.>", + "$JS.API.CONSUMER.INFO.SEAT_node-engine_running.>", "$JS.API.CONSUMER.INFO.NODES.NODE", "$JS.API.DIRECT.GET.KV_composed-NODE.>", "$JS.API.DIRECT.GET.KV_mesh-config_resolved.$KV.mesh-config_resolved.NODE.>", @@ -79,6 +83,7 @@ "$JS.API.STREAM.INFO.KV_mesh-provisioner_provisions", "$JS.API.STREAM.INFO.KV_node-engine_have", "$JS.API.STREAM.INFO.SEAT_mesh-conditions_findings-asked", + "$JS.API.STREAM.INFO.SEAT_node-engine_running", "$JS.FC.KV_composed-NODE.>", "$JS.FC.KV_mesh-config_resolved.>", "$JS.FC.KV_mesh-delivery-assigner_assignments.>", @@ -110,6 +115,7 @@ "$SRV.PING.>", "$SRV.PING.node-tools.NODE", "mesh.control.NODE.>", + "mesh.seat.mesh-catalog.call.entry.node~NODE", "mesh.seat.mesh-catalog.call.seat-definitions.node~NODE", "mesh.seat.node-backup.call.backed-up.NODE.node~NODE", "mesh.seat.node-backup.call.now.NODE.node~NODE", diff --git a/internal/selfcompose/composedstate.go b/internal/selfcompose/composedstate.go index aef6dfbc..34c0f2c9 100644 --- a/internal/selfcompose/composedstate.go +++ b/internal/selfcompose/composedstate.go @@ -86,6 +86,11 @@ func (c *Composer) publishComposed(ctx context.Context) { // the indexer's, where the builder holds a done build of it; else the want's; else what runs of it. A module with none // is left out. func (c *Composer) newestFPs(ctx context.Context, at *published) map[string]string { + if c.byEntries() { + // The newest build is the indexer's (and the builder's done key says it is built): neither is read, and the node + // follows no branch (design 47 G §421, §628), so the state holds the wanted builds alone. + return maps.Clone(at.fps) + } out := map[string]string{} r := reader{c: c} for _, module := range at.placed { diff --git a/internal/selfcompose/entries.go b/internal/selfcompose/entries.go new file mode 100644 index 00000000..6b00c5b1 --- /dev/null +++ b/internal/selfcompose/entries.go @@ -0,0 +1,516 @@ +package selfcompose + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "reflect" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + + "github.com/novox/mesh-host/internal/agent" +) + +// **The catalog half, read by fingerprint** (design 47 G §421, §683, §698; the lead's answers A-47 and A-50). The node-engine +// follows no branch: for each build its wants and the runnings it heard name, it holds the catalog graph's entry of that +// fingerprint, asked of the seat mesh-catalog's verb `entry` once (meshcatalogclient.Entries) and kept in its composition +// state `composed-` beside the compositions (`entry.`, the entry as `entry` answers it), with each want it +// composes from (`want.`), written whole before the composition that uses them and read at its start before it asks +// anything. A module's manifest is the entry's, and each artifact's reference is made from the digest its want's parts name +// (referenceOf), so it reads neither the indexer's states nor the builder's. + +// The keys of the kept entries and wants in the composition state. +const ( + keptEntryPrefix = "entry." + keptWantPrefix = "want." + // keptWithin bounds one write of the composition state from an answer of the catalog half. + keptWithin = 15 * time.Second +) + +// keptWant is a want as the composition state keeps it (§683): the want beside when this node first kept it. +type keptWant struct { + V int `json:"v"` + Since time.Time `json:"since"` + Want meshwants.WantsByWantNodeModuleValue `json:"want"` +} + +// keptEntry is an entry held from the composition state or an answer, with when it was kept. +type keptEntry struct { + entry meshcatalog.CatalogEntry + since time.Time +} + +// EntryRow is one entry the node-engine keeps, as the seat's verb first-views lists it (§698). +type EntryRow struct { + Fingerprint string `json:"fingerprint"` + State string `json:"state"` + Since time.Time `json:"since"` +} + +// WantRow is one want the node-engine keeps, as the seat's verb first-views lists it (§698). +type WantRow struct { + Module string `json:"module"` + Fingerprint string `json:"fingerprint"` + KeptSince time.Time `json:"kept-since"` +} + +// byEntries says the composer reads the catalog half by fingerprint (the node's own composer), as the shadow, which keeps +// its reads of the indexer until it goes with the checker (§628), does not. +func (c *Composer) byEntries() bool { return c.entries != nil } + +// newEntries makes the composer's cache of entries over env's call of `entry`. +func (c *Composer) newEntries() { + if c.env.Entry == nil { + return + } + c.entries = meshcatalogclient.NewEntries(c.env.Entry) + c.entries.Now = c.now + c.entries.Changed = c.entryChanged + c.keptEnt = map[string]keptEntry{} + c.keptWants = map[string]bool{} + c.keptWantVal = map[string]keptWant{} +} + +// entryKey is an entry's key in the composition state, wantKey a want's. +// +// The fingerprint is keyed by its digits, as the compositions' keys are (engcomp.Key): a key holds no colon. +func entryKey(fp string) string { + return keptEntryPrefix + strings.TrimPrefix(fp, meshcatalogclient.SourcePrefix2) +} + +func wantStateKey(module string) string { return keptWantPrefix + module } + +// loadKept reads the entries and wants the composition state keeps, once at the start, before anything is asked of the catalog +// half (§683). A want read stands only until the live read of the wants answers (endKeptWants). A state that cannot be read +// is said, and what it would have kept is asked of `entry` and read from the wants as they come. +func (c *Composer) loadKept(ctx context.Context) { + if !c.byEntries() || c.env.ComposedState == nil { + return + } + kept, err := c.env.ComposedState.Entries(ctx) + if err != nil { + c.sayOnceEach("the entries and wants this node kept cannot be read, so each entry its wants name is asked of the catalog half: " + err.Error()) + return + } + now := c.now() + c.mu.Lock() + defer c.mu.Unlock() + for key, e := range kept { + switch { + case strings.HasPrefix(key, keptEntryPrefix): + var a meshcatalog.EntryAnswer + if json.Unmarshal(e.Value, &a) != nil || a.Entry.Module == "" { + c.env.Say("the kept entry " + key + " cannot be read, and is asked of the catalog half again") + continue + } + c.keptEnt[meshcatalogclient.SourcePrefix2+strings.TrimPrefix(key, keptEntryPrefix)] = keptEntry{entry: a.Entry, since: now} + case strings.HasPrefix(key, keptWantPrefix): + var w keptWant + module := strings.TrimPrefix(key, keptWantPrefix) + if json.Unmarshal(e.Value, &w) != nil || w.Want.Fp == "" { + c.env.Say("the kept want " + key + " cannot be read, so it stands until the wants are read") + continue + } + if _, live := c.wants[module]; !live { + c.wants[module] = w.Want + c.keptWants[module] = true + } + c.keptWantVal[module] = w + } + } +} + +// endKeptWants ends the standing of the kept wants once the live read of the wants answered: from then the wants the seat +// says are the ones the node composes from, never a want older than its stream's (§683). A kept want the read did not +// announce is gone. +func (c *Composer) endKeptWants() { + if !c.byEntries() { + return + } + c.mu.Lock() + defer c.mu.Unlock() + for module := range c.keptWants { + delete(c.wants, module) + delete(c.keptWants, module) + } +} + +// entryChanged is told of each entry answered or found missing: an answer is kept in the composition state before the pass it +// asks (so the composition that uses it finds it kept), and a pass is asked. +func (c *Composer) entryChanged(row meshcatalogclient.EntryRow) { + if row.State == meshcatalogclient.EntryAnswered && row.Entry != nil { + c.mu.Lock() + c.keptEnt[row.Fingerprint] = keptEntry{entry: *row.Entry, since: row.Since} + c.mu.Unlock() + c.keep(entryKey(row.Fingerprint), meshcatalog.EntryAnswer{Entry: *row.Entry}) + } + c.askRehand() +} + +// keep writes one value into the composition state; a state that cannot be written is said, and the value is kept again by +// the next pass. +func (c *Composer) keep(key string, v any) { + if c.env.ComposedState == nil { + return + } + raw, err := json.Marshal(v) + if err != nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), keptWithin) + defer cancel() + c.keptWrite.Lock() + defer c.keptWrite.Unlock() + if _, err := c.env.ComposedState.Put(ctx, key, raw); err != nil { + c.sayOnceEach("the node's composition state cannot keep " + key + ": " + err.Error()) + } +} + +// forget deletes one key from the composition state. +func (c *Composer) forget(key string) { + if c.env.ComposedState == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), keptWithin) + defer cancel() + c.keptWrite.Lock() + defer c.keptWrite.Unlock() + if err := c.env.ComposedState.Delete(ctx, key); err != nil { + c.sayOnceEach("the node's composition state cannot drop " + key + ": " + err.Error()) + } +} + +// entryOf is the entry of fp as the composer holds it: kept, or asked of the catalog half once (Entries.Need). +func (c *Composer) entryOf(fp string) (meshcatalog.CatalogEntry, meshcatalogclient.EntryState) { + c.mu.Lock() + k, ok := c.keptEnt[fp] + c.mu.Unlock() + if ok { + return k.entry, meshcatalogclient.EntryAnswered + } + return c.entries.Need(context.Background(), fp) +} + +// wantedBuilds is each fingerprint a want of this node or a running heard names, of the new form, with whether a want names it. +func (c *Composer) wantedBuilds() map[string]bool { + out := map[string]bool{} + c.mu.Lock() + defer c.mu.Unlock() + for module, w := range c.wants { + if module != agent.NodeModule && strings.HasPrefix(w.Fp, meshcatalogclient.SourcePrefix2) { + out[w.Fp] = true + } + } + for _, mods := range c.heard { + for _, run := range mods { + if !run.stopped && strings.HasPrefix(run.fingerprint, meshcatalogclient.SourcePrefix2) { + if _, named := out[run.fingerprint]; !named { + out[run.fingerprint] = false + } + } + } + } + return out +} + +// syncEntries asks the entry of each build a want or a running names, drops the one nothing names any more, keeps each want it +// composes from, and lays the picture frombus reads (c.modules, c.manifests) from the entries it holds. It runs at the start of +// each pass. +func (c *Composer) syncEntries(ctx context.Context) { + if !c.byEntries() { + return + } + need := c.wantedBuilds() + for fp := range need { + c.entryOf(fp) + } + for _, row := range c.entries.Rows() { + if _, named := need[row.Fingerprint]; !named { + c.entries.Release(row.Fingerprint) + } + } + c.mu.Lock() + var drop []string + for fp := range c.keptEnt { + if _, named := need[fp]; !named { + delete(c.keptEnt, fp) + drop = append(drop, entryKey(fp)) + } + } + type pending struct { + module string + kw keptWant + } + var put []pending + var gone []string + now := c.now() + for module, w := range c.wants { + if module == agent.NodeModule || c.keptWants[module] { + continue + } + had, ok := c.keptWantVal[module] + if ok && reflect.DeepEqual(had.Want, w) { + continue + } + since := now + if ok && had.Want.Fp == w.Fp { + since = had.Since + } + kw := keptWant{V: 1, Since: since, Want: w} + c.keptWantVal[module] = kw + put = append(put, pending{module, kw}) + } + for module := range c.keptWantVal { + if _, held := c.wants[module]; !held { + delete(c.keptWantVal, module) + gone = append(gone, wantStateKey(module)) + } + } + c.mu.Unlock() + // Written whole before the composition that uses them (§683). + for _, p := range put { + c.keep(wantStateKey(p.module), p.kw) + } + for _, key := range append(drop, gone...) { + c.forget(key) + } + c.layModules() +} + +// layModules lays c.modules and c.manifests from the entries held: what each node was heard to run first, then each module this +// node's want names, at the want's fingerprint (a want wins). +func (c *Composer) layModules() { + c.mu.Lock() + defer c.mu.Unlock() + modules := map[string]meshcatalog.ModulesByModuleValue{} + lay := func(module, fp string) { + k, ok := c.keptEnt[fp] + if !ok || k.entry.Module == "" { + return + } + e := k.entry + digest, raw, err := entryManifest(e) + if err != nil { + return + } + m := meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: e.Repository, Commit: e.Commit, Path: e.Path, + Needs: e.Needs, Packages: e.Packages, Verdict: e.Verdict} + if e.Inputs != nil { + m.Problems, m.Warnings, m.Unknown = e.Inputs.Problems, e.Inputs.Warnings, e.Inputs.Unknown + } + modules[module] = m + c.manifests[digest] = raw + } + nodes := make([]string, 0, len(c.heard)) + for node := range c.heard { + nodes = append(nodes, node) + } + sort.Strings(nodes) + for _, node := range nodes { + for module, run := range c.heard[node] { + if !run.stopped && run.fingerprint != "" { + lay(module, run.fingerprint) + } + } + } + for module, w := range c.wants { + if module != agent.NodeModule && w.Fp != "" { + lay(module, w.Fp) + } + } + // A module an assignment places whose entry is not held (not wanted here, not heard running, or its entry asked and not + // answered) stands in the catalogue as its name alone, so the placement still finds it: a module held for its entry offers + // nothing to the rest until its entry is held (§421). + for _, p := range c.assignments { + for _, pm := range p.Modules { + if _, held := modules[pm.Module]; held || pm.Module == "" { + continue + } + raw, _ := json.Marshal(map[string]any{"module": pm.Module, "version": "0"}) + sum := sha256.Sum256(raw) + digest := "sha256:" + hex.EncodeToString(sum[:]) + // At the fingerprint its want names, so it is no build behind the catalog's and its hold (entryHolds) is what says it. + modules[pm.Module] = meshcatalog.ModulesByModuleValue{Manifest: digest, Fp2: c.wants[pm.Module].Fp} + c.manifests[digest] = raw + } + } + c.modules = modules +} + +// entryManifest is an entry's manifest as a reader takes it, with its digest: the one the indexer set at take-in, or, where it set +// none, the digest of the manifest's bytes. +func entryManifest(e meshcatalog.CatalogEntry) (digest string, raw []byte, err error) { + if raw, err = json.Marshal(e.Manifest); err != nil { + return "", nil, err + } + digest = e.ManifestDigest + if digest == "" { + sum := sha256.Sum256(raw) + digest = "sha256:" + hex.EncodeToString(sum[:]) + } + return digest, raw, nil +} + +// entryHolds is, of each module this node's want names, why its entry is not held: asked and not answered, or answered not +// found. It is the phase held with a detail opening entry-asked: or entry-missing: and naming the fingerprint (§421, §614). +func (c *Composer) entryHolds() map[string]string { + if !c.byEntries() { + return nil + } + c.mu.Lock() + wants := map[string]meshwants.WantsByWantNodeModuleValue{} + for module, w := range c.wants { + if module != agent.NodeModule { + wants[module] = w + } + } + c.mu.Unlock() + out := map[string]string{} + for module, w := range wants { + if !strings.HasPrefix(w.Fp, meshcatalogclient.SourcePrefix2) { + continue + } + switch e, state := c.entryOf(w.Fp); state { + case meshcatalogclient.EntryAsked: + out[module] = fmt.Sprintf("%s the entry of the build %s is asked of the catalog half and not yet answered, so the node composes nothing new of %s", + agent.EntryAsked, w.Fp, module) + case meshcatalogclient.EntryNotFound: + out[module] = fmt.Sprintf("%s the catalog half holds no entry of the build %s, so the node composes nothing new of %s", + agent.EntryMissing, w.Fp, module) + default: + if missing := partsMissing(e, w); missing != "" { + out[module] = fmt.Sprintf("the want of %s names no digest for %s, so the node composes nothing new of it", module, missing) + } + } + } + return out +} + +// partsMissing names the artifacts of e whose digest w's parts do not give; "" when each has one. +func partsMissing(e meshcatalog.CatalogEntry, w meshwants.WantsByWantNodeModuleValue) string { + var out []string + for _, a := range e.Artifacts { + if w.Parts[a.Ref] == "" { + out = append(out, a.Ref) + } + } + sort.Strings(out) + return strings.Join(out, ", ") +} + +// referenceOf is an artifact's reference made from the digest its want's part names, in the form the artifact store is +// addressed by (the lead's answer A-50): artifact-store:////blobs/. +func referenceOf(module, artifact, digest string) string { + return "artifact-store://" + module + "/" + artifact + "/blobs/" + digest +} + +// buildOf is the build a want names, made from its entry and its parts alone (the builder's done key is not read): done, with +// each artifact of the entry at the digest its part names and the reference made from it. Nil where the entry is not held, the +// want names another fingerprint, or a part is missing. +func (c *Composer) buildOf(module, fp string) *builds.BuilderBuild { + c.mu.Lock() + w, wanted := c.wants[module] + k, ok := c.keptEnt[fp] + c.mu.Unlock() + if !wanted || w.Fp != fp || !ok || partsMissing(k.entry, w) != "" { + return nil + } + digest, raw, err := entryManifest(k.entry) + if err != nil { + return nil + } + c.mu.Lock() + c.manifests[digest] = raw + c.mu.Unlock() + b := builds.BuilderBuild{State: builds.BuildDone, Manifest: digest} + for _, a := range k.entry.Artifacts { + _, name, _ := strings.Cut(a.Ref, ":") + digest := w.Parts[a.Ref] + b.Artifacts = append(b.Artifacts, builds.BuiltArtifact{Name: name, Kind: a.Kind, Digest: digest, + Reference: referenceOf(module, name, digest)}) + } + return &b +} + +// FirstViews is the node-engine's first views as the seat's verb first-views lists them (§698): the views it follows (none: it +// follows no branch), the entries it keeps and the wants it composes from. +func (c *Composer) FirstViews() (views []map[string]any, entries []EntryRow, wants []WantRow) { + views, entries, wants = []map[string]any{}, []EntryRow{}, []WantRow{} + if !c.byEntries() { + return + } + seen := map[string]bool{} + for _, r := range c.entries.Rows() { + seen[r.Fingerprint] = true + entries = append(entries, EntryRow{Fingerprint: r.Fingerprint, State: string(r.State), Since: r.Since}) + } + c.mu.Lock() + for fp, k := range c.keptEnt { + if !seen[fp] { + entries = append(entries, EntryRow{Fingerprint: fp, State: string(meshcatalogclient.EntryAnswered), Since: k.since}) + } + } + for module, w := range c.wants { + if module == agent.NodeModule || w.Fp == "" { + continue + } + wants = append(wants, WantRow{Module: module, Fingerprint: w.Fp, KeptSince: c.keptWantVal[module].Since}) + } + c.mu.Unlock() + sort.Slice(entries, func(i, j int) bool { return entries[i].Fingerprint < entries[j].Fingerprint }) + sort.Slice(wants, func(i, j int) bool { return wants[i].Module < wants[j].Module }) + return +} + +// EntryAgain asks fp's entry of the catalog half again now (the seat's verb entry-again, §698): one the composer keeps, whatever +// its state. The entry kept of it is dropped, so a pass holds the module until the answer. +func (c *Composer) EntryAgain(ctx context.Context, fp string) error { + if !c.byEntries() { + return fmt.Errorf("this engine reads no entry by fingerprint") + } + c.mu.Lock() + _, kept := c.keptEnt[fp] + delete(c.keptEnt, fp) + c.mu.Unlock() + held := kept + for _, r := range c.entries.Rows() { + held = held || r.Fingerprint == fp + } + if !held { + return fmt.Errorf("no entry of %s is kept", fp) + } + if !kept { + return c.entries.Again(ctx, fp) + } + c.entries.Need(ctx, fp) // makes the row and asks once + c.askRehand() + return nil +} + +// EntryCall is env.Entry over the engine's link session: the seat mesh-catalog's verb `entry`, read in the engine's own name +// (the manifest's invokes names it). A refusal in the verb's own words that it keeps no entry of the build is the holder's +// answer that it holds none, and is told so; any other failure is a call that did not reach the holder, and is told so too. +func EntryCall(ask func(ctx context.Context, seat, verb string, args any) (json.RawMessage, error)) meshcatalogclient.Call { + return func(key string, body any) (json.RawMessage, error) { + seatName, verb, _ := strings.Cut(strings.TrimPrefix(key, "seat:"), ".") + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + raw, err := ask(ctx, seatName, verb, body) + if err != nil && strings.Contains(err.Error(), noEntryWords) { + out, _ := json.Marshal(map[string]string{"error": err.Error()}) + return out, nil + } + return raw, err + } +} + +// noEntryWords are the words the seat's holder refuses an unknown fingerprint in. +const noEntryWords = "no entry of fingerprint" diff --git a/internal/selfcompose/entries_test.go b/internal/selfcompose/entries_test.go new file mode 100644 index 00000000..7136b509 --- /dev/null +++ b/internal/selfcompose/entries_test.go @@ -0,0 +1,145 @@ +package selfcompose + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + + "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" + "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" + nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "github.com/novox/mesh-host/internal/agent" + "github.com/novox/mesh-host/internal/buckets" + "github.com/novox/mesh-host/internal/buckets/bucketstest" + "github.com/novox/mesh-host/internal/compositions" +) + +// Design 47 G §421, §683, §698 (the lead's answers A-47, A-50): the node-engine reads the entry its want names by the +// fingerprint, through its own calls of the seat mesh-catalog's verb entry, and reads neither the indexer's states nor the +// builder's; a want whose entry the catalog half holds none of is held, its have saying entry-missing and naming the +// fingerprint; the entries and wants it composes from are kept in its composition state, written before the composition that +// uses them and read at its start before anything is asked; a want removed takes its kept want and entry away. +func composeFromEntries(t *testing.T) { + b := newFakeBus() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) + ghost := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64) + var manifest map[string]any + if err := json.Unmarshal([]byte(lemurs), &manifest); err != nil { + t.Fatal(err) + } + entry := meshcatalog.CatalogEntry{Module: "lemurs", Fp: fp, Manifest: manifest, Repository: "novox/x", Commit: "c", + ManifestDigest: meshcatalogclient.ManifestDigest([]byte(lemurs))} + var mu sync.Mutex + var asked []string + ask := func(_ context.Context, seat, verb string, args any) (json.RawMessage, error) { + in := args.(meshcatalog.EntryInput) + mu.Lock() + asked = append(asked, in.Fingerprint) + mu.Unlock() + if seat != "mesh-catalog" || verb != "entry" { + t.Errorf("asked %s.%s", seat, verb) + } + if in.Fingerprint != fp { + return nil, errors.New("no entry of fingerprint " + in.Fingerprint + " is kept") + } + return json.Marshal(meshcatalog.EntryAnswer{Entry: entry}) + } + state := &fakeComposed{kv: map[string]ComposedEntry{}} + var held map[string]string + env := b.env() + env.Entry, env.ComposedState = EntryCall(ask), state + env.Refused = func(_ string, h map[string]string) { mu.Lock(); held = h; mu.Unlock() } + c := New(env) + out, err := c.Watch(ctx, "n1.>") + if err != nil { + t.Fatal(err) + } + place := assignedValues(t, meshmoduleassigner.Assignments{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"}, + Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}, {Module: "ghost"}}}) + have, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}}) + nodeWant, _ := bucketstest.NodeWant(meshwants.WantsByWantNodeNodeValue{Self: true}) + b.putAssigned(t, place) + b.put(t, buckets.Have, "n1.node", have) + b.put(t, buckets.Wants, "want.n1.node", nodeWant) + for module, w := range map[string]meshwants.WantsByWantNodeModuleValue{"lemurs": {Fp: fp, Self: true}, "ghost": {Fp: ghost, Self: true}} { + raw, _ := bucketstest.Want(w) + b.put(t, buckets.Wants, "want.n1."+module, raw) + } + for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, ResolvedBucket} { + b.send(bucket, agent.Update{Caught: true}) + } + lemursKey, _ := compositions.Key("n1", "lemurs", fp) + handed := map[string]bool{} + for !handed[lemursKey] { + handed[next(t, out).Key] = true + } + // Neither the indexer's states nor the builder's are watched or read. + b.mu.Lock() + for _, r := range b.reads { + if r[0] == buckets.Modules || r[0] == buckets.Manifests || r[0] == buckets.Verdicts || r[0] == builds.BuilderBuildsBucket { + t.Errorf("the node-engine read %s of %s", r[1], r[0]) + } + } + b.mu.Unlock() + // A want whose entry the half holds none of is held, saying entry-missing and the fingerprint; the rest is composed. + mu.Lock() + why := held["ghost"] + mu.Unlock() + if !strings.HasPrefix(why, agent.EntryMissing) || !strings.Contains(why, ghost) || held["lemurs"] != "" { + t.Errorf("held %q, want ghost held with %s naming %s", held, agent.EntryMissing, ghost) + } + // Each entry and want it composes from is kept in the composition state before the composition that uses them. + state.mu.Lock() + _, hasEntry := state.kv[entryKey(fp)] + _, hasWant := state.kv["want.lemurs"] + state.mu.Unlock() + if !hasEntry || !hasWant { + t.Errorf("the composition state holds entry %v and want %v of lemurs: %v", hasEntry, hasWant, state.written()) + } + // The seat's first-views lists them. + _, entries, wants := c.FirstViews() + if len(entries) < 2 || len(wants) != 2 { + t.Errorf("first-views lists %v and %v", entries, wants) + } + // A restart reads what is kept before it asks anything: lemurs's entry is not asked again. + mu.Lock() + asked = nil + mu.Unlock() + env2 := b.env() + env2.Entry, env2.ComposedState = EntryCall(ask), state + c2 := New(env2) + if _, err := c2.Watch(ctx, "n1.>"); err != nil { + t.Fatal(err) + } + c2.mu.Lock() + kept, keptWant := c2.keptEnt[fp], c2.wants["lemurs"].Fp + c2.mu.Unlock() + if kept.entry.Module != "lemurs" || keptWant != fp { + t.Errorf("a restart read entry %q and want %q from the state", kept.entry.Module, keptWant) + } + // A want removed takes its kept want and entry away. + raw, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fp, Removed: true}) + b.put(t, buckets.Wants, "want.n1.lemurs", raw) + for range 20 { + state.mu.Lock() + _, hasWant = state.kv["want.lemurs"] + _, hasEntry = state.kv[entryKey(fp)] + state.mu.Unlock() + if !hasWant && !hasEntry { + break + } + next(t, out) + } + if hasWant || hasEntry { + t.Errorf("a removed want left its kept want %v and entry %v", hasWant, hasEntry) + } +} diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index d4c84594..91753a13 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -126,6 +126,10 @@ type Env struct { // SeatListChanged says the reader's held set changed: the node composes again, so what waited for the list is // handed at the read and not on a timer. Nil hears nothing. SeatListChanged <-chan struct{} + // Entry asks the seat mesh-catalog's verb `entry` of a fingerprint (meshcatalogclient.EntryVerb), in the engine's own name. + // Set, the composer reads the catalog half by the fingerprints its wants and the runnings it heard name and nothing else of + // the indexer's or the builder's states (design 47 G §421, §628); nil reads the indexer's states, as the shadow does. + Entry meshcatalogclient.Call } // ManifestOf is the digest of the manifest of the build a module runs at fp, from the composer's picture of the catalog half; @@ -175,6 +179,14 @@ type Composer struct { notTaken map[string]string // unmadeSaid is, by module, the stable part of the reason its last journal line said (without where the picture stands). unmadeSaid map[string]string + // entries is the cache of entries asked by fingerprint, nil where the composer reads the indexer's states; keptEnt the entries + // held from the composition state or an answer, keptWantVal the wants it keeps there and keptWants the modules whose want + // stands from that state alone until the live read of the wants answers (entries.go). Under mu; keptWrite orders the writes. + entries *meshcatalogclient.Entries + keptEnt map[string]keptEntry + keptWantVal map[string]keptWant + keptWants map[string]bool + keptWrite sync.Mutex // modulesRev is the revision of the last update of the indexer's modules taken: where the picture of them stands. modulesRev uint64 // heard is, by node and module, what each node's event running last said it runs, kept from the seat's stream by this @@ -374,6 +386,12 @@ func New(env Env) *Composer { if env.Say == nil { env.Say = func(string) {} } + c := newComposer(env) + c.newEntries() + return c +} + +func newComposer(env Env) *Composer { return &Composer{env: env, wants: map[string]meshwants.WantsByWantNodeModuleValue{}, assignments: map[string]meshmoduleassigner.Assignments{}, assignmentKeys: meshmoduleassignerclient.NewKeys(), records: map[string]nodeengine.NodeRecord{}, inOwn: map[string]bool{}, modules: map[string]meshcatalog.ModulesByModuleValue{}, values: map[string]map[string]any{}, manifests: map[string][]byte{}, archives: map[string][]byte{}, builds: map[string]builds.BuilderBuild{}, handed: map[string]string{}, notTaken: map[string]string{}, unmadeSaid: map[string]string{}, rehand: make(chan struct{}, 1), sealedOwn: map[string]ownSealed{}, @@ -437,6 +455,21 @@ type input struct { } func (c *Composer) inputs() []input { + ins := c.allInputs() + if !c.byEntries() { + return ins + } + // Reading the catalog half by fingerprint, it watches neither the indexer's modules nor the builder's builds (§421, §628). + out := ins[:0:0] + for _, in := range ins { + if in.bucket != buckets.Modules && in.bucket != builds.BuilderBuildsBucket { + out = append(out, in) + } + } + return out +} + +func (c *Composer) allInputs() []input { node := c.env.Node return []input{ {buckets.Wants, "want." + node + ".>", (*Composer).takeWant, nil}, @@ -491,6 +524,8 @@ func (c *Composer) takeBuildDone(u agent.Update) bool { // input's watch ends. func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, error) { ins := c.inputs() + // What the node kept of the entries and the wants it composes from is read before anything is asked (§683). + c.loadKept(ctx) type tagged struct { in int u agent.Update @@ -598,6 +633,9 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er dirty = true // a secret changed: its new generation is asked of the vault at the compose case t.u.Caught: caught[t.in] = true + if ins[t.in].bucket == buckets.Wants { + c.endKeptWants() // the wants the seat says now are the ones composed from (§683) + } case ins[t.in].changed != nil: dirty = ins[t.in].changed(c, t.u) || dirty default: @@ -759,6 +797,7 @@ func (c *Composer) takeWantLocked(u agent.Update) string { return "" } delete(c.notTaken, module) + delete(c.keptWants, module) w, err := meshwants.DecodeWantsByWantNodeModuleValue(u.Value) if u.Deleted || err != nil || w.Removed || w.Fp == "" { delete(c.wants, module) @@ -960,6 +999,7 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { } } }() + c.syncEntries(ctx) c.mu.Lock() fps := map[string]string{} kept := map[string]string{} @@ -1024,6 +1064,15 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { // (ADR 0636 §1): the module is held as it runs before anything of its build is read, so no take-in or artifact // reference of it (the artifact store's seat is not bootstrapped) can refuse the pass (issue 1966). scope.Held = c.heldUntilListed(ctx, r, own) + // A want whose entry the catalog half has not answered, or holds none of, is held with what runs of it (§421). + if holds := c.entryHolds(); len(holds) > 0 { + if scope.Held == nil { + scope.Held = map[string]string{} + } + for module, why := range holds { + scope.Held[module] = why + } + } missed := c.missed.Load() compositions, skipped, held, err := frombus.Compose(ctx, r, c.env.Mesh, scope, fps) err = c.missedIn(missed, err) @@ -1533,6 +1582,9 @@ func (r reader) Manifest(ctx context.Context, digest string) ([]byte, error) { if ok { return raw, nil } + if r.c.byEntries() { + return nil, fmt.Errorf("the catalog half's entries hold no manifest %s", digest) + } key, err := meshcatalogclient.ManifestsKey(digest) if err != nil { return nil, err @@ -1554,6 +1606,9 @@ func (r reader) Manifest(ctx context.Context, digest string) ([]byte, error) { } func (r reader) Build(ctx context.Context, module, fp string) (*builds.BuilderBuild, error) { + if r.c.byEntries() { + return r.c.buildOf(module, fp), nil + } key, err := builds.BuilderBuildKey(module, fp) if err != nil { return nil, err diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index f6d1f386..af655ddb 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -138,7 +138,7 @@ func next(t *testing.T, out <-chan agent.Update) agent.Update { case u := <-out: return u case <-time.After(5 * time.Second): - t.Fatal("no update from the composer") + t.Fatalf("no update from the composer") return agent.Update{} } } @@ -156,6 +156,7 @@ func quiet(t *testing.T, out <-chan agent.Update) { // every input was read, hands each, then Caught; an input that changes nothing composed hands nothing; a module no want // names any more is taken back; a node that cannot compose says so once and hands nothing. func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { + t.Run("it reads the catalog half by the fingerprints its wants name", composeFromEntries) b := newFakeBus() fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64) digest := meshcatalogclient.ManifestDigest([]byte(lemurs)) diff --git a/module.json b/module.json index 82b00dec..3d95bb27 100644 --- a/module.json +++ b/module.json @@ -21,6 +21,7 @@ } ], "invokes": [ + "seat:mesh-catalog.entry", "seat:mesh-catalog.seat-definitions" ], "consumes": [ -- 2.54.0 From dc1aee51d953d805d91e63cd45e4cdc267008e48 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 00:16:31 +0100 Subject: [PATCH 21/59] W13.5: the node agent asks mesh-recovery for a restore point before a form change and restores it at a revert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before a move across a kept data's form change the node agent asks snapshot-kept and waits for the ended of its own act; at a revert it stops the newer build (Env.Stop), asks kept-points, asks restore-kept with the snapshot of the point of the form and fingerprint it reverts from, waits for restored, applies the reverted build, and calls the point's catch-up as the module's tool through the tool runner. The wait is in the have (ask-at, act), bounded by a bounds key of the state bounds (recovery-wait-minutes, 90), and each ending is read as the table of 560 says. The manifest consumes mesh-recovery's ended and restored and invokes its three verbs. Builds: design 47 G §296, §297, §554, §555, §560, §630, §631, §632, §633, A-77, A-92, A-98, A-108 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- cmd/mesh-host/nodeagent.go | 7 +- cmd/mesh-host/recovery.go | 83 ++++++ cmd/mesh-host/recovery_integration_test.go | 125 ++++++++ go.mod | 1 + internal/agent/agent.go | 12 + internal/agent/agent_test.go | 1 + internal/agent/kept.go | 323 +++++++++++++++++++++ internal/agent/kept_test.go | 257 ++++++++++++++++ internal/link/have.go | 1 + internal/link/messages.go | 2 + internal/link/recovery.go | 241 +++++++++++++++ internal/selfcompose/entries.go | 30 ++ module.json | 13 +- packaging/operator_keys_test.go | 15 +- 14 files changed, 1106 insertions(+), 5 deletions(-) create mode 100644 cmd/mesh-host/recovery.go create mode 100644 cmd/mesh-host/recovery_integration_test.go create mode 100644 internal/agent/kept.go create mode 100644 internal/agent/kept_test.go create mode 100644 internal/link/recovery.go diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 9a10734f..de5b7c8d 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -90,7 +90,7 @@ func (h sessionHaves) Read(ctx context.Context, module string) (agent.Have, bool return agent.Have{}, false } return agent.Have{Module: module, FP: got.Fp, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail, - Sealed: got.SealedAt}, true + Sealed: got.SealedAt, Wait: got.Wait}, true } // ReadErr is Read telling a have absent from one the bus could not read (agent.HavesReadErr). @@ -108,7 +108,7 @@ func (h sessionHaves) ReadErr(ctx context.Context, module string) (agent.Have, b return agent.Have{}, false, err } return agent.Have{Module: module, FP: got.Fp, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail, - Sealed: got.SealedAt}, true, nil + Sealed: got.SealedAt, Wait: got.Wait}, true, nil } func (h sessionHaves) Write(ctx context.Context, x agent.Have) { @@ -125,7 +125,7 @@ func (h sessionHaves) Write(ctx context.Context, x agent.Have) { } h.session.NoteModule(ctx, link.ModuleFact{Module: x.Module, FP: x.FP, Composed: x.Composed, Phase: phase, Failed: phase == string(nodeengineclient.PhaseFailed), Detail: shorten(x.Detail), Judgement: x.Judgement, Waits: x.Waits, - BuildAlone: x.BuildAlone, Kept: x.Kept, Sealed: x.Sealed, PullWait: x.PullWait}) + BuildAlone: x.BuildAlone, Kept: x.Kept, Sealed: x.Sealed, PullWait: x.PullWait, Wait: x.Wait}) } func (h sessionHaves) Removed(ctx context.Context, module string) { @@ -688,6 +688,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus go followRunning(run, session, composer, say) apply.UseEngineModule(composer.EngineModule) // the guard of the engine's own trees fixes no name (§527) env.Composed, env.SelfComposed, env.Policy, env.ComposedToday = composer, true, composer.Policy, composer.ComposedToday + env.Recovery = recoveryEnv(session, composer) env.WhyUnmade, env.Waiting = composer.WhyUnmade, composer.Waiting keptFetchFn := keptArtifactFetch(composer.KeptBuildOf, session.AskSeatOn, session.KeptObject) keptFetch.Store(&keptFetchFn) diff --git a/cmd/mesh-host/recovery.go b/cmd/mesh-host/recovery.go new file mode 100644 index 00000000..e7762514 --- /dev/null +++ b/cmd/mesh-host/recovery.go @@ -0,0 +1,83 @@ +package main + +import ( + "context" + "encoding/json" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + stdio "git.novox.be/novox/mesh-sdk/go" + + "github.com/novox/mesh-host/internal/agent" + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/selfcompose" +) + +// recoveryEnv is what the node agent asks of the recovery module and the manifests to move a module's kept data (design 47 G §296, +// §297, §554, §560, §630, A-77, A-98, A-108): the seat's verbs asked in the node-engine's own name, the endings heard from the +// seat's events, the bound of a wait kept as a key of the state bounds, the catch-up called as the module's tool. +func recoveryEnv(session *link.Session, composer *selfcompose.Composer) *agent.Recovery { + ask := func(ctx context.Context, verb string, in, out any) error { + raw, err := session.AskMeshSeat(ctx, meshrecovery.Seat, verb, in) + if err != nil { + return err + } + return json.Unmarshal(raw, out) + } + return &agent.Recovery{ + FormChanges: composer.FormChanges, + SnapshotKept: func(ctx context.Context, in meshrecovery.SnapshotKeptInput) (a meshrecovery.SnapshotKeptAnswer, err error) { + return a, ask(ctx, meshrecovery.VerbSnapshotKept, in, &a) + }, + RestoreKept: func(ctx context.Context, in meshrecovery.RestoreKeptInput) (a meshrecovery.RestoreKeptAnswer, err error) { + return a, ask(ctx, meshrecovery.VerbRestoreKept, in, &a) + }, + KeptPoints: func(ctx context.Context, in meshrecovery.KeptPointsInput) (a meshrecovery.KeptPointsAnswer, err error) { + return a, ask(ctx, meshrecovery.VerbKeptPoints, in, &a) + }, + Hear: func(_ context.Context, kind, module, kept string) (<-chan agent.Ending, func(), error) { + heard, stop, err := session.HearRecoveryEnding(kind, module, kept) + if err != nil { + return nil, nil, err + } + out := make(chan agent.Ending, 64) + done := make(chan struct{}) + go func() { + for { + select { + case ev := <-heard: + select { + case out <- agent.Ending(ev): + case <-done: + return + } + case <-done: + return + } + } + }() + return out, func() { stop(); close(done) }, nil + }, + Bound: func(ctx context.Context, what, id, module string, since time.Time, d time.Duration) (<-chan struct{}, func(), error) { + return session.KeepBound(ctx, what, id, module, since, d) + }, + CatchUp: func(ctx context.Context, module string, c meshrecovery.CatchUp) error { + return session.CallTool(ctx, module, c.Tool, c.Arguments) + }, + WaitMinutes: recoveryWaitMinutes(), + } +} + +// recoveryWaitMinutes is the manifest's config key recovery-wait-minutes, read through mesh-sdk's stdio.Config; its default is +// the design's 90 (design 47 G §633). +func recoveryWaitMinutes() int { + raw, err := stdio.Config("recovery-wait-minutes") + if err != nil { + return agent.RecoveryWaitMinutes + } + var n int + if json.Unmarshal(raw, &n) != nil || n <= 0 { + return agent.RecoveryWaitMinutes + } + return n +} diff --git a/cmd/mesh-host/recovery_integration_test.go b/cmd/mesh-host/recovery_integration_test.go new file mode 100644 index 00000000..092e9090 --- /dev/null +++ b/cmd/mesh-host/recovery_integration_test.go @@ -0,0 +1,125 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "testing" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/nodegrant" +) + +// Design 47 G §560, §630, A-77, A-92, A-108: the node agent's recovery step over a real bus. Its session asks the seat's verbs, hears +// the last `restored` already said and the next, keeps a wait's bound as a key of the state bounds whose time ends with the bus's +// marker, and calls the catch-up as the module's tool with its arguments. The recovery module is a stand-in that answers the verbs and +// says the events; the grants part composes the node's subjects from the manifest, so the test adds them to the node's grant. +func TestNatsTheRecoveryStepAsksHearsBoundsAndCatchesUpOverTheBus(t *testing.T) { + perms := nodegrant.For(itNode) + perms.Publish = append(perms.Publish, "$JS.API.>", "$KV.mesh-host_bounds.>", "mesh.seat.mesh-recovery.call.*.node~"+itNode, + "mesh.mod.db.tool.rebuild."+itNode) + perms.Subscribe = append(perms.Subscribe, "mesh.seat.mesh-recovery.event.>", "$KV.mesh-host_bounds.>", "_INBOX.>", "$JS.ACK.>", "$JS.FC.>") + b := newItBedAs(t, perms) + ctx := context.Background() + + for _, s := range []struct{ stream, subject string }{{meshrecovery.StreamEnded, meshrecovery.SubjectEndedAll}, + {meshrecovery.StreamRestored, meshrecovery.SubjectRestoredAll}} { + if _, err := b.js.AddStream(&nats.StreamConfig{Name: s.stream, Subjects: []string{s.subject}, MaxMsgsPerSubject: 1}); err != nil { + t.Fatal(err) + } + } + js, err := jetstream.New(b.conn) + if err != nil { + t.Fatal(err) + } + if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: link.BoundsBucket, History: 1, LimitMarkerTTL: time.Second}); err != nil { + t.Fatal(err) + } + say := func(act, outcome string) { + raw, _ := json.Marshal(meshrecovery.RestoredEvent{Act: act, Module: "db", Kept: "cache", Outcome: outcome, At: time.Now().UTC()}) + if _, err := b.js.Publish(meshrecovery.SubjectRestored("db", "cache"), raw); err != nil { + t.Fatal(err) + } + } + var restoreAsked meshrecovery.RestoreKeptInput + for verb, answer := range map[string]func(m *nats.Msg) any{ + meshrecovery.VerbKeptPoints: func(*nats.Msg) any { + return meshrecovery.KeptPointsAnswer{Points: []meshrecovery.Point{{Module: "db", Kept: "cache", Form: 1, Fingerprint: "f2", Snapshot: "snap-1", + CatchUp: &meshrecovery.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": true}}}}} + }, + meshrecovery.VerbRestoreKept: func(m *nats.Msg) any { + _ = json.Unmarshal(m.Data, &restoreAsked) + say("r1", meshrecovery.RestoredEventOutcomeDone) + return meshrecovery.RestoreKeptAnswer{Accepted: "r1"} + }, + } { + answer := answer + sub, err := b.conn.Subscribe("mesh.seat.mesh-recovery.call."+verb+".*", func(m *nats.Msg) { + out, _ := json.Marshal(map[string]any{"result": answer(m)}) + _ = m.Respond(out) + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + } + var toolArgs map[string]any + toolSub, err := b.conn.Subscribe(link.ToolSubject("db", "rebuild", itNode), func(m *nats.Msg) { + _ = json.Unmarshal(m.Data, &toolArgs) + _ = m.Respond([]byte(`{"result":{}}`)) + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = toolSub.Unsubscribe() }) + + say("old", meshrecovery.RestoredEventOutcomeFailed) // said before anyone listens: the last message of the subject + session := b.startSession() + r := recoveryEnv(session, nil) + + events, stop, err := r.Hear(ctx, "restored", "db", "cache") + if err != nil { + t.Fatal(err) + } + defer stop() + if first := <-events; first.Act != "old" || first.Outcome != "failed" { + t.Fatalf("the first event heard is %+v, want the last one said before", first) + } + pts, err := r.KeptPoints(ctx, meshrecovery.KeptPointsInput{Module: "db", Kept: "cache"}) + if err != nil || len(pts.Points) != 1 { + t.Fatalf("kept-points: %+v, %v", pts, err) + } + got, err := r.RestoreKept(ctx, meshrecovery.RestoreKeptInput{Module: "db", Kept: "cache", Form: 1, Fingerprint: "f2", Snapshot: pts.Points[0].Snapshot, Why: "w"}) + if err != nil || got.Accepted != "r1" || restoreAsked.Snapshot != "snap-1" { + t.Fatalf("restore-kept: %+v, %v, asked %+v", got, err, restoreAsked) + } + if ev := <-events; ev.Act != "r1" || ev.Outcome != "done" { + t.Fatalf("the ending heard is %+v", ev) + } + if err := r.CatchUp(ctx, "db", *pts.Points[0].CatchUp); err != nil || toolArgs["all"] != true { + t.Fatalf("catch-up: %v, arguments %v", err, toolArgs) + } + expired, end, err := r.Bound(ctx, "recovery", "db.restored.cache", "db", time.Now(), time.Second) + if err != nil { + t.Fatal(err) + } + defer end() + select { + case <-expired: + case <-time.After(10 * time.Second): + t.Fatal("the bound's end, the bus's marker, was not heard") + } +} + +// startSession is a session on the node's own connection, with no node agent run on it. +func (b *itBed) startSession() *link.Session { + b.session = &link.Session{Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second, + Say: b.say, Keys: func() link.RecordKeys { return link.RecordKeys{} }} + b.session.Attach(context.Background(), link.OverNATS{Conn: b.nodeConn, JS: b.nodeJS, Node: itNode}) + return b.session +} diff --git a/go.mod b/go.mod index a5028435..d40e3f1a 100644 --- a/go.mod +++ b/go.mod @@ -33,6 +33,7 @@ require ( require ( git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/meshrecovery v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshvault v0.0.0-00010101000000-000000000000 diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 4e288c90..ba620a77 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -75,6 +75,8 @@ type Have struct { // PullWait is, on a failed have, the pull from a store its apply failed on and waits on the holder of (issue 1966 // ENG-9, nodeengine.HaveByNodeModuleValue.PullWait): read through the have, as node_haves shows it. PullWait *nodeengine.PullWait + // Wait is the hold or wait the have says, in the recovery wait with its ask-at and act (design 47 G §560). + Wait *nodeengine.Wait } // Haves is the node's have bucket as the agent uses it. @@ -140,6 +142,9 @@ type Env struct { // build with it, restores the module's kept data, and then applies the reverted build (design 47 G A-98, §297). Nil // stops nothing. Stop func(ctx context.Context, module string) Result + // Recovery takes a module's kept data as a restore point before a form change moves it and restores it at a revert + // (design 47 G §296, §297, §554); nil moves none. + Recovery *Recovery // Backup backs a module up before a new composition of it is applied; nil backs up nothing. Backup func(ctx context.Context, module string) Backup // Kept is the resources the node last applied, by module, from the declaration it kept before the switch (ADR 0539 @@ -1632,6 +1637,12 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo return } + // A kept data on the bus whose form changes is taken as a restore point first, or restored at a revert (design 47 G §296, §297). + held, catchUps := a.keptStep(ctx, module, w, comp, ranBefore) + if held { + return + } + // A data module is backed up before a new composition of it moves (ADR 0429 §14), unless the want says to go on // without one (ADR 0539 §16). if comp.C.Data && ranBefore && a.env.Backup != nil && w.Want.Backup != meshwants.WantsByWantNodeModuleValueBackupSkip { @@ -1736,6 +1747,7 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo a.env.Settled(module) } a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(nodeengineclient.PhaseRunning), Detail: res.Detail}) + a.catchUp(ctx, module, catchUps) } // ranFailed marks what runs of a module as failed once an apply of it failed (novox/hq issue 1966, 2026-11-08): a failed diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 34268265..8a7b9791 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -930,6 +930,7 @@ func TestADataModuleIsBackedUpFirstAndHeldWhenItFails(t *testing.T) { // A data module with a good backup is applied after it; one that never ran has nothing to back up. func TestABackupDoneIsFollowedByTheApplyAndANewModuleIsNotBackedUp(t *testing.T) { + t.Run("a move across a kept data's form change is taken as a restore point and restored at a revert", testAMoveAcrossAFormChange) b := newBed(t) stop := b.run() defer stop() diff --git a/internal/agent/kept.go b/internal/agent/kept.go new file mode 100644 index 00000000..a4f38113 --- /dev/null +++ b/internal/agent/kept.go @@ -0,0 +1,323 @@ +package agent + +import ( + "context" + "fmt" + "time" + + "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" + "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// A module's kept data on the bus, taken as a restore point before a form change moves it and restored at a revert (design 47 G +// §296, §297, §554, §555, §560, §630 to §632, the lead's answers A-77, A-98 and A-108): the node agent asks the seat +// `mesh-recovery`'s `snapshot-kept` and waits for the `ended` of its own act, or, at a revert, stops the newer build, asks +// `kept-points`, asks `restore-kept` with the snapshot of the point of the form it reverts from, waits for its `restored`, and +// after the reverted build's apply calls the point's catch-up as the module's tool. + +// Ending is the ending event of an act of the recovery module as the node agent reads it: `ended` for a restore point taken, +// `restored` for one restored (design 47 G §560). +type Ending struct { + Act string + Outcome string + Error string + Snapshot string +} + +// RecoveryWaitMinutes is the default of the manifest's config key `recovery-wait-minutes`, the bound of the wait for an +// ending event, counted from the first ask (design 47 G §554, §633). +const RecoveryWaitMinutes = 90 + +// Recovery is what the node agent needs of the recovery module and the manifests to move kept data. Nil moves none. +type Recovery struct { + // FormChanges are the kept data whose form differs between the running build's manifest and the applied build's (the + // catalog half's entries by fingerprint); an error says it is not known. + FormChanges func(ctx context.Context, module, runningFp, appliedFp string) ([]catalogue.FormChange, error) + // The seat's verbs, asked in the node-engine's own name; an error is the ask not answered. + SnapshotKept func(ctx context.Context, in meshrecovery.SnapshotKeptInput) (meshrecovery.SnapshotKeptAnswer, error) + RestoreKept func(ctx context.Context, in meshrecovery.RestoreKeptInput) (meshrecovery.RestoreKeptAnswer, error) + KeptPoints func(ctx context.Context, in meshrecovery.KeptPointsInput) (meshrecovery.KeptPointsAnswer, error) + // Hear follows the ending events of kind "ended" or "restored" of one module's kept data: the last one said already first, + // then each after it, until stop is called. It is made before the verb is asked (design 47 G §560). + Hear func(ctx context.Context, kind, module, kept string) (events <-chan Ending, stop func(), err error) + // Bound writes the bound key `.` of a wait in the state bounds with its time to live, so its end is the bus's + // marker, heard as expired (design 47 G A-77, A-92); end deletes it. Nil keeps no bound. + Bound func(ctx context.Context, what, id, module string, since time.Time, d time.Duration) (expired <-chan struct{}, end func(), err error) + // CatchUp calls the module's tool on this node through the tool runner (design 47 G A-108). + CatchUp func(ctx context.Context, module string, c meshrecovery.CatchUp) error + // WaitMinutes is the bound in minutes, the config key recovery-wait-minutes; 0 reads as RecoveryWaitMinutes. + WaitMinutes int + // Now is the clock; nil reads the machine's. + Now func() time.Time +} + +func (r *Recovery) now() time.Time { + if r.Now != nil { + return r.Now() + } + return time.Now() +} + +func (r *Recovery) bound() time.Duration { + if r.WaitMinutes > 0 { + return time.Duration(r.WaitMinutes) * time.Minute + } + return RecoveryWaitMinutes * time.Minute +} + +// KeptMove is one kept data a want moves across a change of its form, as the seat mesh-recovery's verbs name it. +type KeptMove struct { + // Name is the kept data's local name, Form the form the restore point holds (the older one), and Fingerprint the + // fingerprint the restore point is taken before (the newer build's) or reverted from. + Name string + Form int + Fingerprint string + // CatchUp is, when taking the point, the tool the newer manifest declares beside the kept data. + CatchUp *meshrecovery.CatchUp +} + +// keptMoves are the restore points a move needs: a point to take before the newer build starts (the kept data's form rises, +// or the newer manifest declares it no more) and a point to restore before the older build starts (the kept data goes back to +// a lower form). A kept data the running build does not declare is neither: nothing was written before it. +func keptMoves(changes []catalogue.FormChange, runningFp, appliedFp string) (take, restore []KeptMove) { + for _, c := range changes { + switch { + case c.Running == nil: + case c.Applied == nil || c.Applied.Form > c.Running.Form: + m := KeptMove{Name: c.Name, Form: c.Running.Form, Fingerprint: appliedFp} + if c.Applied != nil && c.Applied.CatchUp != nil { + m.CatchUp = &meshrecovery.CatchUp{Tool: c.Applied.CatchUp.Tool, Arguments: c.Applied.CatchUp.Arguments} + } + take = append(take, m) + case c.Applied.Form < c.Running.Form: + restore = append(restore, KeptMove{Name: c.Name, Form: c.Applied.Form, Fingerprint: runningFp}) + } + } + return take, restore +} + +// recoveryHeld is how a wait for the recovery module ended when the move cannot go on: the words are the reason of the hold. +type recoveryHeld struct { + Words string + Interrupted bool +} + +// askAndWait asks the verb until its act is accepted and ended, as the table of design 47 G §560 says: `busy` waits for the +// ending of the act it names and asks again, `failed` with `stopped` asks again within the same bound, a failure with any +// other error holds. The wait is written in the have with its first ask's time and the act, so a restart waits for that act +// and asks nothing again; past the bound the have says the wait, and the wait goes on (the watchman raises it). +func (a *Agent) askAndWait(ctx context.Context, h Have, kind string, k KeptMove, ask func() (accepted, busy, errWords string, err error)) (Ending, *recoveryHeld) { + r := a.env.Recovery + module := h.Module + step := nodeengine.WaitStepCompose + wait := nodeengine.Wait{Owner: nodeengine.Seat, Step: step, On: "mesh-recovery." + kind + "." + module + "." + k.Name, + Reason: "recovery waits", Bound: r.bound().String()} + // A restart: the have names the wait with its ask time and act. + last, ok := a.env.Haves.Read(ctx, module) + if ok && last.Wait != nil && last.Wait.On == wait.On && last.Wait.AskAt != nil { + wait.AskAt, wait.Act, wait.Since = last.Wait.AskAt, last.Wait.Act, last.Wait.Since + } + now := r.now() + if wait.AskAt == nil { + wait.AskAt, wait.Since = &now, now + } + events, stop, err := r.Hear(ctx, kind, module, k.Name) + if err != nil { + return Ending{}, &recoveryHeld{Words: "the end of the recovery module's act cannot be heard: " + err.Error()} + } + defer stop() + var expired <-chan struct{} + if r.Bound != nil { + bound := r.bound() - now.Sub(*wait.AskAt) + if bound < time.Second { + bound = time.Second + } + exp, end, berr := r.Bound(ctx, "recovery", module+"."+kind+"."+k.Name, module, *wait.AskAt, bound) + if berr != nil { + return Ending{}, &recoveryHeld{Words: "the bound of the wait for the recovery module cannot be written: " + berr.Error()} + } + defer end() + expired = exp + } + write := func() { + w := wait + h.Wait = &w + a.writeHave(ctx, h) + } + act, busyOn := wait.Act, "" + for { + if act == "" && busyOn == "" { + write() // the first ask's time is in the have before the ask + accepted, busy, words, err := ask() + switch { + case err != nil: + if ctx.Err() != nil { + return Ending{}, &recoveryHeld{Words: BackupInterrupted, Interrupted: true} + } + return Ending{}, &recoveryHeld{Words: "the recovery module could not be asked: " + err.Error()} + case words != "": + return Ending{}, &recoveryHeld{Words: words} + case accepted != "": + act, wait.Act = accepted, accepted + write() + case busy != "": + busyOn = busy + default: + return Ending{}, &recoveryHeld{Words: "the recovery module answered neither accepted nor busy"} + } + } + select { + case <-ctx.Done(): + return Ending{}, &recoveryHeld{Words: BackupInterrupted, Interrupted: true} + case <-expired: + expired = nil + wait.Words = fmt.Sprintf("the recovery module has not ended its act within %s", r.bound()) + write() + case ev, open := <-events: + if !open { + return Ending{}, &recoveryHeld{Words: "the end of the recovery module's act is no longer heard"} + } + switch { + case busyOn != "" && ev.Act == busyOn: + busyOn = "" + case act != "" && ev.Act == act: + switch { + case ev.Outcome == meshrecovery.EndedEventOutcomeDone: + return ev, nil + case ev.Error == "stopped": + act, wait.Act = "", "" + case ev.Error == "cancelled": + return ev, &recoveryHeld{Words: "cancelled"} + default: + words := ev.Error + if words == "" { + words = "the recovery module's act failed" + } + return ev, &recoveryHeld{Words: words} + } + } + } + } +} + +// keptStep takes the restore points a move of a module needs before it applies and restores the ones a revert needs: it +// answers whether the module is held (its have says why, and the running build is kept) and, after a restore, the catch-up +// of the point to call once the reverted build runs. A want that says to go on without a backup asks neither. +func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp compNow, ranBefore bool) (held bool, catchUps []meshrecovery.CatchUp) { + r := a.env.Recovery + if !ranBefore || r == nil || r.FormChanges == nil || w.Want.Backup == meshwants.WantsByWantNodeModuleValueBackupSkip { + return false, nil + } + fp, dig := w.Want.Fp, w.Want.Composed + a.mu.Lock() + prev, runs := a.running[module] + a.mu.Unlock() + if !runs { + if h, found, _ := a.readHave(ctx, module); found { + prev = nodeengine.HaveByNodeModuleValue{Fp: h.FP, Composed: h.Composed} + } + } + if prev.Fp == "" || prev.Fp == fp { + return false, nil + } + changes, err := r.FormChanges(ctx, module, prev.Fp, fp) + if err != nil { + // Not known whether the move changes a form: said, and the move goes on as before, since holding every module for a + // build record that is gone would stop deliveries a form never changed. + a.sayOnce("kept:"+module+":"+fp, fmt.Sprintf("%s: whether its kept data changes form between %s and %s is not known: %v", module, prev.Fp, fp, err)) + return false, nil + } + take, restore := keptMoves(changes, prev.Fp, fp) + if len(take)+len(restore) == 0 { + return false, nil + } + hold := func(words string, interrupted bool) { + if interrupted || ctx.Err() != nil { + words = BackupInterrupted + a.env.Say(fmt.Sprintf("%s waits: %s; it asks again at the next start", module, words)) + } else { + a.env.Say(fmt.Sprintf("%s is held before %s: %s", module, fp, words)) + } + a.mu.Lock() + a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev, Held: true, AnsweredRev: w.Rev} + a.mu.Unlock() + since := r.now() + a.writeHave(ctx, Have{Module: module, FP: prev.Fp, Composed: prev.Composed, Phase: string(nodeengineclient.PhaseHeld), Detail: words, + Wait: &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepCompose, Reason: words, Since: since, Words: words}}) + } + backing := Have{Module: module, FP: fp, Composed: dig, Phase: string(nodeengineclient.PhaseBackingUp)} + for _, m := range take { + why := fmt.Sprintf("form change of %s's %s to %s", module, m.Name, fp) + _, held := a.askAndWait(ctx, backing, "ended", m, func() (string, string, string, error) { + got, err := r.SnapshotKept(ctx, meshrecovery.SnapshotKeptInput{Module: module, Kept: m.Name, Form: int64(m.Form), + Fingerprint: m.Fingerprint, CatchUp: m.CatchUp, Why: why}) + return got.Accepted, got.Busy, got.Error, err + }) + if held != nil { + hold(fmt.Sprintf("the restore point of %s's %s (form %d) was not taken: %s", module, m.Name, m.Form, held.Words), held.Interrupted) + return true, nil + } + } + if len(restore) == 0 { + return false, nil + } + // A revert: the newer build stops first (it removes nothing), so the live data is not written while it is set aside. + if a.env.Stop != nil { + if res := a.env.Stop(ctx, module); res.Failed { + hold("the newer build could not be stopped before the restore: "+res.Detail, false) + return true, nil + } + } + for _, m := range restore { + pts, err := r.KeptPoints(ctx, meshrecovery.KeptPointsInput{Module: module, Kept: m.Name}) + if err != nil { + hold(fmt.Sprintf("restore point missing: the points of %s's %s cannot be read: %v", module, m.Name, err), false) + return true, nil + } + var point *meshrecovery.Point + for i := range pts.Points { + p := pts.Points[i] + if p.Module == module && p.Kept == m.Name && int(p.Form) == m.Form && p.Fingerprint == m.Fingerprint { + if point == nil || p.At.After(point.At) { + point = &pts.Points[i] + } + } + } + if point == nil { + hold(fmt.Sprintf("restore point missing: %s's %s has none of form %d taken before %s", module, m.Name, m.Form, m.Fingerprint), false) + return true, nil + } + why := fmt.Sprintf("revert of %s's %s from %s", module, m.Name, m.Fingerprint) + _, held := a.askAndWait(ctx, backing, "restored", m, func() (string, string, string, error) { + got, err := r.RestoreKept(ctx, meshrecovery.RestoreKeptInput{Module: module, Kept: m.Name, Form: int64(m.Form), + Fingerprint: m.Fingerprint, Snapshot: point.Snapshot, Why: why}) + return got.Accepted, got.Busy, got.Error, err + }) + if held != nil { + hold(fmt.Sprintf("%s's %s (form %d) is not restored: %s", module, m.Name, m.Form, held.Words), held.Interrupted) + return true, nil + } + if point.CatchUp != nil { + catchUps = append(catchUps, *point.CatchUp) + } + } + return false, catchUps +} + +// catchUp calls, once the reverted build of a module runs, the catch-up of each restore point restored, as the module's own +// tool through the tool runner (design 47 G A-108). A tool that fails is said: the build runs, and the tool can be called by hand. +func (a *Agent) catchUp(ctx context.Context, module string, calls []meshrecovery.CatchUp) { + if a.env.Recovery == nil || a.env.Recovery.CatchUp == nil { + return + } + for _, c := range calls { + if err := a.env.Recovery.CatchUp(ctx, module, c); err != nil { + a.env.Say(fmt.Sprintf("%s runs the reverted build, and its catch-up tool %s failed: %v", module, c.Tool, err)) + continue + } + a.env.Say(fmt.Sprintf("%s's catch-up tool %s ran after the restore", module, c.Tool)) + } +} diff --git a/internal/agent/kept_test.go b/internal/agent/kept_test.go new file mode 100644 index 00000000..5c2f571b --- /dev/null +++ b/internal/agent/kept_test.go @@ -0,0 +1,257 @@ +package agent + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strings" + "sync" + "testing" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "github.com/novox/mesh-host/internal/compositions" +) + +// A recovery module that answers as scripted: each ask takes the next answer and says the events it carries after it. +type recoveryFake struct { + mu sync.Mutex + asks []string // verb and its input, in order + answers []fakeAnswer + points []meshrecovery.Point + events chan Ending + changes []catalogue.FormChange + calls []string // stop, ask, apply, catch-up, in order + catchUp []meshrecovery.CatchUp + expired chan struct{} +} + +type fakeAnswer struct { + accepted, busy, errWords string + then []Ending +} + +func (f *recoveryFake) next(verb string, in any) (string, string, string) { + f.mu.Lock() + defer f.mu.Unlock() + f.asks = append(f.asks, fmt.Sprintf("%s %s", verb, asJSON(in))) + f.calls = append(f.calls, verb) + if len(f.answers) == 0 { + return "", "", "no answer scripted" + } + a := f.answers[0] + f.answers = f.answers[1:] + for _, e := range a.then { + f.events <- e + } + return a.accepted, a.busy, a.errWords +} + +func (f *recoveryFake) env() *Recovery { + return &Recovery{ + FormChanges: func(context.Context, string, string, string) ([]catalogue.FormChange, error) { return f.changes, nil }, + SnapshotKept: func(_ context.Context, in meshrecovery.SnapshotKeptInput) (meshrecovery.SnapshotKeptAnswer, error) { + a, b, e := f.next("snapshot-kept", in) + return meshrecovery.SnapshotKeptAnswer{Accepted: a, Busy: b, Error: e}, nil + }, + RestoreKept: func(_ context.Context, in meshrecovery.RestoreKeptInput) (meshrecovery.RestoreKeptAnswer, error) { + a, b, e := f.next("restore-kept", in) + return meshrecovery.RestoreKeptAnswer{Accepted: a, Busy: b, Error: e}, nil + }, + KeptPoints: func(context.Context, meshrecovery.KeptPointsInput) (meshrecovery.KeptPointsAnswer, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, "kept-points") + return meshrecovery.KeptPointsAnswer{Points: f.points}, nil + }, + Hear: func(context.Context, string, string, string) (<-chan Ending, func(), error) { + return f.events, func() {}, nil + }, + Bound: func(context.Context, string, string, string, time.Time, time.Duration) (<-chan struct{}, func(), error) { + return f.expired, func() {}, nil + }, + CatchUp: func(_ context.Context, _ string, c meshrecovery.CatchUp) error { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, "catch-up") + f.catchUp = append(f.catchUp, c) + return nil + }, + Now: time.Now, + } +} + +func (f *recoveryFake) seen() []string { + f.mu.Lock() + defer f.mu.Unlock() + return slices.Clone(f.calls) +} + +// moveDB runs a data module at fp "a", then wants it at fp "b", and answers the bed once the second want is settled or held. +func moveDB(t *testing.T, f *recoveryFake, ranFrom, to string, settle func(b *bed) bool, before ...func(b *bed, dig string)) *bed { + t.Helper() + b := newBed(t) + inner, applies := b.env.Apply, 0 + b.env.Apply = func(ctx context.Context, c compositions.Composed, others []compositions.Composed) Result { + if applies++; applies >= 2 { + f.mu.Lock() + f.calls = append(f.calls, "apply") + f.mu.Unlock() + } + return inner(ctx, c, others) + } + f.events, f.expired = make(chan Ending, 16), make(chan struct{}) + b.env.Recovery = f.env() + b.env.Stop = func(context.Context, string) Result { + f.mu.Lock() + f.calls = append(f.calls, "stop") + f.mu.Unlock() + return Result{} + } + stop := b.run() + t.Cleanup(stop) + fp1, fp2 := fpOf(ranFrom), fpOf(to) + ck, craw, dig := composition(t, "db", fp1, true, file("db", "x", "1")) + wk, wraw := wantFor(t, "db", fp1, dig, "") + b.comps.put(ck, craw) + b.wants.put(wk, wraw) + eventually(t, "the first build is not applied", func() bool { return len(b.appliedList()) == 1 }) + b.idle() + ck2, craw2, dig2 := composition(t, "db", fp2, true, file("db", "x", "2")) + for _, h := range before { + h(b, dig) + } + wk2, wraw2 := wantFor(t, "db", fp2, dig2, "") + b.comps.put(ck2, craw2) + b.wants.put(wk2, wraw2) + eventually(t, "the move is neither applied nor held", func() bool { return settle(b) }) + return b +} + +// testAMoveAcrossAFormChange is design 47 G §296, §297, §554, §560, §630 to §632 and A-98, A-108 at the node agent. +func testAMoveAcrossAFormChange(t *testing.T) { + up := []catalogue.FormChange{{Name: "cache", Kind: catalogue.KeptKindState, Running: &catalogue.KeptLayout{Form: 1}, + Applied: &catalogue.KeptLayout{Form: 2, CatchUp: &catalogue.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": true}}}}} + down := []catalogue.FormChange{{Name: "cache", Kind: catalogue.KeptKindState, Running: &catalogue.KeptLayout{Form: 2}, + Applied: &catalogue.KeptLayout{Form: 1}}} + fp := func(c string) string { return fpOf(c) } + done := func(act string) Ending { return Ending{Act: act, Outcome: meshrecovery.EndedEventOutcomeDone} } + failed := func(act, why string) Ending { + return Ending{Act: act, Outcome: meshrecovery.EndedEventOutcomeFailed, Error: why} + } + applied := func(b *bed) bool { return len(b.appliedList()) == 2 } + heldHave := func(b *bed) bool { return b.haves.last("db").Phase == "held" } + + t.Run("a form that rises is taken as a restore point before the move, with the why and the catch-up, and then applied", func(t *testing.T) { + f := &recoveryFake{changes: up, answers: []fakeAnswer{{accepted: "a1", then: []Ending{done("a1")}}}} + moveDB(t, f, "a", "b", applied) + if got := f.seen(); !slices.Equal(got, []string{"snapshot-kept", "apply"}) { + t.Fatalf("calls %v", got) + } + want := fmt.Sprintf("snapshot-kept %s", asJSON(meshrecovery.SnapshotKeptInput{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("b"), + Why: "form change of db's cache to " + fp("b"), CatchUp: &meshrecovery.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": true}}})) + if f.asks[0] != want { + t.Fatalf("asked %s, want %s", f.asks[0], want) + } + }) + + t.Run("busy waits for the other act's ending and asks again, and a stopped act is asked again; another act's failure wakes nothing", func(t *testing.T) { + f := &recoveryFake{changes: up, answers: []fakeAnswer{ + {busy: "x1", then: []Ending{failed("zz", "cap"), done("x1")}}, + {accepted: "a2", then: []Ending{failed("a2", "stopped")}}, + {accepted: "a3", then: []Ending{done("a3")}}, + }} + moveDB(t, f, "a", "b", applied) + if got := f.seen(); !slices.Equal(got, []string{"snapshot-kept", "snapshot-kept", "snapshot-kept", "apply"}) { + t.Fatalf("calls %v", got) + } + }) + + for _, row := range []struct{ error, held string }{ + {"cancelled", "cancelled"}, + {"cap", "cap"}, + {"restore point damaged", "restore point damaged"}, + {"disk full", "disk full"}, + } { + t.Run("an ending failed with "+row.error+" holds the move and asks nothing again", func(t *testing.T) { + f := &recoveryFake{changes: up, answers: []fakeAnswer{{accepted: "a1", then: []Ending{failed("a1", row.error)}}}} + b := moveDB(t, f, "a", "b", heldHave) + if h := b.haves.last("db"); h.FP != fp("a") || !contains(h.Detail, row.held) || h.Wait == nil || h.Wait.Reason == "" { + t.Fatalf("the have %+v", h) + } + if got := f.seen(); len(got) != 1 { + t.Fatalf("asked again: %v", got) + } + if n := len(b.appliedList()); n != 1 { + t.Fatalf("applied %d", n) + } + }) + } + + t.Run("the bound passed says the wait in the have and the wait goes on to the ending", func(t *testing.T) { + f := &recoveryFake{changes: up, answers: []fakeAnswer{{accepted: "a1"}}} + b := moveDB(t, f, "a", "b", func(b *bed) bool { + b.haves.mu.Lock() + defer b.haves.mu.Unlock() + for _, h := range b.haves.log { + if h.Wait != nil && h.Wait.Words != "" && h.Phase == "backing-up" { + return true + } + } + return false + }, func(*bed, string) { close(f.expired) }) + f.events <- done("a1") + eventually(t, "the move went on after the ending", func() bool { return applied(b) }) + }) + + t.Run("a revert stops the newer build, restores the point of the form it reverts from, applies, then calls its catch-up", func(t *testing.T) { + f := &recoveryFake{changes: down, + points: []meshrecovery.Point{ + {Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("z"), Snapshot: "other", At: time.Now()}, + {Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("a"), Snapshot: "snap-1", At: time.Now(), CatchUp: &meshrecovery.CatchUp{Tool: "rebuild"}}}, + answers: []fakeAnswer{{accepted: "r1", then: []Ending{done("r1")}}}} + moveDB(t, f, "a", "b", func(b *bed) bool { return len(f.seen()) >= 5 }) + if got := f.seen(); !slices.Equal(got, []string{"stop", "kept-points", "restore-kept", "apply", "catch-up"}) { + t.Fatalf("calls %v", got) + } + if f.catchUp[0].Tool != "rebuild" { + t.Fatalf("catch-up %+v", f.catchUp) + } + want := fmt.Sprintf("restore-kept %s", asJSON(meshrecovery.RestoreKeptInput{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("a"), + Snapshot: "snap-1", Why: "revert of db's cache from " + fp("a")})) + if f.asks[0] != want { + t.Fatalf("asked %s, want %s", f.asks[0], want) + } + }) + + t.Run("a revert with no point of its form holds as a restore point missing", func(t *testing.T) { + f := &recoveryFake{changes: down} + b := moveDB(t, f, "a", "b", heldHave) + if h := b.haves.last("db"); !contains(h.Detail, "restore point missing") { + t.Fatalf("the have %+v", h) + } + }) + + t.Run("a restart waits for the act its have names and asks nothing again", func(t *testing.T) { + f := &recoveryFake{changes: up} + b := moveDB(t, f, "a", "b", applied, func(b *bed, dig string) { + at := time.Now().Add(-time.Minute) + b.haves.mu.Lock() + b.haves.stored["db"] = Have{Module: "db", FP: fp("a"), Phase: "backing-up", Wait: &nodeengine.Wait{ + On: "mesh-recovery.ended.db.cache", AskAt: &at, Act: "a1", Since: at}} + b.haves.mu.Unlock() + f.events <- done("a1") + }) + _ = b + if got := f.seen(); !slices.Equal(got, []string{"apply"}) { + t.Fatalf("calls %v", got) + } + }) +} + +func contains(s, sub string) bool { return strings.Contains(s, sub) } + +func asJSON(v any) string { raw, _ := json.Marshal(v); return string(raw) } diff --git a/internal/link/have.go b/internal/link/have.go index ed4f2f36..e4c6bcf1 100644 --- a/internal/link/have.go +++ b/internal/link/have.go @@ -409,6 +409,7 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) { if h.Phase == nodeengineclient.PhaseFailed { h.PullWait = f.PullWait } + h.Wait = f.Wait h.Health = s.healthOf(module, f) if h.Detail == "" { h.Detail = s.wordsOf(module, h.Health) diff --git a/internal/link/messages.go b/internal/link/messages.go index 16c32ebf..e84f31e6 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -647,4 +647,6 @@ type ModuleFact struct { Sealed compositions.Seals // PullWait is, on a failed fact, the pull from a store its apply waits on (nodeengine.HaveByNodeModuleValue.PullWait, issue 1966 ENG-9). PullWait *nodeengine.PullWait + // Wait is the hold or wait the module has, with the ask-at and act of a recovery wait (nodeengine.HaveByNodeModuleValue.Wait, design 47 G §560). + Wait *nodeengine.Wait } diff --git a/internal/link/recovery.go b/internal/link/recovery.go new file mode 100644 index 00000000..63be88bb --- /dev/null +++ b/internal/link/recovery.go @@ -0,0 +1,241 @@ +package link + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +// The node agent's wait for the recovery module (design 47 G §554, §560, A-77, A-92): the ending events of one module's kept data +// are heard through an ordered consumer that begins at the last message of the subject and follows the rest, and a wait's bound +// is a key of the state bounds with its own time to live, ended by the bus's marker. + +// BoundsBucket is the node-engine's state `bounds`, named by its holder as every state is. +const BoundsBucket = "mesh-host_bounds" + +// RecoveryBus is a link that hears the recovery module's events and keeps a bound. +type RecoveryBus interface { + // HearEnding calls heard with the last message of the subject on the stream, then each after it, until stop is called. + HearEnding(stream, subject string, heard func([]byte)) (stop func(), err error) + // BoundCreate writes the bound key with its time to live; one that stands is left as it is. BoundWatch calls ended once the + // bus's marker says the key's time ended; BoundDelete ends it. + BoundCreate(ctx context.Context, key string, value []byte, ttl time.Duration) error + BoundWatch(ctx context.Context, key string, ended func()) (stop func(), err error) + BoundDelete(ctx context.Context, key string) error +} + +func hearEnding(js nats.JetStreamContext, stream, subject string, heard func([]byte)) (func(), error) { + sub, err := js.Subscribe(subject, func(m *nats.Msg) { heard(m.Data) }, nats.OrderedConsumer(), nats.DeliverLast(), nats.BindStream(stream)) + if err != nil { + return nil, fmt.Errorf("the stream %s cannot be followed: %w", stream, err) + } + return func() { _ = sub.Unsubscribe() }, nil +} + +func boundKV(ctx context.Context, conn *nats.Conn) (jetstream.KeyValue, error) { + js, err := jetstream.New(conn) + if err != nil { + return nil, err + } + kv, err := js.KeyValue(ctx, BoundsBucket) + if err != nil { + return nil, fmt.Errorf("the state %s cannot be bound: %w", BoundsBucket, err) + } + return kv, nil +} + +func boundCreate(ctx context.Context, conn *nats.Conn, key string, value []byte, ttl time.Duration) error { + kv, err := boundKV(ctx, conn) + if err != nil { + return err + } + if _, err := kv.Create(ctx, key, value, jetstream.KeyTTL(ttl)); err != nil && !errors.Is(err, jetstream.ErrKeyExists) { + return err + } + return nil +} + +func boundWatch(ctx context.Context, conn *nats.Conn, key string, ended func()) (func(), error) { + kv, err := boundKV(ctx, conn) + if err != nil { + return nil, err + } + w, err := kv.Watch(ctx, key, jetstream.UpdatesOnly()) + if err != nil { + return nil, err + } + go func() { + for e := range w.Updates() { + if e != nil && e.Operation() == jetstream.KeyValuePurge { + ended() + } + } + }() + return func() { _ = w.Stop() }, nil +} + +func boundDelete(ctx context.Context, conn *nats.Conn, key string) error { + kv, err := boundKV(ctx, conn) + if err != nil { + return err + } + if err := kv.Delete(ctx, key); err != nil && !errors.Is(err, jetstream.ErrKeyNotFound) { + return err + } + return nil +} + +func (l *natsLink) HearEnding(stream, subject string, heard func([]byte)) (func(), error) { + return hearEnding(l.js, stream, subject, heard) +} +func (l *natsLink) BoundCreate(ctx context.Context, key string, value []byte, ttl time.Duration) error { + return boundCreate(ctx, l.conn, key, value, ttl) +} +func (l *natsLink) BoundWatch(ctx context.Context, key string, ended func()) (func(), error) { + return boundWatch(ctx, l.conn, key, ended) +} +func (l *natsLink) BoundDelete(ctx context.Context, key string) error { + return boundDelete(ctx, l.conn, key) +} + +func (b OverNATS) HearEnding(stream, subject string, heard func([]byte)) (func(), error) { + return hearEnding(b.JS, stream, subject, heard) +} +func (b OverNATS) BoundCreate(ctx context.Context, key string, value []byte, ttl time.Duration) error { + return boundCreate(ctx, b.Conn, key, value, ttl) +} +func (b OverNATS) BoundWatch(ctx context.Context, key string, ended func()) (func(), error) { + return boundWatch(ctx, b.Conn, key, ended) +} +func (b OverNATS) BoundDelete(ctx context.Context, key string) error { + return boundDelete(ctx, b.Conn, key) +} + +var ( + _ RecoveryBus = (*natsLink)(nil) + _ RecoveryBus = OverNATS{} +) + +func (q *Session) recoveryBus() (RecoveryBus, error) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + h, ok := bus.(RecoveryBus) + if bus == nil || !ok { + return nil, ErrNoLink + } + return h, nil +} + +// RecoveryEnding is the ending event of an act as the node agent reads it, the fields every ending carries. +type RecoveryEnding struct { + Act string `json:"act"` + Outcome string `json:"outcome"` + Error string `json:"error,omitempty"` + Snapshot string `json:"snapshot,omitempty"` +} + +// HearRecoveryEnding follows the events `ended` (kind "ended") or `restored` (kind "restored") of one module's kept data: the last +// said first, then each after it, until stop is called. +func (q *Session) HearRecoveryEnding(kind, module, kept string) (<-chan RecoveryEnding, func(), error) { + bus, err := q.recoveryBus() + if err != nil { + return nil, nil, err + } + stream, subject := meshrecovery.StreamEnded, meshrecovery.SubjectEnded(module, kept) + if kind == "restored" { + stream, subject = meshrecovery.StreamRestored, meshrecovery.SubjectRestored(module, kept) + } + out := make(chan RecoveryEnding, 64) + stop, err := bus.HearEnding(stream, subject, func(raw []byte) { + var ev RecoveryEnding + if json.Unmarshal(raw, &ev) == nil && ev.Act != "" { + select { + case out <- ev: + default: + } + } + }) + if err != nil { + return nil, nil, err + } + return out, stop, nil +} + +// KeepBound writes the bound key `.` of a wait with its time to live and watches it: expired is closed when the bus's +// marker says its time ended, and end deletes it. +func (q *Session) KeepBound(ctx context.Context, what, id, module string, since time.Time, ttl time.Duration) (<-chan struct{}, func(), error) { + bus, err := q.recoveryBus() + if err != nil { + return nil, nil, err + } + key := what + "." + id + value, err := json.Marshal(struct { + Module string `json:"module,omitempty"` + Since string `json:"since"` + V int64 `json:"v"` + }{module, since.UTC().Format(time.RFC3339), 1}) + if err != nil { + return nil, nil, err + } + expired := make(chan struct{}) + stopWatch, err := bus.BoundWatch(ctx, key, func() { + select { + case <-expired: + default: + close(expired) + } + }) + if err != nil { + return nil, nil, err + } + if err := bus.BoundCreate(ctx, key, value, ttl); err != nil { + stopWatch() + return nil, nil, err + } + return expired, func() { + stopWatch() + _ = bus.BoundDelete(context.WithoutCancel(ctx), key) + }, nil +} + +// CallTool calls a module's tool on this machine's own node tools with its arguments and answers its refusal as an error (design 47 G +// A-108: the catch-up of a restore point is called so, as the node-engine is granted to). +func (q *Session) CallTool(ctx context.Context, module, tool string, args map[string]any) error { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + asker, ok := bus.(ToolBus) + if bus == nil || !ok { + return ErrNoLink + } + if args == nil { + args = map[string]any{} + } + body, err := json.Marshal(args) + if err != nil { + return err + } + reply, err := asker.Ask(ctx, ToolSubject(module, tool, q.Membership.Node), body) + if err != nil { + return err + } + var env struct { + Error string `json:"error"` + } + if err := json.Unmarshal(reply, &env); err != nil { + return fmt.Errorf("its answer is not a tool's answer: %w", err) + } + if env.Error != "" { + return errors.New(env.Error) + } + return nil +} diff --git a/internal/selfcompose/entries.go b/internal/selfcompose/entries.go index 6b00c5b1..171d0092 100644 --- a/internal/selfcompose/entries.go +++ b/internal/selfcompose/entries.go @@ -15,6 +15,7 @@ import ( "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-host/internal/agent" ) @@ -514,3 +515,32 @@ func EntryCall(ask func(ctx context.Context, seat, verb string, args any) (json. // noEntryWords are the words the seat's holder refuses an unknown fingerprint in. const noEntryWords = "no entry of fingerprint" + +// FormChanges are the kept data whose form differs between the manifests of the build a module runs and the one a want moves it +// to, each read from the entry the composer holds of its fingerprint (design 47 G §296, §297). An entry not held is an error: the +// node agent then says it is not known and the move goes on. +func (c *Composer) FormChanges(_ context.Context, module, runningFp, appliedFp string) ([]catalogue.FormChange, error) { + if !c.byEntries() { + return nil, fmt.Errorf("this engine reads no entry by fingerprint") + } + read := func(fp string) (catalogue.Manifest, error) { + e, state := c.entryOf(fp) + if state != meshcatalogclient.EntryAnswered { + return catalogue.Manifest{}, fmt.Errorf("the entry of %s is %s", fp, state) + } + _, raw, err := entryManifest(e) + if err != nil { + return catalogue.Manifest{}, err + } + return catalogue.Decode(raw) + } + running, err := read(runningFp) + if err != nil { + return nil, fmt.Errorf("%s's running manifest: %w", module, err) + } + applied, err := read(appliedFp) + if err != nil { + return nil, fmt.Errorf("%s's wanted manifest: %w", module, err) + } + return catalogue.FormChanges(running, applied), nil +} diff --git a/module.json b/module.json index 3d95bb27..9d3b87bd 100644 --- a/module.json +++ b/module.json @@ -22,13 +22,18 @@ ], "invokes": [ "seat:mesh-catalog.entry", - "seat:mesh-catalog.seat-definitions" + "seat:mesh-catalog.seat-definitions", + "seat:mesh-recovery.kept-points", + "seat:mesh-recovery.restore-kept", + "seat:mesh-recovery.snapshot-kept" ], "consumes": [ "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-conditions.findings-asked", "mesh-delivery-view.settled", + "mesh-recovery.ended", + "mesh-recovery.restored", "node-engine.running", "restic.backed-up" ], @@ -60,6 +65,12 @@ } ], "config": { + "recovery-wait-minutes": { + "type": "number", + "description": "How long the node agent waits for the recovery module to end an act it asked before a module moves across a change of a kept data's form, or at a revert, in minutes, counted from its first ask; past it the have says the wait (design 47 G section 554).", + "minimum": 1, + "default": 90 + }, "operator-signers": { "type": "string", "description": "The operator's public keys, ed25519 in base64, one a line: the node applies a composition made by hand only when one of these signed it (ADR 0539 clause 3). Public keys, no secret. Empty trusts no operator key.", diff --git a/packaging/operator_keys_test.go b/packaging/operator_keys_test.go index 2c6ca2ca..d4387d04 100644 --- a/packaging/operator_keys_test.go +++ b/packaging/operator_keys_test.go @@ -62,12 +62,25 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) { } var m struct { Consumes []string `json:"consumes"` + Invokes []string `json:"invokes"` + Config map[string]struct { + Default any `json:"default"` + } `json:"config"` } if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) } + // Design 47 G §630, §633, A-77: the recovery module's three verbs it asks, and its bound as a config key of 90. + for _, v := range []string{"seat:mesh-recovery.kept-points", "seat:mesh-recovery.restore-kept", "seat:mesh-recovery.snapshot-kept"} { + if !slices.Contains(m.Invokes, v) { + t.Errorf("invokes %v lacks %s", m.Invokes, v) + } + } + if m.Config["recovery-wait-minutes"].Default != float64(90) { + t.Errorf("config recovery-wait-minutes is %+v, want default 90", m.Config["recovery-wait-minutes"]) + } // The backup event, the status view's settled, and the two events of the seat that serves the seat list (ADR 0636 §4), the keeper's ask (design 47 G §387) and the seat's running (§199). - if want := []string{"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-conditions.findings-asked", "mesh-delivery-view.settled", "node-engine.running", "restic.backed-up"}; !slices.Equal(m.Consumes, want) { + if want := []string{"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-conditions.findings-asked", "mesh-delivery-view.settled", "mesh-recovery.ended", "mesh-recovery.restored", "node-engine.running", "restic.backed-up"}; !slices.Equal(m.Consumes, want) { t.Fatalf("consumes %v, want %v", m.Consumes, want) } // 774: the acts bucket mesh-host_acts is not declared as state for now: a declared state of a module with no account -- 2.54.0 From ece79788a3138e18052a44f9f00cd90ad668f48b Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 00:21:10 +0100 Subject: [PATCH 22/59] A-112: the node-engine reads recovery-wait-minutes from a file its manifest declares, 90 when missing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The manifest declares the file recovery-wait (content ${config:recovery-wait-minutes}); the node-engine reads it at its start and does not call stdio.Config, which needs the runtime that launches a bundle. Builds: design 47 G §633, A-112 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- cmd/mesh-host/recovery.go | 18 ++++++++++++------ cmd/mesh-host/recovery_integration_test.go | 17 +++++++++++++++++ module.json | 7 +++++++ packaging/operator_keys_test.go | 3 +++ 4 files changed, 39 insertions(+), 6 deletions(-) diff --git a/cmd/mesh-host/recovery.go b/cmd/mesh-host/recovery.go index e7762514..9ce2acde 100644 --- a/cmd/mesh-host/recovery.go +++ b/cmd/mesh-host/recovery.go @@ -3,10 +3,12 @@ package main import ( "context" "encoding/json" + "os" + "strconv" + "strings" "time" "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" - stdio "git.novox.be/novox/mesh-sdk/go" "github.com/novox/mesh-host/internal/agent" "github.com/novox/mesh-host/internal/link" @@ -68,15 +70,19 @@ func recoveryEnv(session *link.Session, composer *selfcompose.Composer) *agent.R } } -// recoveryWaitMinutes is the manifest's config key recovery-wait-minutes, read through mesh-sdk's stdio.Config; its default is -// the design's 90 (design 47 G §633). +// recoveryWaitFile is the file the manifest declares with the content ${config:recovery-wait-minutes} (design 47 G §633, A-112): the +// node-engine is no bundle under a runtime, so it reads the setting from its file at its start, as the builder does (A-71). +var recoveryWaitFile = "/var/lib/mesh-host/recovery-wait-minutes" + +// recoveryWaitMinutes is the bound of the wait for the recovery module, in minutes: the file's number, the design's 90 when the +// file is missing, empty or not a positive number (design 47 G §554, A-112). func recoveryWaitMinutes() int { - raw, err := stdio.Config("recovery-wait-minutes") + raw, err := os.ReadFile(recoveryWaitFile) if err != nil { return agent.RecoveryWaitMinutes } - var n int - if json.Unmarshal(raw, &n) != nil || n <= 0 { + n, err := strconv.Atoi(strings.TrimSpace(string(raw))) + if err != nil || n <= 0 { return agent.RecoveryWaitMinutes } return n diff --git a/cmd/mesh-host/recovery_integration_test.go b/cmd/mesh-host/recovery_integration_test.go index 092e9090..2071af0d 100644 --- a/cmd/mesh-host/recovery_integration_test.go +++ b/cmd/mesh-host/recovery_integration_test.go @@ -5,6 +5,8 @@ package main import ( "context" "encoding/json" + "os" + "path/filepath" "testing" "time" @@ -78,6 +80,21 @@ func TestNatsTheRecoveryStepAsksHearsBoundsAndCatchesUpOverTheBus(t *testing.T) } t.Cleanup(func() { _ = toolSub.Unsubscribe() }) + // A-112: the bound is the number in the file the manifest declares, 90 where it is missing or not a number. + for content, want := range map[string]int{"45\n": 45, "": 90, "soon\n": 90, "0\n": 90} { + recoveryWaitFile = filepath.Join(t.TempDir(), "w") + if err := os.WriteFile(recoveryWaitFile, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + if got := recoveryWaitMinutes(); got != want { + t.Errorf("the file %q gives %d minutes, want %d", content, got, want) + } + } + recoveryWaitFile = filepath.Join(t.TempDir(), "missing") + if got := recoveryWaitMinutes(); got != 90 { + t.Errorf("no file gives %d minutes, want 90", got) + } + say("old", meshrecovery.RestoredEventOutcomeFailed) // said before anyone listens: the last message of the subject session := b.startSession() r := recoveryEnv(session, nil) diff --git a/module.json b/module.json index 9d3b87bd..74cdcb86 100644 --- a/module.json +++ b/module.json @@ -105,6 +105,13 @@ "path": "/var/lib/mesh-host/operator-keys", "mode": "0644", "content": "${config:operator-signers}\n" + }, + { + "id": "recovery-wait", + "type": "file", + "path": "/var/lib/mesh-host/recovery-wait-minutes", + "mode": "0644", + "content": "${config:recovery-wait-minutes}\n" } ] } diff --git a/packaging/operator_keys_test.go b/packaging/operator_keys_test.go index d4387d04..fdf38d77 100644 --- a/packaging/operator_keys_test.go +++ b/packaging/operator_keys_test.go @@ -76,6 +76,9 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) { t.Errorf("invokes %v lacks %s", m.Invokes, v) } } + if !strings.Contains(string(raw), `"content": "${config:recovery-wait-minutes}\n"`) { + t.Errorf("module.json declares no file with the content ${config:recovery-wait-minutes} (design 47 G A-112)") + } if m.Config["recovery-wait-minutes"].Default != float64(90) { t.Errorf("config recovery-wait-minutes is %+v, want default 90", m.Config["recovery-wait-minutes"]) } -- 2.54.0 From 6441ae7e969fe22a8f77d6fa20adc96eee36ebd3 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:01:44 +0100 Subject: [PATCH 23/59] gofmt cmd/mesh-host/keptartifact_test.go Builds: design 47 G A-95 (the owed gofmt) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- cmd/mesh-host/keptartifact_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/mesh-host/keptartifact_test.go b/cmd/mesh-host/keptartifact_test.go index b662f01b..b1ffc3f7 100644 --- a/cmd/mesh-host/keptartifact_test.go +++ b/cmd/mesh-host/keptartifact_test.go @@ -41,7 +41,9 @@ func TestAKeptArtifactIsFetchedThroughTheBuildSeatOfItsNode(t *testing.T) { t.Run(row.name, func(t *testing.T) { var seats []string fetch := keptArtifactFetch( - func(context.Context, string, string) (string, string, bool, error) { return "novox", "f1", row.kept, nil }, + func(context.Context, string, string) (string, string, bool, error) { + return "novox", "f1", row.kept, nil + }, func(_ context.Context, node, seat, verb string, _ any, _ time.Duration) (json.RawMessage, error) { if node != "novox" || verb != "kept-artifact" { t.Errorf("asked %s of %s on %s", verb, seat, node) -- 2.54.0 From 31697e696df4d9ef649970f03bf16839180deb22 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:01:44 +0100 Subject: [PATCH 24/59] A-113: a revert whose restore fails starts the newer build again; a form change not known waits; holds say wait-held MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G A-113, §297, §560, §541, §629 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- internal/agent/agent.go | 2 +- internal/agent/kept.go | 53 +++++++++++++++++++++++++++++++------ internal/agent/kept_test.go | 28 +++++++++++++++++++- internal/link/have.go | 40 ++++++++++++++++++++++++++++ internal/link/have_test.go | 11 ++++++++ 5 files changed, 124 insertions(+), 10 deletions(-) diff --git a/internal/agent/agent.go b/internal/agent/agent.go index ba620a77..77fee06f 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -1638,7 +1638,7 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo } // A kept data on the bus whose form changes is taken as a restore point first, or restored at a revert (design 47 G §296, §297). - held, catchUps := a.keptStep(ctx, module, w, comp, ranBefore) + held, catchUps := a.keptStep(ctx, module, w, comp, ranBefore, others) if held { return } diff --git a/internal/agent/kept.go b/internal/agent/kept.go index a4f38113..056728fa 100644 --- a/internal/agent/kept.go +++ b/internal/agent/kept.go @@ -10,6 +10,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" "git.novox.be/novox/mesh-sdk/go/catalogue" + "github.com/novox/mesh-host/internal/compositions" ) // A module's kept data on the bus, taken as a restore point before a form change moves it and restored at a revert (design 47 G @@ -206,7 +207,7 @@ func (a *Agent) askAndWait(ctx context.Context, h Have, kind string, k KeptMove, // keptStep takes the restore points a move of a module needs before it applies and restores the ones a revert needs: it // answers whether the module is held (its have says why, and the running build is kept) and, after a restore, the catch-up // of the point to call once the reverted build runs. A want that says to go on without a backup asks neither. -func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp compNow, ranBefore bool) (held bool, catchUps []meshrecovery.CatchUp) { +func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp compNow, ranBefore bool, others []compositions.Composed) (held bool, catchUps []meshrecovery.CatchUp) { r := a.env.Recovery if !ranBefore || r == nil || r.FormChanges == nil || w.Want.Backup == meshwants.WantsByWantNodeModuleValueBackupSkip { return false, nil @@ -225,10 +226,17 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com } changes, err := r.FormChanges(ctx, module, prev.Fp, fp) if err != nil { - // Not known whether the move changes a form: said, and the move goes on as before, since holding every module for a - // build record that is gone would stop deliveries a form never changed. - a.sayOnce("kept:"+module+":"+fp, fmt.Sprintf("%s: whether its kept data changes form between %s and %s is not known: %v", module, prev.Fp, fp, err)) - return false, nil + // Whether the move changes a form is not known: it waits for the entries and never moves without its restore point (design 47 G + // A-113). No attempt is kept, so the composer handing the composition again, or any settle, asks again. + words := fmt.Sprintf("whether %s's kept data changes form between %s and %s is not known, so it is not moved: %v", module, prev.Fp, fp, err) + a.sayOnce("kept:"+module+":"+fp, module+" waits: "+words) + since := r.now() + a.writeHave(ctx, Have{Module: module, FP: prev.Fp, Composed: prev.Composed, Phase: string(nodeengineclient.PhaseHeld), Detail: words, + Wait: &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepCompose, Reason: "entry-missing", Since: since, Words: words}}) + if a.env.Waiting != nil { + a.env.Waiting(module, fp) + } + return true, nil } take, restore := keptMoves(changes, prev.Fp, fp) if len(take)+len(restore) == 0 { @@ -265,16 +273,26 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com return false, nil } // A revert: the newer build stops first (it removes nothing), so the live data is not written while it is set aside. + stopped := false if a.env.Stop != nil { if res := a.env.Stop(ctx, module); res.Failed { hold("the newer build could not be stopped before the restore: "+res.Detail, false) return true, nil } + stopped = true + } + // A revert that cannot restore starts the newer build again and holds the move, saying why: a module is never left stopped + // (design 47 G A-113). + holdRevert := func(words string, interrupted bool) { + if stopped { + words += "; " + a.startAgain(ctx, module, prev, others) + } + hold(words, interrupted) } for _, m := range restore { pts, err := r.KeptPoints(ctx, meshrecovery.KeptPointsInput{Module: module, Kept: m.Name}) if err != nil { - hold(fmt.Sprintf("restore point missing: the points of %s's %s cannot be read: %v", module, m.Name, err), false) + holdRevert(fmt.Sprintf("restore point missing: the points of %s's %s cannot be read: %v", module, m.Name, err), false) return true, nil } var point *meshrecovery.Point @@ -287,7 +305,7 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com } } if point == nil { - hold(fmt.Sprintf("restore point missing: %s's %s has none of form %d taken before %s", module, m.Name, m.Form, m.Fingerprint), false) + holdRevert(fmt.Sprintf("restore point missing: %s's %s has none of form %d taken before %s", module, m.Name, m.Form, m.Fingerprint), false) return true, nil } why := fmt.Sprintf("revert of %s's %s from %s", module, m.Name, m.Fingerprint) @@ -297,7 +315,7 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com return got.Accepted, got.Busy, got.Error, err }) if held != nil { - hold(fmt.Sprintf("%s's %s (form %d) is not restored: %s", module, m.Name, m.Form, held.Words), held.Interrupted) + holdRevert(fmt.Sprintf("%s's %s (form %d) is not restored: %s", module, m.Name, m.Form, held.Words), held.Interrupted) return true, nil } if point.CatchUp != nil { @@ -307,6 +325,25 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com return false, catchUps } +// startAgain starts the build a module ran before a revert stopped it, from the composition the agent applied or holds of its +// fingerprint, and says whether it did (design 47 G A-113). +func (a *Agent) startAgain(ctx context.Context, module string, prev nodeengine.HaveByNodeModuleValue, others []compositions.Composed) string { + a.mu.Lock() + c, ok := a.applied[module] + if !ok || c.C.Module == "" { + c, ok = a.compositionFor(module, prev.Fp) + } + a.mu.Unlock() + if !ok || a.env.Apply == nil { + return "the newer build could not be started again: its composition is not held" + } + res := a.env.Apply(context.WithoutCancel(ctx), c.C, others) + if res.Failed { + return "the newer build could not be started again: " + res.Detail + } + return "the newer build runs again" +} + // catchUp calls, once the reverted build of a module runs, the catch-up of each restore point restored, as the module's own // tool through the tool runner (design 47 G A-108). A tool that fails is said: the build runs, and the tool can be called by hand. func (a *Agent) catchUp(ctx context.Context, module string, calls []meshrecovery.CatchUp) { diff --git a/internal/agent/kept_test.go b/internal/agent/kept_test.go index 5c2f571b..144c5f86 100644 --- a/internal/agent/kept_test.go +++ b/internal/agent/kept_test.go @@ -27,6 +27,7 @@ type recoveryFake struct { calls []string // stop, ask, apply, catch-up, in order catchUp []meshrecovery.CatchUp expired chan struct{} + unknown error // FormChanges answers it } type fakeAnswer struct { @@ -52,7 +53,7 @@ func (f *recoveryFake) next(verb string, in any) (string, string, string) { func (f *recoveryFake) env() *Recovery { return &Recovery{ - FormChanges: func(context.Context, string, string, string) ([]catalogue.FormChange, error) { return f.changes, nil }, + FormChanges: func(context.Context, string, string, string) ([]catalogue.FormChange, error) { return f.changes, f.unknown }, SnapshotKept: func(_ context.Context, in meshrecovery.SnapshotKeptInput) (meshrecovery.SnapshotKeptAnswer, error) { a, b, e := f.next("snapshot-kept", in) return meshrecovery.SnapshotKeptAnswer{Accepted: a, Busy: b, Error: e}, nil @@ -235,6 +236,31 @@ func testAMoveAcrossAFormChange(t *testing.T) { } }) + t.Run("a revert whose restore fails starts the newer build again and holds the move, saying why (A-113)", func(t *testing.T) { + f := &recoveryFake{changes: down, + points: []meshrecovery.Point{{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("a"), Snapshot: "snap-1", At: time.Now()}}, + answers: []fakeAnswer{{accepted: "r1", then: []Ending{failed("r1", "restore point damaged")}}}} + b := moveDB(t, f, "a", "b", heldHave) + if got := f.seen(); !slices.Equal(got, []string{"stop", "kept-points", "restore-kept", "apply"}) { + t.Fatalf("calls %v", got) + } + if h := b.haves.last("db"); !contains(h.Detail, "restore point damaged") || !contains(h.Detail, "the newer build runs again") || + h.Wait == nil || h.Wait.Step != nodeengine.WaitStepCompose { + t.Fatalf("the have %+v", h) + } + }) + + t.Run("a form change not known moves nothing and waits for the entries (A-113)", func(t *testing.T) { + f := &recoveryFake{unknown: fmt.Errorf("the entry is asked")} + b := moveDB(t, f, "a", "b", heldHave) + if h := b.haves.last("db"); h.FP != fp("a") || !contains(h.Detail, "is not known") || h.Wait == nil || h.Wait.Reason != "entry-missing" { + t.Fatalf("the have %+v", h) + } + if n := len(b.appliedList()); n != 1 || len(f.seen()) != 0 { + t.Fatalf("applied %d, calls %v", n, f.seen()) + } + }) + t.Run("a restart waits for the act its have names and asks nothing again", func(t *testing.T) { f := &recoveryFake{changes: up} b := moveDB(t, f, "a", "b", applied, func(b *bed, dig string) { diff --git a/internal/link/have.go b/internal/link/have.go index e4c6bcf1..5d190a18 100644 --- a/internal/link/have.go +++ b/internal/link/have.go @@ -69,6 +69,9 @@ type haveState struct { userListNoKey map[string]bool // runningSaid is, by module, the fingerprint and manifest its event running last said, so each is said once. runningSaid map[string]string + // waitSaid is, by module, the wait its event wait-held last said (since, step, reason), so each is said once and its end + // is said when the have no longer has it (design 47 G §541, §629, A-113). + waitSaid map[string]nodeengine.Wait // unjudged is each module whose composition moved (an apply began, or the fact names another build) and that no look // has judged since it stopped moving: its health is not yet judged, never the last look's of what ran before. unjudged map[string]bool @@ -440,6 +443,13 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) { if ev, ok := s.runningToSay(module, h, q.manifestOf()); ok { said = append(said, func() { q.sayRunning(ev) }) } + if held, ended, ok := s.waitToSay(module, h.Wait); ok { + if h.Wait != nil { + said = append(said, func() { q.sayWait(nodeengine.EventWaitHeld, nodeengine.SubjectWaitHeld(node, module), held) }) + } else { + said = append(said, func() { q.sayWait(nodeengine.EventWaitEnded, nodeengine.SubjectWaitEnded(node, module), ended) }) + } + } } if !q.writeRemoved(write, bucket, node, &said) { return @@ -931,6 +941,36 @@ func (s *haveState) runningToSay(module string, h nodeengine.HaveByNodeModuleVal return nodeengine.Running{V: sdkkeys.Version, Module: module, Fingerprint: h.Fp, Manifest: digest}, true } +// waitToSay is the event wait-held for a have written with a wait the last one did not say, or wait-ended for one written with +// none after a wait said (design 47 G §541, §629): a wait of the same step, reason and since is said once. Under s.mu. +func (s *haveState) waitToSay(module string, w *nodeengine.Wait) (held, ended nodeengine.Wait, ok bool) { + last, said := s.waitSaid[module] + if w == nil { + if !said { + return held, ended, false + } + delete(s.waitSaid, module) + return held, last, true + } + if said && last.Step == w.Step && last.Reason == w.Reason && last.Since.Equal(w.Since) && last.Words == w.Words { + return held, ended, false + } + if s.waitSaid == nil { + s.waitSaid = map[string]nodeengine.Wait{} + } + s.waitSaid[module] = *w + return *w, ended, true +} + +// sayWait says the event wait-held or wait-ended of a wait, with the time its bound counts from. +func (q *Session) sayWait(event, subject string, w nodeengine.Wait) { + if w.ClockFrom == nil { + since := w.Since + w.ClockFrom = &since + } + q.sayEngineOn(subject, event, w) +} + // sayRunning says the event running, on its subject `running..`. func (q *Session) sayRunning(ev nodeengine.Running) { ev.Node, ev.At = q.Membership.Node, time.Now().UTC() diff --git a/internal/link/have_test.go b/internal/link/have_test.go index 7a76f716..4c2ceb52 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -704,6 +704,17 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth( !strings.Contains(got[0], `"fingerprint":"`+fp+`"`) || !strings.Contains(got[0], `"manifest":"sha256:`+strings.Repeat("c", 64)+`"`) { t.Fatalf("running said %v, want one for the healthy build", got) } + // A have with a wait says wait-held once on wait-held.., and wait-ended when the wait is gone (design 47 G §541, A-113). + hold := &nodeengine.Wait{Owner: nodeengine.Seat, Step: nodeengine.WaitStepCompose, Reason: "restore point missing", Since: time.Now().UTC(), Words: "w"} + q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig, Wait: hold}) + q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig, Wait: hold}) + if got := l.saidOn("mesh.seat.node-engine.event.wait-held.n1.web "); len(got) != 1 || !strings.Contains(got[0], `"reason":"restore point missing"`) || !strings.Contains(got[0], `"clock-from"`) { + t.Fatalf("wait-held said %v, want one", got) + } + q.NoteModule(context.Background(), ModuleFact{Module: "web", FP: fp, Composed: dig}) + if got := l.saidOn("mesh.seat.node-engine.event.wait-ended.n1.web "); len(got) != 1 { + t.Fatalf("wait-ended said %v, want one", got) + } // Starting: it waits. Past its start window the check fails: failed. q.SayHealth(context.Background(), *healthOf(map[string]string{"web": StateStarting})) if got := l.have(t, "n1.web"); got.Health != nodeengineclient.HealthWaits || got.Fp != fp || got.Composed != dig { -- 2.54.0 From 0dd32d9722815b3f2094ed43c1b0729db446d792 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:03:22 +0100 Subject: [PATCH 25/59] A-101: the node-engine reads the router's record through mesh-sdk's operatorask, the bucket from the seat list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G A-101, §700, A-74 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- internal/agent/kept_test.go | 4 +++- internal/link/engineseat.go | 13 +++++-------- internal/link/have_test.go | 26 ++++++++++++++++++++++++++ internal/link/session.go | 16 +++++++++++++++- module.json | 1 + 5 files changed, 50 insertions(+), 10 deletions(-) diff --git a/internal/agent/kept_test.go b/internal/agent/kept_test.go index 144c5f86..9578891b 100644 --- a/internal/agent/kept_test.go +++ b/internal/agent/kept_test.go @@ -53,7 +53,9 @@ func (f *recoveryFake) next(verb string, in any) (string, string, string) { func (f *recoveryFake) env() *Recovery { return &Recovery{ - FormChanges: func(context.Context, string, string, string) ([]catalogue.FormChange, error) { return f.changes, f.unknown }, + FormChanges: func(context.Context, string, string, string) ([]catalogue.FormChange, error) { + return f.changes, f.unknown + }, SnapshotKept: func(_ context.Context, in meshrecovery.SnapshotKeptInput) (meshrecovery.SnapshotKeptAnswer, error) { a, b, e := f.next("snapshot-kept", in) return meshrecovery.SnapshotKeptAnswer{Accepted: a, Busy: b, Error: e}, nil diff --git a/internal/link/engineseat.go b/internal/link/engineseat.go index 88e01c1f..f63404b1 100644 --- a/internal/link/engineseat.go +++ b/internal/link/engineseat.go @@ -236,14 +236,10 @@ func sayOnce(publish func(id string) error, subject string, raw []byte) error { // 0563 §3 changes it): its ask and cancel on the seat's work stream, its warrants heard on its own decided subject, and // its asks' state read from the router's records under its own name. -// RouterRecords is the router's bucket of asks, which each asker reads under its own name: the state `asks` of the -// operator-channel's holder, the messenger (mesh-catalog modules/messenger). -const RouterRecords = "messenger_asks" - // OperatorAsks is a link that asks the operator as this node's engine. Optional. type OperatorAsks interface { PublishAsk(subject string, body any, id string) error - RouterRecord(key string) (*stdio.StateEntry, error) + RouterRecord(bucket, key string) (*stdio.StateEntry, error) HearWarrants(asker string, heard func(oc.Warrant)) error } @@ -259,11 +255,12 @@ func (l *natsLink) PublishAsk(subject string, body any, id string) error { return err } -// RouterRecord reads one key of the router's records, `.`; nil when it holds none. -func (l *natsLink) RouterRecord(key string) (*stdio.StateEntry, error) { +// RouterRecord reads one key of the router's records, `.`, in the bucket the seat list names (mesh-sdk's operatorask); nil +// when it holds none. +func (l *natsLink) RouterRecord(bucket, key string) (*stdio.StateEntry, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - msg, err := l.conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+RouterRecords+".$KV."+RouterRecords+"."+key, nil) + msg, err := l.conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+bucket+".$KV."+bucket+"."+key, nil) if err != nil { return nil, err } diff --git a/internal/link/have_test.go b/internal/link/have_test.go index 4c2ceb52..084557bf 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -7,6 +7,8 @@ import ( "fmt" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + stdio "git.novox.be/novox/mesh-sdk/go" sdkkeys "git.novox.be/novox/mesh-sdk/go/keys" "github.com/novox/mesh-host/internal/compositions" "slices" @@ -46,6 +48,8 @@ type haveLink struct { said []string // onSaid are the events said on a subject of their own (running), by subject, in order. onSaid []string + // asked are the subjects asked of the bus, records the router's records read as bucket and key. + asked, records []string } func (l *haveLink) SayEngineOn(subject string, body any) error { @@ -97,6 +101,23 @@ type haveEntry struct { rev uint64 } +// The link as the operator's asker and a tool bus: the seat list answers the holder of the seat operator-channel, and the +// router's records are read from the bucket named for it (design 47 G A-101). +func (l *haveLink) Ask(_ context.Context, subject string, _ []byte) ([]byte, error) { + l.hmu.Lock() + l.asked = append(l.asked, subject) + l.hmu.Unlock() + return []byte(`{"result":{"seats":[{"seat":"operator-channel","holders":[{"module":"the-router"}]}]}}`), nil +} +func (l *haveLink) PublishAsk(string, any, string) error { return nil } +func (l *haveLink) HearWarrants(string, func(oc.Warrant)) error { return nil } +func (l *haveLink) RouterRecord(bucket, key string) (*stdio.StateEntry, error) { + l.hmu.Lock() + l.records = append(l.records, bucket+" "+key) + l.hmu.Unlock() + return nil, nil +} + func newHaveLink() *haveLink { return &haveLink{quietLink: newQuietLink(), kv: map[string]haveEntry{}} } func (l *haveLink) Health(context.Context, string, []byte) error { return nil } @@ -715,6 +736,11 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth( if got := l.saidOn("mesh.seat.node-engine.event.wait-ended.n1.web "); len(got) != 1 { t.Fatalf("wait-ended said %v, want one", got) } + // The router's record of an ask is read from the bucket of the seat's holder on the seat list (design 47 G A-101). + if _, err := q.RouterRecord("engine.a1"); err != nil || !slices.Equal(l.records, []string{"the-router_asks engine.a1"}) || + !strings.Contains(strings.Join(l.asked, " "), "assignments") { + t.Fatalf("the router's record read as %v after asking %v: %v", l.records, l.asked, err) + } // Starting: it waits. Past its start window the check fails: failed. q.SayHealth(context.Background(), *healthOf(map[string]string{"web": StateStarting})) if got := l.have(t, "n1.web"); got.Health != nodeengineclient.HealthWaits || got.Fp != fp || got.Composed != dig { diff --git a/internal/link/session.go b/internal/link/session.go index 59bfb062..8bb427f5 100644 --- a/internal/link/session.go +++ b/internal/link/session.go @@ -12,9 +12,12 @@ import ( "sync" "time" + "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/health" + "git.novox.be/novox/mesh-sdk/go/operatorask" ) // Session is the node-engine's hold on the bus (novox/hq ADR 0539 §9): the link open now, what is asked of this @@ -403,7 +406,18 @@ func (q *Session) RouterRecord(key string) (*stdio.StateEntry, error) { if err != nil { return nil, err } - return asking.RouterRecord(key) + // The bucket is the one of the seat's holder on the seat list, found by mesh-sdk's operatorask (design 47 G A-101). + bucket, err := operatorask.Bucket(func(call string, body any) (json.RawMessage, error) { + ctx, cancel := context.WithTimeout(context.Background(), VaultWithin) + defer cancel() + return q.AskMeshSeat(ctx, meshmoduleassigner.Seat, meshmoduleassigner.VerbAssignments, body) + }, operatorask.Contract{Lists: "seat:" + meshmoduleassigner.Seat + "." + meshmoduleassigner.VerbAssignments, + ListBody: meshmoduleassigner.AssignmentsInput{Seats: meshmoduleassigner.AssignmentsInputSeatsTrue}, + Seat: occ.Seat, StateAsks: occ.StateAsks}) + if err != nil { + return nil, err + } + return asking.RouterRecord(bucket, key) } // StatedReads is what this engine states it reads of a manifest and when it made that statement, at its start or at the diff --git a/module.json b/module.json index 29d7ff97..c66f253f 100644 --- a/module.json +++ b/module.json @@ -42,6 +42,7 @@ "invokes": [ "seat:mesh-catalog.entry", "seat:mesh-catalog.seat-definitions", + "seat:mesh-module-assigner.assignments", "seat:mesh-recovery.kept-points", "seat:mesh-recovery.restore-kept", "seat:mesh-recovery.snapshot-kept" -- 2.54.0 From 0f108fd8ac6e01f41b66d166e676b475ec37f8dd Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:03:54 +0100 Subject: [PATCH 26/59] internal/apply: a test of the places a module is placed in follows no link on the machine it runs on Builds: design 47 G A-95 (the owed hang of TestAModulesOwnPlacesAreStillPlaced on real mounts) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- internal/apply/placement_guard.go | 6 +++++- internal/apply/placement_guard_test.go | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/internal/apply/placement_guard.go b/internal/apply/placement_guard.go index 248dda54..09662a6e 100644 --- a/internal/apply/placement_guard.go +++ b/internal/apply/placement_guard.go @@ -386,11 +386,15 @@ func below(path, dir string) bool { // atOrBelow says whether path is dir or below it. func atOrBelow(path, dir string) bool { return path == dir || below(path, dir) } +// evalSymlinks follows the links of an existing path; a variable so a test judges paths whose machine it does not touch (a mount +// of the machine it runs on may not answer). +var evalSymlinks = filepath.EvalSymlinks + // resolved is a path with every link in it followed, as far as the path exists, and the rest as declared. func resolved(path string) string { rest := "" for p := path; ; p = filepath.Dir(p) { - if real, err := filepath.EvalSymlinks(p); err == nil { + if real, err := evalSymlinks(p); err == nil { return filepath.Clean(filepath.Join(real, rest)) } if filepath.Dir(p) == p { diff --git a/internal/apply/placement_guard_test.go b/internal/apply/placement_guard_test.go index 8bff411d..869dddd9 100644 --- a/internal/apply/placement_guard_test.go +++ b/internal/apply/placement_guard_test.go @@ -178,6 +178,11 @@ func TestAFoundDirectoryIsUsedAsFound(t *testing.T) { // The module's own directories keep being placed: below /var/lib, below /etc as root's, a container's mount of // the machine's docker socket. func TestAModulesOwnPlacesAreStillPlaced(t *testing.T) { + // The paths below are judged as written: no link of them is followed on the machine the test runs on, whose mounts of them + // (a share that does not answer) it does not depend on. + was := evalSymlinks + evalSymlinks = func(p string) (string, error) { return p, nil } + t.Cleanup(func() { evalSymlinks = was }) for _, path := range []string{"/var/lib/notes/data", "/etc/systemd/logind.conf.d", "/var/run/fail2ban", "/srv/notes", "/mnt/plex/config", "/storage/media", "/var/lib/mesh/registry"} { if err := refusePath(path, placingDirectory, "notes", ""); err != nil { -- 2.54.0 From b3b13f2ae1a705f2772cedf3975e5605344c4901 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:06:39 +0100 Subject: [PATCH 27/59] =?UTF-8?q?G=20=C2=A7560:=20a=20cancelled=20recovery?= =?UTF-8?q?=20act=20holds=20the=20move=20with=20the=20hand=20act's=20why?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G A-117, §560 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- internal/agent/kept.go | 8 +++++++- internal/agent/kept_test.go | 8 ++++++-- internal/link/recovery.go | 2 ++ 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/internal/agent/kept.go b/internal/agent/kept.go index 056728fa..19bcebe9 100644 --- a/internal/agent/kept.go +++ b/internal/agent/kept.go @@ -26,6 +26,8 @@ type Ending struct { Outcome string Error string Snapshot string + // Why is the hand act's words for a cancel (design 47 G A-117, §560). + Why string } // RecoveryWaitMinutes is the default of the manifest's config key `recovery-wait-minutes`, the bound of the wait for an @@ -191,7 +193,11 @@ func (a *Agent) askAndWait(ctx context.Context, h Have, kind string, k KeptMove, case ev.Error == "stopped": act, wait.Act = "", "" case ev.Error == "cancelled": - return ev, &recoveryHeld{Words: "cancelled"} + words := "cancelled" + if ev.Why != "" { + words += ": " + ev.Why + } + return ev, &recoveryHeld{Words: words} default: words := ev.Error if words == "" { diff --git a/internal/agent/kept_test.go b/internal/agent/kept_test.go index 9578891b..31b0272c 100644 --- a/internal/agent/kept_test.go +++ b/internal/agent/kept_test.go @@ -143,7 +143,11 @@ func testAMoveAcrossAFormChange(t *testing.T) { fp := func(c string) string { return fpOf(c) } done := func(act string) Ending { return Ending{Act: act, Outcome: meshrecovery.EndedEventOutcomeDone} } failed := func(act, why string) Ending { - return Ending{Act: act, Outcome: meshrecovery.EndedEventOutcomeFailed, Error: why} + e := Ending{Act: act, Outcome: meshrecovery.EndedEventOutcomeFailed, Error: why} + if why == "cancelled" { + e.Why = "the operator stopped it" // the hand act's words (design 47 G A-117) + } + return e } applied := func(b *bed) bool { return len(b.appliedList()) == 2 } heldHave := func(b *bed) bool { return b.haves.last("db").Phase == "held" } @@ -174,7 +178,7 @@ func testAMoveAcrossAFormChange(t *testing.T) { }) for _, row := range []struct{ error, held string }{ - {"cancelled", "cancelled"}, + {"cancelled", "cancelled: the operator stopped it"}, {"cap", "cap"}, {"restore point damaged", "restore point damaged"}, {"disk full", "disk full"}, diff --git a/internal/link/recovery.go b/internal/link/recovery.go index 63be88bb..8bfde8f0 100644 --- a/internal/link/recovery.go +++ b/internal/link/recovery.go @@ -140,6 +140,8 @@ type RecoveryEnding struct { Outcome string `json:"outcome"` Error string `json:"error,omitempty"` Snapshot string `json:"snapshot,omitempty"` + // Why is the hand act's words for a cancel (design 47 G A-117, §560). + Why string `json:"why,omitempty"` } // HearRecoveryEnding follows the events `ended` (kind "ended") or `restored` (kind "restored") of one module's kept data: the last -- 2.54.0 From 054ca5a6b3a734ac9e99b9b2c6312b4690d8c7f1 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:07:53 +0100 Subject: [PATCH 28/59] =?UTF-8?q?G=20=C2=A7614:=20the=20node-engine=20writ?= =?UTF-8?q?es=20the=20finding=20entry-not-found=20for=20a=20fingerprint=20?= =?UTF-8?q?the=20catalog=20half=20holds=20none=20of?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G A-117, §614, A-15, A-95 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- cmd/mesh-host/nodeagent.go | 11 ++++++++-- cmd/mesh-host/seatlist.go | 2 +- internal/seatwait/entrynotfound.go | 33 ++++++++++++++++++++++++++++ internal/seatwait/seatwait_test.go | 19 ++++++++++++++++ internal/selfcompose/entries.go | 24 ++++++++++++++++++++ internal/selfcompose/entries_test.go | 9 ++++++++ internal/selfcompose/selfcompose.go | 9 ++++++-- 7 files changed, 102 insertions(+), 5 deletions(-) create mode 100644 internal/seatwait/entrynotfound.go diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index de5b7c8d..0290447d 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -620,6 +620,11 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus marks := newMarker(run) heldFindings := &seatwait.HeldKeeper{Node: node, Observer: findingsModule, Writer: findings, Ctx: run, Say: say} hooks := composerHooks{ + entryNotFound: func(fp string, notFound bool) { + if err := seatwait.SayEntryNotFound(run, findings, node, findingsModule, fp, notFound, time.Now); err != nil { + say("the finding that the catalog half holds no entry of " + fp + " cannot be kept: " + err.Error()) + } + }, seatList: seats, seatListFailed: err != nil, running: func() []compositions.Composed { @@ -1111,7 +1116,9 @@ type composerHooks struct { unseal func(string) ([]byte, error) refused func(why string, held map[string]string) kept func(kept map[string][]string) - held func(module string) (compositions.Seals, []string, error) + // entryNotFound is selfcompose.Env's EntryNotFound. + entryNotFound func(fp string, notFound bool) + held func(module string) (compositions.Seals, []string, error) // vaultAuthority and certify are selfcompose.Env's VaultAuthority and Certify. vaultAuthority func() string certify func(ctx context.Context) error @@ -1173,7 +1180,7 @@ func composerEnv(session *link.Session, node string, say func(string), hooks com Hear: session.Hear, // And a moved module's templates archive, in the artifact store, as every archive is fetched (issue 1126). Fetch: apply.Fetch, - Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held, Kept: hooks.kept, + Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held, Kept: hooks.kept, EntryNotFound: hooks.entryNotFound, VaultAuthority: hooks.vaultAuthority, Certify: hooks.certify} if hooks.seatList != nil { env.SeatList, env.SeatListChanged = hooks.seatList.reader, hooks.seatList.changed diff --git a/cmd/mesh-host/seatlist.go b/cmd/mesh-host/seatlist.go index c06f5a9f..7461d625 100644 --- a/cmd/mesh-host/seatlist.go +++ b/cmd/mesh-host/seatlist.go @@ -125,7 +125,7 @@ func newFindingsOfTheNode(session *link.Session, node string, say func(string)) Module: findingsModule, Node: node, State: link.FindingsState{Session: session}, - Kinds: []string{seatwait.Kind, seatwait.HeldKind}, + Kinds: []string{seatwait.Kind, seatwait.HeldKind, seatwait.EntryNotFoundKind}, Asked: condseat.SubjectFindingsAsked, Logf: func(format string, args ...any) { say(fmt.Sprintf(format, args...)) }, Publish: session.PublishEvent, diff --git a/internal/seatwait/entrynotfound.go b/internal/seatwait/entrynotfound.go new file mode 100644 index 00000000..2f9e37ba --- /dev/null +++ b/internal/seatwait/entrynotfound.go @@ -0,0 +1,33 @@ +package seatwait + +import ( + "context" + "fmt" + "time" + + condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + "git.novox.be/novox/mesh-sdk/go/keys" +) + +// EntryNotFoundKind is the finding a part writes when the catalog half holds no entry of a fingerprint the part holds a want or a +// running for (design 47 G §614, A-15, A-117): scope module, its id the fingerprint escaped as a name. +const EntryNotFoundKind = "entry-not-found" + +// EntryNotFoundID is the finding's id for a fingerprint. +func EntryNotFoundID(fp string) string { return keys.EscapeName(fp) } + +// SayEntryNotFound writes the finding for fp when notFound, and clears it when no longer: the entry answered later, or nothing the +// node holds names the fingerprint any more. A write or a clear that fails is returned for the caller to say. +func SayEntryNotFound(ctx context.Context, w Writer, node, observer, fp string, notFound bool, now func() time.Time) error { + id := EntryNotFoundID(fp) + if !notFound { + return w.Clear(ctx, scopeModule, id, EntryNotFoundKind, "the entry is answered, or nothing the node holds names its fingerprint") + } + return w.Write(ctx, condseat.Finding{Observer: observer, Kind: EntryNotFoundKind, Scope: scopeModule, ID: id, + Severity: condseat.FindingSeverityWarning, Source: EntryNotFoundKind, + Subject: condseat.Subject{Scope: scopeModule, ID: observer, Machine: node}, + Summary: fmt.Sprintf("the catalog half holds no entry of the build %s, so %s composes nothing new of it", fp, node), + Headline: fmt.Sprintf("no entry of the build %s on %s", fp, node), + Explanation: "A want or a running names this build and the catalog half answered that it holds no entry of it. The finding clears when the entry is answered or nothing the node holds names the build.", + At: now()}) +} diff --git a/internal/seatwait/seatwait_test.go b/internal/seatwait/seatwait_test.go index 80f0a534..31f3a4a7 100644 --- a/internal/seatwait/seatwait_test.go +++ b/internal/seatwait/seatwait_test.go @@ -3,10 +3,12 @@ package seatwait import ( "context" "errors" + "strings" "testing" "time" condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/seatlist" ) @@ -214,6 +216,22 @@ func caseAHeldModuleIsAFindingClearedWhenItIsComposedOrNoLongerWanted(t *testing } } +// Design 47 G §614, A-117: the finding entry-not-found is of scope module with the fingerprint, escaped, as its id, and cleared by it. +func caseAnEntryNotFoundIsAFindingOfTheFingerprint(t *testing.T) { + w := &fakeFindings{} + fp := "src2:" + strings.Repeat("a", 64) + now := func() time.Time { return time.Unix(1, 0) } + if err := SayEntryNotFound(context.Background(), w, "n1", "mesh-host", fp, true, now); err != nil { + t.Fatal(err) + } + if len(w.put) != 1 || w.put[0].Kind != EntryNotFoundKind || w.put[0].Scope != "module" || w.put[0].ID != keys.EscapeName(fp) || strings.Contains(w.put[0].ID, ":") { + t.Fatalf("written %+v", w.put) + } + if err := SayEntryNotFound(context.Background(), w, "n1", "mesh-host", fp, false, now); err != nil || len(w.cleared) != 1 || w.cleared[0].ID != w.put[0].ID { + t.Fatalf("cleared %+v, %v", w.cleared, err) + } +} + // ADR 0636 §1, the node's finding of a seat list unread. func TestTheSeatListWait(t *testing.T) { for name, run := range map[string]func(*testing.T){ @@ -221,6 +239,7 @@ func TestTheSeatListWait(t *testing.T) { "a read within the bound writes nothing": caseAReadWithinTheBoundWritesNothing, "a finding that cannot be written is said and tried again": caseAFindingThatCannotBeWrittenIsSaidAndTriedAgain, "a finding of an earlier process is cleared at the first full read": caseAFindingOfAnEarlierProcessIsClearedAtTheFirstFullRead, + "an entry not found is a finding of the fingerprint": caseAnEntryNotFoundIsAFindingOfTheFingerprint, "a held module is a finding cleared when it is composed or no longer wanted": caseAHeldModuleIsAFindingClearedWhenItIsComposedOrNoLongerWanted, } { t.Run(name, run) diff --git a/internal/selfcompose/entries.go b/internal/selfcompose/entries.go index 171d0092..24e0ab1a 100644 --- a/internal/selfcompose/entries.go +++ b/internal/selfcompose/entries.go @@ -153,9 +153,32 @@ func (c *Composer) entryChanged(row meshcatalogclient.EntryRow) { c.mu.Unlock() c.keep(entryKey(row.Fingerprint), meshcatalog.EntryAnswer{Entry: *row.Entry}) } + c.tellNotFound(row.Fingerprint, row.State == meshcatalogclient.EntryNotFound) c.askRehand() } +// tellNotFound tells the finding entry-not-found once when fp's entry is not found, and once when it no longer is (design 47 G §614, A-117). +func (c *Composer) tellNotFound(fp string, notFound bool) { + c.mu.Lock() + was := c.notFound[fp] + if was == notFound { + c.mu.Unlock() + return + } + if c.notFound == nil { + c.notFound = map[string]bool{} + } + if notFound { + c.notFound[fp] = true + } else { + delete(c.notFound, fp) + } + c.mu.Unlock() + if c.env.EntryNotFound != nil { + c.env.EntryNotFound(fp, notFound) + } +} + // keep writes one value into the composition state; a state that cannot be written is said, and the value is kept again by // the next pass. func (c *Composer) keep(key string, v any) { @@ -236,6 +259,7 @@ func (c *Composer) syncEntries(ctx context.Context) { for _, row := range c.entries.Rows() { if _, named := need[row.Fingerprint]; !named { c.entries.Release(row.Fingerprint) + c.tellNotFound(row.Fingerprint, false) } } c.mu.Lock() diff --git a/internal/selfcompose/entries_test.go b/internal/selfcompose/entries_test.go index 7136b509..829659ae 100644 --- a/internal/selfcompose/entries_test.go +++ b/internal/selfcompose/entries_test.go @@ -58,6 +58,8 @@ func composeFromEntries(t *testing.T) { env := b.env() env.Entry, env.ComposedState = EntryCall(ask), state env.Refused = func(_ string, h map[string]string) { mu.Lock(); held = h; mu.Unlock() } + told := map[string]bool{} + env.EntryNotFound = func(fp string, notFound bool) { mu.Lock(); told[fp] = notFound; mu.Unlock() } c := New(env) out, err := c.Watch(ctx, "n1.>") if err != nil { @@ -97,6 +99,13 @@ func composeFromEntries(t *testing.T) { if !strings.HasPrefix(why, agent.EntryMissing) || !strings.Contains(why, ghost) || held["lemurs"] != "" { t.Errorf("held %q, want ghost held with %s naming %s", held, agent.EntryMissing, ghost) } + // And the finding entry-not-found is told for the fingerprint the half holds none of, and for no other (design 47 G §614, A-117). + mu.Lock() + nf, nfOther := told[ghost], told[fp] + mu.Unlock() + if !nf || nfOther { + t.Errorf("entry-not-found told %v for the ghost and %v for the entry held", nf, nfOther) + } // Each entry and want it composes from is kept in the composition state before the composition that uses them. state.mu.Lock() _, hasEntry := state.kv[entryKey(fp)] diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 91753a13..a87ec31a 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -61,6 +61,9 @@ const ResolvedBucket = "mesh-config_resolved" // Env is what the composer needs of the node. type Env struct { Node string + // EntryNotFound is told when the catalog half answered it holds no entry of a fingerprint (notFound), and when that is no longer so: + // the entry answered, or nothing the node holds names it (design 47 G §614, A-117). Nil tells nobody. + EntryNotFound func(fp string, notFound bool) // Watch follows one bucket's keys matching pattern, as the agent's watches do. Watch func(ctx context.Context, bucket, pattern string) (<-chan agent.Update, error) // Get reads one key's latest value; nil and no error when there is none. @@ -182,8 +185,10 @@ type Composer struct { // entries is the cache of entries asked by fingerprint, nil where the composer reads the indexer's states; keptEnt the entries // held from the composition state or an answer, keptWantVal the wants it keeps there and keptWants the modules whose want // stands from that state alone until the live read of the wants answers (entries.go). Under mu; keptWrite orders the writes. - entries *meshcatalogclient.Entries - keptEnt map[string]keptEntry + entries *meshcatalogclient.Entries + keptEnt map[string]keptEntry + // notFound is each fingerprint whose entry-not-found the composer told, so it is told once and cleared once. + notFound map[string]bool keptWantVal map[string]keptWant keptWants map[string]bool keptWrite sync.Mutex -- 2.54.0 From c6df8fa20f11cdbdbb42a7dd6a6ad8a6a21c8e54 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:12:06 +0100 Subject: [PATCH 29/59] =?UTF-8?q?G=20=C2=A7683:=20a=20start=20composes=20t?= =?UTF-8?q?he=20modules=20whose=20kept=20want=20and=20entry=20it=20holds?= =?UTF-8?q?=20before=20its=20other=20inputs=20are=20read?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The node's own record comes from the node-engine's statement of itself, taken in memory with no bus. Builds: design 47 G A-117, A-120, §683, A-95 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- cmd/mesh-host/nodeagent.go | 4 +- internal/link/session.go | 17 +++++ internal/selfcompose/entries_test.go | 11 +++- internal/selfcompose/selfcompose.go | 97 ++++++++++++++++++++++++++++ 4 files changed, 127 insertions(+), 2 deletions(-) diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 0290447d..90c93e3d 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -702,6 +702,8 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus ag = agent.New(env) own.set(ag) current.set(ag) + // The node's own record is stated before the composer starts, from what the node knows of itself and with no bus (A-120). + sayFacts(run, nil) err = ag.Run(run) stopRun() cancel() @@ -1152,7 +1154,7 @@ func selfComposer(session *link.Session, node string, hooks composerHooks, say f // composerEnv is what the node's composers read of the bus, the vault and the agent: its own composer and its answer // to assignment candidates (selfcompose.Shadow) alike. func composerEnv(session *link.Session, node string, say func(string), hooks composerHooks) selfcompose.Env { - env := selfcompose.Env{Node: node, + env := selfcompose.Env{Node: node, NodeRecord: session.OwnRecord, Watch: func(ctx context.Context, bucket, pattern string) (<-chan agent.Update, error) { if bucket == buckets.Assignments || bucket == buckets.ProposalCandidates { // The seat's states are made when their holder starts: a node-engine that starts first waits for the diff --git a/internal/link/session.go b/internal/link/session.go index 8bb427f5..74767dd5 100644 --- a/internal/link/session.go +++ b/internal/link/session.go @@ -183,6 +183,10 @@ func (q *Session) SayFacts(ctx context.Context, r Report) bool { bus := q.bus q.mu.Unlock() if bus == nil { + // With no link open the facts are still taken in memory, so the node's own record is known to what composes before the + // bus is read (design 47 G A-120); nothing is written. + r.Node = q.Membership.Node + q.noteApply(r) return false } r.Node = q.Membership.Node @@ -196,6 +200,19 @@ func (q *Session) SayFacts(ctx context.Context, r Report) bool { return true } +// OwnRecord is the record of this node the node-engine states of itself (tunnel key, sealing key, capabilities), from the facts it +// last took, held in memory (design 47 G A-120); false before any. +func (q *Session) OwnRecord() (nodeengine.NodeRecord, bool) { + q.init() + s := &q.haves + s.mu.Lock() + defer s.mu.Unlock() + if s.node == nil || s.node.Record == nil { + return nodeengine.NodeRecord{}, false + } + return *s.node.Record, true +} + // FactsAsked is the mesh asking this node to say its facts again: run on the link open now. func (q *Session) FactsAsked(ctx context.Context) { q.init() diff --git a/internal/selfcompose/entries_test.go b/internal/selfcompose/entries_test.go index 829659ae..f8b6196b 100644 --- a/internal/selfcompose/entries_test.go +++ b/internal/selfcompose/entries_test.go @@ -125,10 +125,19 @@ func composeFromEntries(t *testing.T) { mu.Unlock() env2 := b.env() env2.Entry, env2.ComposedState = EntryCall(ask), state + // The node's own record, stated by the node-engine of itself in memory (A-120). + env2.NodeRecord = func() (nodeengine.NodeRecord, bool) { + return nodeengine.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}, true + } c2 := New(env2) - if _, err := c2.Watch(ctx, "n1.>"); err != nil { + out2, err := c2.Watch(ctx, "n1.>") + if err != nil { t.Fatal(err) } + // And composes the module it kept a want and an entry of before any input is read, nothing of the others (design 47 G §683, A-117). + if u := next(t, out2); u.Key != lemursKey || u.Deleted { + t.Errorf("the early pass handed %q, want the composition %s of lemurs", u.Key, lemursKey) + } c2.mu.Lock() kept, keptWant := c2.keptEnt[fp], c2.wants["lemurs"].Fp c2.mu.Unlock() diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index a87ec31a..633ff794 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -61,6 +61,9 @@ const ResolvedBucket = "mesh-config_resolved" // Env is what the composer needs of the node. type Env struct { Node string + // NodeRecord is the node's own record as the node-engine states it of itself, held in memory; the early pass of a start composes + // with it before the have bucket is read (design 47 G A-120). Nil, or false, has none. + NodeRecord func() (nodeengine.NodeRecord, bool) // EntryNotFound is told when the catalog half answered it holds no entry of a fingerprint (notFound), and when that is no longer so: // the entry answered, or nothing the node holds names it (design 47 G §614, A-117). Nil tells nobody. EntryNotFound func(fp string, notFound bool) @@ -241,6 +244,8 @@ type Composer struct { asked chan chan Pass // passing is set while a pass is made. passing atomic.Bool + // early is true while the early pass of a start runs (design 47 G §683, A-117). + early atomic.Bool // last is how the last pass went, under mu. last Pass // watching says the composer's loop runs (Watch), so an ask of a pass can be answered. @@ -611,6 +616,20 @@ func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, er // input moves (issue 1126, 2026-10-28). var again <-chan time.Time var replies []chan Pass + // At a start, until its assignments, haves and resolved values are read, the node composes the modules whose kept want and + // kept entry it holds, so the bus module and the indexer start while both are down (design 47 G §683, A-95, A-117). + if c.standing() { + c.passing.Store(true) + updates := c.composeEarly(ctx) + c.passing.Store(false) + for _, u := range updates { + select { + case out <- u: + case <-ctx.Done(): + return + } + } + } for { // Drain whatever came, then compose once for all of it. select { @@ -975,6 +994,7 @@ func (c *Composer) takeValue(u agent.Update) { // compose composes the node once and answers the updates to hand: each composition whose digest differs from the one // handed, and a deletion for each handed one no want names any more. func (c *Composer) compose(ctx context.Context) []agent.Update { + early := c.early.Load() pass := Pass{} defer func() { pass.At = time.Now() @@ -1013,6 +1033,9 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { if module == agent.NodeModule { continue } + if early && !c.stands(module, w) { + continue + } fps[module] = w.Fp if m, ok := c.modules[module]; ok && m.Fp2 != w.Fp { behind = append(behind, module) @@ -1113,6 +1136,11 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { } else { c.paid() } + if early { + // Inputs not yet read may refuse it: nothing is marked, and the full pass follows when they are read. + c.env.Say("the node's early composition of the modules it kept is refused, and the full pass follows: " + why) + return nil + } pass.Refused = why // Said before the agent marks the modules: the marks read each have first, which a paused bus delays. c.sayOnce("the node cannot compose its modules, and keeps running what it runs: " + why) @@ -1232,6 +1260,9 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { } pubComps[key] = comp } + if early { + return out // what the early pass handed is composed from kept state alone: nothing is published from it + } c.pubAt = &published{scope: scope, fps: maps.Clone(fps), comps: pubComps, held: maps.Clone(held), placed: placedMods, ran: ranOf(running)} c.pubDue = true @@ -2431,3 +2462,69 @@ func (c *Composer) EngineModule() string { defer c.mu.Unlock() return c.engine } + +// stands says the want of a module stands for its placement in the early pass of a start: its want is a kept one the live read has +// not answered, and the entry of its build is kept (design 47 G A-117). +func (c *Composer) stands(module string, w meshwants.WantsByWantNodeModuleValue) bool { + if !c.byEntries() || !c.keptWants[module] { + return false + } + k, ok := c.keptEnt[w.Fp] + return ok && k.entry.Module != "" +} + +// standing says the composer holds a kept want and the kept entry of its build, so an early pass has something to compose. +func (c *Composer) standing() bool { + c.mu.Lock() + defer c.mu.Unlock() + for module, w := range c.wants { + if module != agent.NodeModule && c.stands(module, w) { + return true + } + } + return false +} + +// composeEarly is the pass of a start before the assignments, haves and resolved values are read: it composes only the modules whose +// kept want and kept entry it holds, a kept want standing for their placement, and leaves every other module as it runs; the full +// pass follows once the inputs are read (design 47 G §683, A-95, A-117). +func (c *Composer) composeEarly(ctx context.Context) []agent.Update { + node := c.env.Node + c.mu.Lock() + _, hadRecord := c.records[node] + if !hadRecord && c.env.NodeRecord != nil { + if rec, ok := c.env.NodeRecord(); ok { + c.records[node] = rec + } else { + hadRecord = true // nothing laid, nothing to take away + } + } + _, had := c.assignments[node] + if !had { + p := meshmoduleassigner.Assignments{Node: node} + for module, w := range c.wants { + if module != agent.NodeModule && c.stands(module, w) { + p.Modules = append(p.Modules, meshmoduleassigner.PlacedModule{Module: module}) + } + } + sort.Slice(p.Modules, func(i, j int) bool { return p.Modules[i].Module < p.Modules[j].Module }) + c.assignments[node] = p + } + c.mu.Unlock() + c.early.Store(true) + defer func() { + c.early.Store(false) + if !hadRecord { + c.mu.Lock() + delete(c.records, node) + c.mu.Unlock() + } + if !had { + c.mu.Lock() + delete(c.assignments, node) + c.mu.Unlock() + } + }() + c.env.Say("the node composes the modules it kept a want and an entry of, before its other inputs are read") + return c.compose(ctx) +} -- 2.54.0 From e8bff57b55301db710322f03dc8b6d7437cdf59a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 01:51:54 +0100 Subject: [PATCH 30/59] Join 9: the node-engine's link names no seat list verb in its operatorask contract (A-126) Builds: design 47 G A-126 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- internal/link/session.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/link/session.go b/internal/link/session.go index 74767dd5..e0a8a8fe 100644 --- a/internal/link/session.go +++ b/internal/link/session.go @@ -428,9 +428,7 @@ func (q *Session) RouterRecord(key string) (*stdio.StateEntry, error) { ctx, cancel := context.WithTimeout(context.Background(), VaultWithin) defer cancel() return q.AskMeshSeat(ctx, meshmoduleassigner.Seat, meshmoduleassigner.VerbAssignments, body) - }, operatorask.Contract{Lists: "seat:" + meshmoduleassigner.Seat + "." + meshmoduleassigner.VerbAssignments, - ListBody: meshmoduleassigner.AssignmentsInput{Seats: meshmoduleassigner.AssignmentsInputSeatsTrue}, - Seat: occ.Seat, StateAsks: occ.StateAsks}) + }, operatorask.Contract{Seat: occ.Seat, StateAsks: occ.StateAsks}) if err != nil { return nil, err } -- 2.54.0 From efee30e243edab868f05123d00ad675652f695ea Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 02:26:55 +0100 Subject: [PATCH 31/59] Fix 10: the compositions read the catalog key codec from mesh-delivery's sdk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §614, §670 (ADR 0625 §4; lead answer A-144) --- go.mod | 1 - internal/compositions/composed.go | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d40e3f1a..0f641d6a 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,6 @@ require ( git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogkey v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 diff --git a/internal/compositions/composed.go b/internal/compositions/composed.go index c1cac11c..924a0eeb 100644 --- a/internal/compositions/composed.go +++ b/internal/compositions/composed.go @@ -7,7 +7,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogkey" + "git.novox.be/novox/mesh-delivery/sdk/catalogkey" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/declaration" "git.novox.be/novox/mesh-sdk/go/keys" @@ -135,7 +135,7 @@ func Same(a, b Composed) bool { // Key is the key of a module's composition for a node at a fingerprint: `..`. func Key(node, module, fingerprint string) (string, error) { - part, err := meshcatalogkey.KeyPart(fingerprint) + part, err := catalogkey.KeyPart(fingerprint) if err != nil { return "", err } -- 2.54.0 From 7d06ef7997b74c5ed40e6deebbb3b1f3c534ea84 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 02:51:10 +0100 Subject: [PATCH 32/59] Join 10: the composer command may import the catalog key codec where A-144 moved it The codec moved from the mesh-delivery contracts into mesh-delivery's domain SDK (sdk/catalogkey), still a pure package of the standard library and mesh-sdk; the command's import rule named the old place only. Builds: design 47 G A-144 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- cmd/mesh-compose/main_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/mesh-compose/main_test.go b/cmd/mesh-compose/main_test.go index ae0fcf47..9cb3adf1 100644 --- a/cmd/mesh-compose/main_test.go +++ b/cmd/mesh-compose/main_test.go @@ -322,7 +322,7 @@ func ownPackages(t *testing.T) map[string][]string { } // 1621/AC2 and AC4: the command imports, through every package of this repository it reaches, only the standard -// library, mesh-sdk, the contracts modules' pure packages (seats' types, clients' codecs), the delivery SDK's composer (the command's own request and verdict types), the builder's builds package (a build's value and its keys), the private network's overlay package (mesh-wireguard's own pure package: names, graph and +// library, mesh-sdk, the contracts modules' pure packages (seats' types, clients' codecs), the delivery SDK's composer (the command's own request and verdict types) and catalog key codec (a catalog entry's key, design 47 G A-144), the builder's builds package (a build's value and its keys), the private network's overlay package (mesh-wireguard's own pure package: names, graph and // generator, no bus) and this repository's own packages, and none that reaches a bus, a store, a network service or // another process. func TestTheCommandImportsNoBusStoreNetworkOrProcess(t *testing.T) { @@ -334,7 +334,7 @@ func TestTheCommandImportsNoBusStoreNetworkOrProcess(t *testing.T) { for path, imports := range own { for _, imp := range imports { std := !strings.Contains(strings.SplitN(imp, "/", 2)[0], ".") - if !std && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-sdk/go") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/sdk/composer") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds") && !strings.HasPrefix(imp, "github.com/novox/mesh-host/") { + if !std && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-sdk/go") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/sdk/composer") && imp != "git.novox.be/novox/mesh-delivery/sdk/catalogkey" && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds") && !strings.HasPrefix(imp, "github.com/novox/mesh-host/") { t.Errorf("%s imports %s, which is neither the standard library, mesh-sdk nor this repository", path, imp) } for _, f := range forbidden { -- 2.54.0 From 94f6c3dba490207a1eb08500ad1425de5f3c28d9 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 04:54:19 +0100 Subject: [PATCH 33/59] H-18: running carries healthy and is said again when it changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §520 --- internal/link/have.go | 22 ++++++++++++++-------- internal/link/have_test.go | 6 +++++- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/internal/link/have.go b/internal/link/have.go index 5d190a18..f1a5a696 100644 --- a/internal/link/have.go +++ b/internal/link/have.go @@ -919,26 +919,32 @@ func nothingJudged(f ModuleFact, looked bool) string { return nodeengineclient.HealthUnknown } -// runningToSay is the event running for a module whose have was just written running and healthy at a fingerprint whose -// manifest is known, when it has not been said for that fingerprint and manifest (design 47, *What each step says*): the -// module, the fingerprint and the digest of the manifest of that build. A module not yet healthy, or one whose manifest -// the node does not know, says nothing, and is said at the write that knows. Under s.mu. +// runningToSay is the event running for a module whose have was just written running with a health of ok or failed at a +// fingerprint whose manifest is known, when it has not been said for that fingerprint, manifest and health (design 47 G +// §520): `healthy` says the build's health, and it is said again each time that changes and, the said set being empty, for +// every module the have holds at the node-engine's start. A module whose health is waits or unknown, or whose manifest the +// node does not know, says nothing, and is said at the write that knows. Under s.mu. func (s *haveState) runningToSay(module string, h nodeengine.HaveByNodeModuleValue, manifest func(module, fp string) string) (nodeengine.Running, bool) { - if h.Phase != nodeengineclient.PhaseRunning || h.Health != nodeengineclient.HealthOK || h.Fp == "" || manifest == nil { + if h.Phase != nodeengineclient.PhaseRunning || h.Fp == "" || manifest == nil { + return nodeengine.Running{}, false + } + if h.Health != nodeengineclient.HealthOK && h.Health != nodeengineclient.HealthFailed { return nodeengine.Running{}, false } digest := manifest(module, h.Fp) if digest == "" { return nodeengine.Running{}, false } - if s.runningSaid[module] == h.Fp+"\x00"+digest { + healthy := h.Health == nodeengineclient.HealthOK + said := fmt.Sprintf("%s\x00%s\x00%t", h.Fp, digest, healthy) + if s.runningSaid[module] == said { return nodeengine.Running{}, false } if s.runningSaid == nil { s.runningSaid = map[string]string{} } - s.runningSaid[module] = h.Fp + "\x00" + digest - return nodeengine.Running{V: sdkkeys.Version, Module: module, Fingerprint: h.Fp, Manifest: digest}, true + s.runningSaid[module] = said + return nodeengine.Running{V: sdkkeys.Version, Module: module, Fingerprint: h.Fp, Manifest: digest, Healthy: &healthy}, true } // waitToSay is the event wait-held for a have written with a wait the last one did not say, or wait-ended for one written with diff --git a/internal/link/have_test.go b/internal/link/have_test.go index 084557bf..132fa730 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -722,7 +722,7 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth( q.SayHealth(context.Background(), *healthOf(map[string]string{"web": StateHealthy})) q.SayHealth(context.Background(), *healthOf(map[string]string{"web": StateHealthy})) if got := l.saidOn("mesh.seat.node-engine.event.running.n1.web "); len(got) != 1 || - !strings.Contains(got[0], `"fingerprint":"`+fp+`"`) || !strings.Contains(got[0], `"manifest":"sha256:`+strings.Repeat("c", 64)+`"`) { + !strings.Contains(got[0], `"healthy":true`) || !strings.Contains(got[0], `"fingerprint":"`+fp+`"`) || !strings.Contains(got[0], `"manifest":"sha256:`+strings.Repeat("c", 64)+`"`) { t.Fatalf("running said %v, want one for the healthy build", got) } // A have with a wait says wait-held once on wait-held.., and wait-ended when the wait is gone (design 47 G §541, A-113). @@ -750,6 +750,10 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth( if got := l.have(t, "n1.web"); got.Health != nodeengineclient.HealthFailed || got.Fp != fp || got.Composed != dig { t.Fatalf("a failing check after the start window: %+v", got) } + // §520: running is said again when healthy changes, now false, and the stream's newest says it. + if got := l.saidOn("mesh.seat.node-engine.event.running.n1.web "); len(got) != 2 || !strings.Contains(got[1], `"healthy":false`) { + t.Fatalf("running said %v after the check failed, want a second with healthy false", got) + } // The bus's user list written alone rides on the module's have, and an apply after it keeps it (ADR 0547 §8). w := nodeengine.UserListWritten{Source: "n1.web." + fp, Revision: 4, Wanted: dig, Digest: dig, Users: 3, -- 2.54.0 From 4d656fe4e16b3ad0895f4ee271fea8ee49b02a12 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 04:55:33 +0100 Subject: [PATCH 34/59] H-20: a tools bundle's words hold its module's want fingerprint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §519, §618 --- internal/compose/composed.go | 4 +- internal/compose/composed_test.go | 7 ++- internal/compositions/compare_test.go | 21 +++++++++ internal/compositions/composed.go | 63 +++++++++++++++++++++++++++ 4 files changed, 92 insertions(+), 3 deletions(-) diff --git a/internal/compose/composed.go b/internal/compose/composed.go index 5bdc1c0a..7516d9e3 100644 --- a/internal/compose/composed.go +++ b/internal/compose/composed.go @@ -388,8 +388,8 @@ func ToolsRunning(running []engcomp.Composed) map[string][]catalogue.Bundle { } var own map[string]string for w, v := range words[module] { - if w == catalogue.FilesOwnedPaths || w == catalogue.ReadsSecretPlaces { - continue // the mesh's to say, composed again + if w == catalogue.FilesOwnedPaths || w == catalogue.ReadsSecretPlaces || w == engcomp.FingerprintVar { + continue // the mesh's to say, composed again (the fingerprint is the want's, §519) } if own == nil { own = map[string]string{} diff --git a/internal/compose/composed_test.go b/internal/compose/composed_test.go index c33c7490..6abb069e 100644 --- a/internal/compose/composed_test.go +++ b/internal/compose/composed_test.go @@ -180,7 +180,12 @@ func TestCompositionsAreTheDeclarationSplitByModule(t *testing.T) { } } } - if env[catalogue.RuntimeToolModules] != served || env[catalogue.RuntimeToolEnv] != `{"lamp":{"LAMP_CONF":"/var/lib/lamp/conf"}}` { + // §519: the words each bundle's module is given hold the fingerprint of the want it applies, where there is one. + words := `{"lamp":{"LAMP_CONF":"/var/lib/lamp/conf"}}` + if fp := row.fps["lamp"]; fp != "" { + words = `{"lamp":{"LAMP_CONF":"/var/lib/lamp/conf","MESH_FINGERPRINT":"` + fp + `"}}` + } + if env[catalogue.RuntimeToolModules] != served || env[catalogue.RuntimeToolEnv] != words { t.Errorf("%s: the runtime serves %v with %v, want what runs of lamp, %s", row.name, env[catalogue.RuntimeToolModules], env[catalogue.RuntimeToolEnv], served) } diff --git a/internal/compositions/compare_test.go b/internal/compositions/compare_test.go index a4987123..434deff4 100644 --- a/internal/compositions/compare_test.go +++ b/internal/compositions/compare_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" seat "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-sdk/go/catalogue" "slices" "strings" "testing" @@ -203,6 +204,26 @@ func TestComparingCompositions(t *testing.T) { t.Errorf("the node's own carries %s: %s", FingerprintVar, raw) } } + + // §519, §618: the tool runner's process hands each bundle's module its own want's fingerprint in the words of its + // bundles, a module with bundles and no words included, and the runner's own fingerprint reaches no other module's. + fpLamp, fpBare := "src2:"+strings.Repeat("1", 64), "src2:"+strings.Repeat("2", 64) + runner, err := Split("novox", []json.RawMessage{json.RawMessage(`{"id":"r","type":"process","module":"nats","run":["x"],"env":{` + + `"` + catalogue.RuntimeToolModules + `":"bare=/b/x,lamp=/l/y","` + catalogue.RuntimeToolEnv + `":"{\"lamp\":{\"LAMP_CONF\":\"/c\"}}"}}`)}, + map[string]string{"nats": fp, "lamp": fpLamp, "bare": fpBare}, nil, NodeOwn{}) + if err != nil { + t.Fatal(err) + } + var proc struct{ Env map[string]string } + _ = json.Unmarshal(runner[mustKey(t, "novox", "nats", fp)].Resources[0], &proc) + var given map[string]map[string]string + if err := json.Unmarshal([]byte(proc.Env[catalogue.RuntimeToolEnv]), &given); err != nil { + t.Fatal(err) + } + if given["lamp"][FingerprintVar] != fpLamp || given["lamp"]["LAMP_CONF"] != "/c" || given["bare"][FingerprintVar] != fpBare || + len(given["nats"]) != 0 || proc.Env[FingerprintVar] != fp { + t.Errorf("the bundles' words are %v and the runner's env %v", given, proc.Env) + } } func mustKey(t *testing.T, node, module, fp string) string { diff --git a/internal/compositions/composed.go b/internal/compositions/composed.go index 924a0eeb..a2d5223f 100644 --- a/internal/compositions/composed.go +++ b/internal/compositions/composed.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "regexp" + "strings" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" @@ -239,6 +240,9 @@ func Split(node string, resources []json.RawMessage, fps map[string]string, fill if raw, err = withFingerprint(raw, head.Type, fps[head.Module]); err != nil { return nil, err } + if raw, err = withBundleFingerprints(raw, head.Type, fps); err != nil { + return nil, err + } } c := out[key] c.Resources = append(c.Resources, raw) @@ -286,3 +290,62 @@ func withFingerprint(raw json.RawMessage, kind, fp string) (json.RawMessage, err fields["env"] = body return json.Marshal(fields) } + +// withBundleFingerprints is raw, when it is a process whose env hands tools bundles their words (catalogue.RuntimeToolEnv), +// with each module's words holding FingerprintVar set to that module's fingerprint in fps: a bundle reads the fingerprint of +// its own module's want from the environment the tool runner starts it with (design 47 G §519). A module fps names no +// fingerprint for keeps its words, and the runner's own fingerprint reaches no bundle but its own module's (§618). +func withBundleFingerprints(raw json.RawMessage, kind string, fps map[string]string) (json.RawMessage, error) { + if kind != string(declaration.TypeProcess) { + return raw, nil + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(raw, &fields); err != nil { + return nil, fmt.Errorf("compositions: a %s resource is not JSON: %w", kind, err) + } + var env map[string]string + if had, ok := fields["env"]; !ok || json.Unmarshal(had, &env) != nil || env[catalogue.RuntimeToolModules] == "" { + return raw, nil + } + given := map[string]map[string]string{} + if had := env[catalogue.RuntimeToolEnv]; had != "" { + if err := json.Unmarshal([]byte(had), &given); err != nil { + return nil, fmt.Errorf("compositions: the %s of a process is not words by module: %w", catalogue.RuntimeToolEnv, err) + } + } + for module, words := range given { + if fp := fps[module]; fp != "" { + if words == nil { + words = map[string]string{} + given[module] = words + } + words[FingerprintVar] = fp + } + } + for module, fp := range fps { // a module with bundles and no words of its own still reads its fingerprint + if _, has := given[module]; !has && fp != "" && hasLoadedBundle(env, module) { + given[module] = map[string]string{FingerprintVar: fp} + } + } + body, err := json.Marshal(given) + if err != nil { + return nil, err + } + env[catalogue.RuntimeToolEnv] = string(body) + envBody, err := json.Marshal(env) + if err != nil { + return nil, err + } + fields["env"] = envBody + return json.Marshal(fields) +} + +// hasLoadedBundle is whether the runtime's served list (catalogue.RuntimeToolModules, `module=path` entries) names a bundle of module. +func hasLoadedBundle(env map[string]string, module string) bool { + for _, entry := range strings.Split(env[catalogue.RuntimeToolModules], ",") { + if strings.HasPrefix(entry, module+"=") { + return true + } + } + return false +} -- 2.54.0 From ac3422f91435f048ba3cf75becde10dda001d192 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 04:58:28 +0100 Subject: [PATCH 35/59] H-17: the node-engine's own name is read from its applied record or first composing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Discovery, the finding writer and the vault login names read it; no fixed name. Builds: design 47 G §527, §634 (A-149) --- cmd/mesh-host/applied_store.go | 20 +++++ cmd/mesh-host/logins.go | 13 ++-- cmd/mesh-host/main.go | 6 ++ cmd/mesh-host/nodeagent.go | 20 +++-- cmd/mesh-host/nodeagent_integration_test.go | 13 +++- cmd/mesh-host/ownname_test.go | 56 ++++++++++++++ cmd/mesh-host/recovery_integration_test.go | 2 +- cmd/mesh-host/seatlist.go | 81 +++++++++++++++++++-- internal/link/engineseat.go | 33 +++++++-- internal/link/hearing_nats.go | 10 +++ internal/link/ownname.go | 61 ++++++++++++++++ internal/link/ownname_test.go | 32 ++++++++ internal/link/run.go | 3 + internal/link/session.go | 3 + internal/seatwait/held.go | 6 +- internal/seatwait/seatwait.go | 19 ++++- internal/selfcompose/selfcompose.go | 9 ++- 17 files changed, 346 insertions(+), 41 deletions(-) create mode 100644 cmd/mesh-host/ownname_test.go create mode 100644 internal/link/ownname.go create mode 100644 internal/link/ownname_test.go diff --git a/cmd/mesh-host/applied_store.go b/cmd/mesh-host/applied_store.go index 75243ed1..5409d471 100644 --- a/cmd/mesh-host/applied_store.go +++ b/cmd/mesh-host/applied_store.go @@ -1,12 +1,16 @@ package main import ( + "encoding/json" "errors" "fmt" "os" "path/filepath" "regexp" "strings" + + "github.com/novox/mesh-host/internal/agent" + "github.com/novox/mesh-host/internal/compositions" ) // appliedStore keeps every composition the agent applied, one file per module (the node's own as `node.json`), in the @@ -105,3 +109,19 @@ func (s appliedStore) Drop(module string) error { } return nil } + +// ownModuleApplied is the module of the composition the store holds at the fingerprint fp, the engine's own build's +// (design 47 G §527, A-149): "" when fp is none or no record holds it. +func ownModuleApplied(s appliedStore, fp string) string { + if fp == "" { + return "" + } + all, _ := s.Load() + for module, raw := range all { + var c compositions.Composed + if module != agent.NodeModule && json.Unmarshal(raw, &c) == nil && c.FP == fp && c.Module != "" { + return c.Module + } + } + return "" +} diff --git a/cmd/mesh-host/logins.go b/cmd/mesh-host/logins.go index a8ba1644..3484442d 100644 --- a/cmd/mesh-host/logins.go +++ b/cmd/mesh-host/logins.go @@ -87,15 +87,16 @@ func followLogins(ctx context.Context, session *link.Session, node string, hear } } -// loginSecretModule is the module the vault names this engine's login values under: own//mesh-host/ (ADR 0583 §3, -// insight). -const loginSecretModule = "mesh-host" - // fetchLogin asks the vault, through its existing hand-out of a machine's own secrets, for the value of login, sealed to this // node's key, and opens it with the sealing key; no new verb is asked for it. func fetchLogin(queue *link.Session, statePath string) func(context.Context, string) (string, error) { unseal := sealOpener(statePath) return func(ctx context.Context, login string) (string, error) { + // The vault names the login values under the engine's own module, which is §527's (§634). + own, err := queue.Own.Wait(ctx) + if err != nil { + return "", fmt.Errorf("the node-engine's own module is not named yet, so the vault's name of its login value is not known: %w", err) + } raw, err := queue.AskVault(ctx, "own", map[string]any{}) if err != nil { return "", err @@ -111,7 +112,7 @@ func fetchLogin(queue *link.Session, statePath string) func(context.Context, str return "", fmt.Errorf("the vault's own answer is not one: %w", err) } for _, o := range a.Own { - if o.Module != loginSecretModule || o.Name != login { + if o.Module != own || o.Name != login { continue } sealed := string(o.Sealed) @@ -125,7 +126,7 @@ func fetchLogin(queue *link.Session, statePath string) func(context.Context, str } return string(value), nil } - return "", fmt.Errorf("the vault hands this node no value for %s (own/%s/%s/%s)", login, queueNode(queue), loginSecretModule, login) + return "", fmt.Errorf("the vault hands this node no value for %s (own/%s/%s/%s)", login, queueNode(queue), own, login) } } diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 193efcc6..eb47ba29 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -36,6 +36,7 @@ import ( "github.com/novox/mesh-host/internal/agent" "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/bundle" + "github.com/novox/mesh-host/internal/compositions" "github.com/novox/mesh-host/internal/firewall" "github.com/novox/mesh-host/internal/identity" "github.com/novox/mesh-host/internal/inventory" @@ -987,7 +988,12 @@ func runLink(ctx context.Context, opts options) error { // What this engine trusts a mesh signature from: the vault's key alone, once read from its signers (novox/hq ADR 0550 // §2; issue 1126: the controller's key never, the controller ended by its unassignment). trust := newMeshTrust() + own := link.NewOwnName() + // The engine's own module, first from its applied record of the build whose fingerprint is its own (design 47 G §527, + // §634, A-149); on a first start with no such record, from its first composing. + own.Set(ownModuleApplied(appliedStoreFor(opts.state), os.Getenv(compositions.FingerprintVar))) queue := &link.Session{ + Own: own, Membership: membership, // And the public halves and certificate its record says (novox/hq ADR 0555 §3), read at each facts report. Keys: recordKeys(opts.state, trust.Trusts), diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 90c93e3d..d81de981 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -601,16 +601,13 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus // And the seat list it reads through the seat mesh-catalog, from its bootstrapped definitions until it has: the // composer hands nothing new meanwhile, and the node says it has waited five minutes (novox/hq ADR 0636 §1). // The finding writer is started first, so it holds what an earlier run left before the reader writes or clears any. - findings, ferr := newFindingsOfTheNode(session, node, say) - if ferr != nil { // a writer that cannot be made is a fault of its configuration: loud, never a node without findings - say("the node-engine cannot make its finding writer, so its node agent stops: " + ferr.Error()) - stopRun() - cancel() - return - } - if werr := findings.Start(run); werr != nil { - say("the node-engine's findings are kept in its state and not said as events until it is started again: " + werr.Error()) + if session.Own == nil { + session.Own = link.NewOwnName() } + findings := &lateFindings{own: session.Own, make: func(module string) (*findingsOfTheNode, error) { + return newFindingsOfTheNode(session, node, module, say) + }} + findings.Start(run, say) seats, err := startSeatList(run, session, node, findings, say) if err != nil { // No reader is no gate that lets everything through: the node acts on nothing new, as one that waits does. @@ -618,10 +615,10 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus } // The marks of the haves run off the composition pass, each bounded (see marker). marks := newMarker(run) - heldFindings := &seatwait.HeldKeeper{Node: node, Observer: findingsModule, Writer: findings, Ctx: run, Say: say} + heldFindings := &seatwait.HeldKeeper{Node: node, ObserverOf: findings.observer, Writer: findings, Ctx: run, Say: say} hooks := composerHooks{ entryNotFound: func(fp string, notFound bool) { - if err := seatwait.SayEntryNotFound(run, findings, node, findingsModule, fp, notFound, time.Now); err != nil { + if err := seatwait.SayEntryNotFound(run, findings, node, findings.observer(), fp, notFound, time.Now); err != nil { say("the finding that the catalog half holds no entry of " + fp + " cannot be kept: " + err.Error()) } }, @@ -687,6 +684,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus // composition bucket) is never applied. While the seat's assignment holds no key for this node it composes // nothing and keeps running what it runs; a assignment it cannot watch stops the agent, said, and it begins again. composer := selfComposer(session, node, hooks, say) + composer.OnEngine = session.Own.Set // the first composing names the engine's own module on a first start (§527, §634) machinesNow.Store(composer) // the machines a failing network part points at, by the assignments (issue 774) composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone session.SetManifestOf(composer.ManifestOf) // the event running names the manifest of the build that runs diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index 4d3f4021..ff1d7868 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -231,10 +231,19 @@ func newItBedAs(t *testing.T, perms nodegrant.Permissions) *itBed { return b } +// itOwnModule is the node-engine's own module of the bed: what its applied record of its own build says (§527, §634). +const itOwnModule = "mesh-host" + +func itOwn() *link.OwnName { + o := link.NewOwnName() + o.Set(itOwnModule) + return o +} + // start runs the node agent under a session on the bus. func (b *itBed) start() (stop func()) { b.t.Helper() - b.session = &link.Session{Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second, + b.session = &link.Session{Own: itOwn(), Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second, Say: b.say, Keys: func() link.RecordKeys { return link.RecordKeys{} }} b.session.Attach(context.Background(), link.OverNATS{Conn: b.nodeConn, JS: b.nodeJS, Node: itNode}) ctx, cancel := context.WithCancel(context.Background()) @@ -1055,7 +1064,7 @@ func (c *fiveMinutes) pass() { // findingSubject is the subject of one of the engine's finding events on node (views.FindingSubjects). func findingSubject(event, node string) string { - return "mesh.stream." + findingsModule + ".event." + event + "." + node + return "mesh.stream." + itOwnModule + ".event." + event + "." + node } // ADR 0636 §1, §8, whole against a real bus and the stand-in of the seat: while the seat refuses, the node composes and diff --git a/cmd/mesh-host/ownname_test.go b/cmd/mesh-host/ownname_test.go new file mode 100644 index 00000000..1a145b58 --- /dev/null +++ b/cmd/mesh-host/ownname_test.go @@ -0,0 +1,56 @@ +package main + +import ( + "context" + "path/filepath" + "strings" + "testing" + "time" + + condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + "github.com/novox/mesh-host/internal/link" +) + +// Design 47 G §527, §634, A-149: the node-engine's own module is the module of its applied record at its own fingerprint, +// and its finding writer is made, observing as that module, once the name is known: at once from the record, else when the +// first composing names it. No name is fixed in code. +func TestTheEnginesOwnNameIsReadFromItsAppliedRecordOrItsFirstComposing(t *testing.T) { + fp := "src2:" + strings.Repeat("a", 64) + applied := appliedStoreFor(filepath.Join(t.TempDir(), "state", "state.json")) + for module, f := range map[string]string{"node": "", "the-engine": fp, "other": "src2:" + strings.Repeat("b", 64)} { + if err := applied.Keep(module, []byte(`{"module":"`+module+`","fp":"`+f+`"}`)); err != nil { + t.Fatal(err) + } + } + for _, tc := range []struct{ fp, want string }{{fp, "the-engine"}, {"", ""}, {"src2:" + strings.Repeat("c", 64), ""}} { + if got := ownModuleApplied(applied, tc.fp); got != tc.want { + t.Errorf("the own module at %q is %q, want %q", tc.fp, got, tc.want) + } + } + + own := link.NewOwnName() + made := make(chan string, 1) + late := &lateFindings{own: own, make: func(module string) (*findingsOfTheNode, error) { + made <- module + return nil, context.Canceled // the writer itself is the SDK's: only when it is made is tested here + }} + ctx, stop := context.WithCancel(context.Background()) + defer stop() + var said []string + late.Start(ctx, func(s string) { said = append(said, s) }) + if err := late.Write(ctx, condseat.Finding{}); err == nil || late.Held() != nil { + t.Fatalf("a finding was taken before the engine's own module was known: %v", err) + } + if len(made) != 0 { + t.Fatal("the writer was made before a name") + } + own.Set("the-engine") // the first composing names it + select { + case m := <-made: + if m != "the-engine" || late.observer() != "the-engine" { + t.Fatalf("the writer was made as %q, observing as %q", m, late.observer()) + } + case <-time.After(5 * time.Second): + t.Fatal("the writer was never made after the name was known") + } +} diff --git a/cmd/mesh-host/recovery_integration_test.go b/cmd/mesh-host/recovery_integration_test.go index 2071af0d..14718427 100644 --- a/cmd/mesh-host/recovery_integration_test.go +++ b/cmd/mesh-host/recovery_integration_test.go @@ -135,7 +135,7 @@ func TestNatsTheRecoveryStepAsksHearsBoundsAndCatchesUpOverTheBus(t *testing.T) // startSession is a session on the node's own connection, with no node agent run on it. func (b *itBed) startSession() *link.Session { - b.session = &link.Session{Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second, + b.session = &link.Session{Own: itOwn(), Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second, Say: b.say, Keys: func() link.RecordKeys { return link.RecordKeys{} }} b.session.Attach(context.Background(), link.OverNATS{Conn: b.nodeConn, JS: b.nodeJS, Node: itNode}) return b.session diff --git a/cmd/mesh-host/seatlist.go b/cmd/mesh-host/seatlist.go index 7461d625..add27c98 100644 --- a/cmd/mesh-host/seatlist.go +++ b/cmd/mesh-host/seatlist.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "strings" + "sync" "sync/atomic" "time" @@ -102,10 +103,6 @@ func (realClock) AfterFunc(d time.Duration, f func()) seatwait.Timer { return time.AfterFunc(d, f) } -// findingsModule is the module whose own stream the node-engine says its findings on, and whose name its findings carry as -// their observer (module.json's `module` and its stream `mesh-host-findings`, design 47 G §383). -const findingsModule = "mesh-host" - // nodeFindings is the node-engine's finding writer: mesh-sdk's held writer, for the engine's findings are raised as they // happen and not judged afresh (design 47 G A-29, A-40), with the contract of the seat mesh-conditions. type nodeFindings = views.HeldWriter[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent] @@ -120,9 +117,9 @@ type findingsOfTheNode struct { // newFindingsOfTheNode makes the writer at once; it reads the state when it is started (A-40). Its events are published on // the engine's own stream awaiting the bus's acknowledgement and its consumer of `findings-asked` is its own. -func newFindingsOfTheNode(session *link.Session, node string, say func(string)) (*findingsOfTheNode, error) { +func newFindingsOfTheNode(session *link.Session, node, module string, say func(string)) (*findingsOfTheNode, error) { writer, err := views.NewHeldWriter(views.WriterConfig[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]{ - Module: findingsModule, + Module: module, Node: node, State: link.FindingsState{Session: session}, Kinds: []string{seatwait.Kind, seatwait.HeldKind, seatwait.EntryNotFoundKind}, @@ -148,7 +145,7 @@ func (w *findingsOfTheNode) Start(ctx context.Context) error { // startSeatList starts this run's reader: its bootstrapped seats are given to catalogue at once, and every change of what // it holds again, saying so to the composer and to the waiter that writes the five-minute finding, and stating again what // the engine reads (link.Session.RestateReads, ADR 0618 §4): the seats it states are those of the list it holds. -func startSeatList(ctx context.Context, session *link.Session, node string, findings seatwait.Writer, say func(string)) (*seatListState, error) { +func startSeatList(ctx context.Context, session *link.Session, node string, findings *lateFindings, say func(string)) (*seatListState, error) { st := &seatListState{changed: make(chan struct{}, 1)} // Start returns at once and reads the seat on a goroutine, which may announce a change before the waiter exists. var announce atomic.Pointer[func()] @@ -166,7 +163,7 @@ func startSeatList(ctx context.Context, session *link.Session, node string, find return nil, fmt.Errorf("the node-engine cannot follow the seat list: %w", err) } st.reader = reader - st.waiter = &seatwait.Waiter{Node: node, Observer: findingsModule, Reader: reader, Writer: findings, Ctx: ctx, + st.waiter = &seatwait.Waiter{Node: node, ObserverOf: findings.observer, Reader: reader, Writer: findings, Ctx: ctx, Clock: seatWaitClock, Say: say} told := func() { st.waiter.Changed() @@ -194,3 +191,71 @@ func startSeatList(ctx context.Context, session *link.Session, node string, find seatListNow.Store(st) return st, nil } + +// lateFindings is the node-engine's finding writer, made when the engine's own module is known (design 47 G §527, §634, +// A-149): at once from its applied record, or, on a first start with no such record, when the first composing names it. Till +// then a write or a clear is refused in words, never lost in silence. +type lateFindings struct { + own *link.OwnName + make func(module string) (*findingsOfTheNode, error) + mu sync.Mutex + w *findingsOfTheNode +} + +// observer is the module the findings are observed as: the engine's own, "" while it is not known. +func (f *lateFindings) observer() string { return f.own.Get() } + +// Start makes the writer and starts it as soon as the own name is known: in this call when it already is, else on a goroutine. +func (f *lateFindings) Start(ctx context.Context, say func(string)) { + begin := func(module string) { + w, err := f.make(module) + if err != nil { // a writer that cannot be made is a fault of its configuration: loud + say("the node-engine cannot make its finding writer: " + err.Error()) + return + } + if err := w.Start(ctx); err != nil { + say("the node-engine's findings are kept in its state and not said as events until it is started again: " + err.Error()) + } + f.mu.Lock() + f.w = w + f.mu.Unlock() + } + if name := f.own.Get(); name != "" { + begin(name) + return + } + go func() { + if name, err := f.own.Wait(ctx); err == nil { + begin(name) + } + }() +} + +var errOwnNameUnknown = errors.New("the node-engine's own module is not yet named by its applied record or its first composing, so its finding writer is not made") + +func (f *lateFindings) writer() *findingsOfTheNode { + f.mu.Lock() + defer f.mu.Unlock() + return f.w +} + +func (f *lateFindings) Write(ctx context.Context, fi condseat.Finding) error { + if w := f.writer(); w != nil { + return w.Write(ctx, fi) + } + return errOwnNameUnknown +} + +func (f *lateFindings) Clear(ctx context.Context, scope, id, kind, why string) error { + if w := f.writer(); w != nil { + return w.Clear(ctx, scope, id, kind, why) + } + return errOwnNameUnknown +} + +func (f *lateFindings) Held() []condseat.Finding { + if w := f.writer(); w != nil { + return w.Held() + } + return nil +} diff --git a/internal/link/engineseat.go b/internal/link/engineseat.go index f63404b1..d49d8d02 100644 --- a/internal/link/engineseat.go +++ b/internal/link/engineseat.go @@ -131,6 +131,28 @@ func (l *natsLink) ServeEngineSeat(handle EngineFunc) { if handle == nil || l.conn == nil { return } + // Discovery names the module whose code answers: the engine's own, read from §527 and no fixed name (§634). Known at + // once from the applied record; on a first start it is served once the first composing names it. + if l.own == nil || l.own.Get() != "" { + l.serveEngineSeat(handle, l.own.Get()) + return + } + ctx, cancel := context.WithCancel(context.Background()) + l.engineMu.Lock() + l.stopWait = cancel + l.engineMu.Unlock() + go func() { + if name, err := l.own.Wait(ctx); err == nil { + l.serveEngineSeat(handle, name) + } + }() +} + +// UseOwnName gives the link the engine's own name its discovery reads; before ServeEngineSeat. +func (l *natsLink) UseOwnName(o *OwnName) { l.own = o } + +// serveEngineSeat serves the seat as the module module. +func (l *natsLink) serveEngineSeat(handle EngineFunc, module string) { svc, err := micro.AddService(l.conn, micro.Config{Name: EngineSeat, Version: "1.0.0", Description: "the node seat node-engine of " + l.node + " (novox/hq ADR 0563)", QueueGroupDisabled: true, Metadata: map[string]string{"seat": EngineSeat, "scope": "node", "node": l.node}}) @@ -141,13 +163,15 @@ func (l *natsLink) ServeEngineSeat(handle EngineFunc) { for _, v := range ServedEngineSeat().Serves { verb := v.Name answer := EngineAnswer(handle, verb, l.node, apart) - meta := EngineEndpointMetadata(v, l.node) + meta := EngineEndpointMetadata(v, l.node, module) _ = svc.AddEndpoint(verb, answer, micro.WithEndpointSubject(EngineToolSubject(verb, l.node)), micro.WithEndpointMetadata(meta)) _ = svc.AddEndpoint(verb+"-called", answer, micro.WithEndpointSubject(EngineCallPattern(verb, l.node)), micro.WithEndpointMetadata(meta)) } + l.engineMu.Lock() l.engine = svc + l.engineMu.Unlock() } // answeredApart are the verbs answered off the subscription that delivered them: each endpoint's subscription hands its @@ -184,20 +208,17 @@ func EngineAnswer(handle EngineFunc, verb, node string, apart chan struct{}) mic } } -// EngineModule is the module whose code answers the seat, as discovery names it: the node-engine's manifest's. -const EngineModule = "mesh-host" - // EngineEndpointMetadata is what discovery reads of one verb's endpoint, in the shape every tool runner and the // controller announce a seat's verb in (mesh-tools node-tools' announce package): its kind, the seat, its scope, the // module whose code answers, the verb, the node, its description and its schema. Without the kind and the scope the // console listed none of it, and `/node-engine.` answered that nothing in the mesh is called node-engine // (the sweep of 2026-10-30). -func EngineEndpointMetadata(v catalogue.Verb, node string) map[string]string { +func EngineEndpointMetadata(v catalogue.Verb, node, module string) map[string]string { schema, err := json.Marshal(v.Input) if err != nil || v.Input == nil { schema = []byte("{}") } - return map[string]string{"kind": "seat", "seat": EngineSeat, "scope": "node", "module": EngineModule, + return map[string]string{"kind": "seat", "seat": EngineSeat, "scope": "node", "module": module, "tool": v.Name, "node": node, "interchangeable": "false", "description": v.Description, "schema": string(schema)} } diff --git a/internal/link/hearing_nats.go b/internal/link/hearing_nats.go index 50fa1a97..97a415f6 100644 --- a/internal/link/hearing_nats.go +++ b/internal/link/hearing_nats.go @@ -8,6 +8,7 @@ import ( "os" "regexp" "strings" + "sync" "time" "github.com/nats-io/nats.go" @@ -48,6 +49,10 @@ type natsLink struct { moving *nats.Subscription // engine serves the node seat node-engine, and warrants hears the operator's answers to its asks (ADR 0563). engine micro.Service + engineMu sync.Mutex + // own is the engine's own module, which discovery names (§634); stopWait ends a wait for it when the link closes. + own *OwnName + stopWait context.CancelFunc warrants *nats.Subscription node string // user is the bus user this link authenticated as (node. or node.--b, ADR 0583 §1). @@ -295,9 +300,14 @@ func (l *natsLink) Close() { if l.judging != nil { _ = l.judging.Unsubscribe() } + l.engineMu.Lock() + if l.stopWait != nil { + l.stopWait() + } if l.engine != nil { _ = l.engine.Stop() } + l.engineMu.Unlock() if l.warrants != nil { _ = l.warrants.Unsubscribe() } diff --git a/internal/link/ownname.go b/internal/link/ownname.go new file mode 100644 index 00000000..00f6264f --- /dev/null +++ b/internal/link/ownname.go @@ -0,0 +1,61 @@ +package link + +import ( + "context" + "errors" + "sync" +) + +// OwnName is the module that is this node-engine: the module whose catalog entry claims the seat node-engine, which the +// engine learns first from its applied record of the build whose fingerprint is its own and, on a first start with no such +// record, from its first composing (design 47 G §527, §634, A-149). Its discovery, its finding writer and its login names +// in the vault read it here and fix none. +type OwnName struct { + mu sync.Mutex + name string + ready chan struct{} +} + +// NewOwnName makes an own name not yet known. +func NewOwnName() *OwnName { return &OwnName{ready: make(chan struct{})} } + +// Set keeps the name; an empty one changes nothing. The first name opens Wait for everyone. +func (o *OwnName) Set(name string) { + if o == nil || name == "" { + return + } + o.mu.Lock() + defer o.mu.Unlock() + if o.name == "" { + close(o.ready) + } + o.name = name +} + +// Get is the name, "" while it is not known. +func (o *OwnName) Get() string { + if o == nil { + return "" + } + o.mu.Lock() + defer o.mu.Unlock() + return o.name +} + +// Wait returns the name once it is known, or ctx's error. +func (o *OwnName) Wait(ctx context.Context) (string, error) { + if o == nil { + return "", errors.New("this session names no own module") + } + select { + case <-o.ready: + return o.Get(), nil + case <-ctx.Done(): + return "", ctx.Err() + } +} + +// OwnNamed is a link that reads the node-engine's own module from an OwnName when it serves its seat. +type OwnNamed interface { + UseOwnName(*OwnName) +} diff --git a/internal/link/ownname_test.go b/internal/link/ownname_test.go new file mode 100644 index 00000000..7c031d9c --- /dev/null +++ b/internal/link/ownname_test.go @@ -0,0 +1,32 @@ +package link + +import ( + "context" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Design 47 G §527, §634: discovery names the module the engine reads of itself, Wait holds until it is known, and an +// empty name changes nothing. +func TestTheOwnNameIsWaitedForAndDiscoveryNamesIt(t *testing.T) { + o := NewOwnName() + o.Set("") + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) + defer cancel() + if _, err := o.Wait(ctx); err == nil || o.Get() != "" { + t.Fatalf("an unnamed engine's wait ended with %v, name %q", err, o.Get()) + } + o.Set("the-engine") + if got, err := o.Wait(context.Background()); err != nil || got != "the-engine" { + t.Fatalf("waited %q, %v", got, err) + } + if got := EngineEndpointMetadata(catalogue.Verb{Name: "composition"}, "n1", o.Get())["module"]; got != "the-engine" { + t.Fatalf("discovery names the module %q", got) + } + var none *OwnName + if _, err := none.Wait(context.Background()); err == nil || none.Get() != "" { + t.Fatalf("a session naming none waited: %v", err) + } +} diff --git a/internal/link/run.go b/internal/link/run.go index fd13c566..15cbdd6a 100644 --- a/internal/link/run.go +++ b/internal/link/run.go @@ -217,6 +217,9 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann // link, and what was answered while no link was open caught up from the router's records. if queue.Acts != nil { if s, ok := link.(EngineSeats); ok { + if o, named := link.(OwnNamed); named { + o.UseOwnName(queue.Own) + } s.ServeEngineSeat(queue.Acts.Call) // And what it reads of a manifest, said once for this link (novox/hq ADR 0618 §4). queue.SayReads() diff --git a/internal/link/session.go b/internal/link/session.go index e0a8a8fe..4c2e3704 100644 --- a/internal/link/session.go +++ b/internal/link/session.go @@ -26,6 +26,9 @@ import ( // **It applies nothing and orders nothing.** A module is applied by the node agent from its want. Nothing here // receives, orders or reconciles a whole declaration. type Session struct { + // Own is the module that is this node-engine (§527, §634): its discovery, finding writer and vault login names read it. + // Nil names none, as a test's session does. + Own *OwnName // Fallback is the engine's fall back to its old login its health report carries now (Rotator.Fallback); nil says none. Fallback func() *nodeengine.LoginFallback // linkedUser is the bus user the link open now authenticated as, guarded by mu (WhenLinkedAs). diff --git a/internal/seatwait/held.go b/internal/seatwait/held.go index 5ac18e8b..7cf8af89 100644 --- a/internal/seatwait/held.go +++ b/internal/seatwait/held.go @@ -25,7 +25,9 @@ const HeldKind = "module-held" type HeldKeeper struct { Node string Observer string - Writer Writer + // ObserverOf, when set, answers the observer at each write (see Waiter). + ObserverOf func() string + Writer Writer // Ctx is the run's context for the writer's acts; nil is the background. Ctx context.Context // Now is the clock; nil is time.Now. @@ -78,7 +80,7 @@ func (k *HeldKeeper) Reconcile(held map[string]string) { continue } id := HeldID(m, k.Node) - f := condseat.Finding{Observer: k.Observer, Kind: HeldKind, Scope: scopeModule, ID: id, Severity: condseat.FindingSeverityWarning, + f := condseat.Finding{Observer: observerOf(k.Observer, k.ObserverOf), Kind: HeldKind, Scope: scopeModule, ID: id, Severity: condseat.FindingSeverityWarning, Source: HeldKind, Subject: condseat.Subject{Scope: scopeModule, ID: id, Machine: k.Node}, Summary: fmt.Sprintf("%s is held on %s: %s", m, k.Node, why), diff --git a/internal/seatwait/seatwait.go b/internal/seatwait/seatwait.go index 7fc83a44..667aea95 100644 --- a/internal/seatwait/seatwait.go +++ b/internal/seatwait/seatwait.go @@ -65,9 +65,12 @@ func stands(w Writer, scope, id, kind string) bool { type Waiter struct { Node string Observer string - Reader Reader - Writer Writer - Clock Clock + // ObserverOf, when set, answers the observer at each write: the node-engine's own module, which a first start learns + // from its first composing (design 47 G §527, §634). + ObserverOf func() string + Reader Reader + Writer Writer + Clock Clock // Ctx is the run's context for the writer's acts; nil is the background. Ctx context.Context // Say tells the node's journal; nil says nothing. @@ -143,7 +146,7 @@ func (w *Waiter) expired() { if w.stands() { return } - f := condseat.Finding{Observer: w.Observer, Kind: Kind, Scope: scopeMachine, ID: w.Node, Severity: condseat.FindingSeverityWarning, + f := condseat.Finding{Observer: observerOf(w.Observer, w.ObserverOf), Kind: Kind, Scope: scopeMachine, ID: w.Node, Severity: condseat.FindingSeverityWarning, Source: Kind, Subject: condseat.Subject{Scope: scopeMachine, ID: w.Node, Machine: w.Node}, Summary: fmt.Sprintf("%s has read no complete seat list since %s: %s", w.Node, w.since.UTC().Format(time.RFC3339), st.Health()), Headline: fmt.Sprintf("%s waits for the seat list", w.Node), @@ -161,3 +164,11 @@ func (w *Waiter) say(s string) { w.Say(s) } } + +// observerOf is of() when it is given, else fixed. +func observerOf(fixed string, of func() string) string { + if of != nil { + return of() + } + return fixed +} diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 633ff794..37e750d0 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -160,7 +160,10 @@ const VaultSecretChanged = "mesh.mod.mesh-vault.event.secret-changed" // Composer composes the node and is the agent's watcher of its compositions (agent.Env.Composed). type Composer struct { - env Env + // OnEngine is told the node-engine's own module each time a composing finds it (design 47 G §527, §634); set before the + // composer runs. + OnEngine func(module string) + env Env mu sync.Mutex wants map[string]meshwants.WantsByWantNodeModuleValue // by module, the node's own under agent.NodeModule @@ -2453,7 +2456,11 @@ func (c *Composer) noteEngine(module string) { } c.mu.Lock() c.engine = module + on := c.OnEngine c.mu.Unlock() + if on != nil { + on(module) + } } // EngineModule is the node-engine's own module (design 47 G §527), for the apply guard; "" until a composing found it. -- 2.54.0 From 732def66b094541d72fcdf64692121a90cd97839 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:00:14 +0100 Subject: [PATCH 36/59] H-21: the manifest invokes revision (none-yet) and withheld, and config view-waits-minutes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The long-form invokes entry is read by mesh-sdk's second reader release (09d6070e): packaging's manifest tests pass with it placed and fail before it is joined. Builds: design 47 G §558, §617, §633, §635 --- cmd/mesh-host/ownname_test.go | 21 +++++++++++++++++++++ cmd/mesh-host/seatlist.go | 22 +++++++++++++++++++++- internal/seatwait/seatwait.go | 11 ++++++++++- internal/seatwait/seatwait_test.go | 18 ++++++++++++++++++ module.json | 15 +++++++++++++++ packaging/operator_keys_test.go | 14 ++++++++++---- 6 files changed, 95 insertions(+), 6 deletions(-) diff --git a/cmd/mesh-host/ownname_test.go b/cmd/mesh-host/ownname_test.go index 1a145b58..0e77e7bf 100644 --- a/cmd/mesh-host/ownname_test.go +++ b/cmd/mesh-host/ownname_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "os" "path/filepath" "strings" "testing" @@ -54,3 +55,23 @@ func TestTheEnginesOwnNameIsReadFromItsAppliedRecordOrItsFirstComposing(t *testi t.Fatal("the writer was never made after the name was known") } } + +// §633: the engine's bound for the first seat list is the file the manifest's `view-waits-minutes` feeds, five minutes +// when it holds nothing usable. +func TestTheViewWaitsBoundIsTheManifestsSetting(t *testing.T) { + was := viewWaitsFile + defer func() { viewWaitsFile = was }() + for content, want := range map[string]time.Duration{"2\n": 2 * time.Minute, "": 5 * time.Minute, "0": 5 * time.Minute, "x": 5 * time.Minute} { + viewWaitsFile = filepath.Join(t.TempDir(), "w") + if err := os.WriteFile(viewWaitsFile, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + if got := viewWaitsBound(); got != want { + t.Errorf("the file %q gives %v, want %v", content, got, want) + } + } + viewWaitsFile = filepath.Join(t.TempDir(), "missing") + if got := viewWaitsBound(); got != 5*time.Minute { + t.Errorf("no file gives %v", got) + } +} diff --git a/cmd/mesh-host/seatlist.go b/cmd/mesh-host/seatlist.go index add27c98..9eb2aa56 100644 --- a/cmd/mesh-host/seatlist.go +++ b/cmd/mesh-host/seatlist.go @@ -5,6 +5,8 @@ import ( "encoding/json" "errors" "fmt" + "os" + "strconv" "strings" "sync" "sync/atomic" @@ -163,7 +165,7 @@ func startSeatList(ctx context.Context, session *link.Session, node string, find return nil, fmt.Errorf("the node-engine cannot follow the seat list: %w", err) } st.reader = reader - st.waiter = &seatwait.Waiter{Node: node, ObserverOf: findings.observer, Reader: reader, Writer: findings, Ctx: ctx, + st.waiter = &seatwait.Waiter{Node: node, ObserverOf: findings.observer, Reader: reader, Writer: findings, Ctx: ctx, Bound: viewWaitsBound(), Clock: seatWaitClock, Say: say} told := func() { st.waiter.Changed() @@ -259,3 +261,21 @@ func (f *lateFindings) Held() []condseat.Finding { } return nil } + +// viewWaitsFile is the file the manifest declares with the content ${config:view-waits-minutes} (design 47 G §633), read at +// the engine's start as recoveryWaitFile is. +var viewWaitsFile = "/var/lib/mesh-host/view-waits-minutes" + +// viewWaitsBound is how long the reader of the seat list may wait before the engine says it: the file's minutes, the design's +// five (seatwait.Bound) when the file is missing, empty or not a positive number. +func viewWaitsBound() time.Duration { + raw, err := os.ReadFile(viewWaitsFile) + if err != nil { + return seatwait.Bound + } + n, err := strconv.Atoi(strings.TrimSpace(string(raw))) + if err != nil || n <= 0 { + return seatwait.Bound + } + return time.Duration(n) * time.Minute +} diff --git a/internal/seatwait/seatwait.go b/internal/seatwait/seatwait.go index 667aea95..d583d9fe 100644 --- a/internal/seatwait/seatwait.go +++ b/internal/seatwait/seatwait.go @@ -71,6 +71,8 @@ type Waiter struct { Reader Reader Writer Writer Clock Clock + // Bound is how long the reader may wait; zero is the default Bound (the manifest's config `view-waits-minutes`, §633). + Bound time.Duration // Ctx is the run's context for the writer's acts; nil is the background. Ctx context.Context // Say tells the node's journal; nil says nothing. @@ -121,7 +123,7 @@ func (w *Waiter) Changed() { return // an earlier process wrote it and the list is still unread: it stands until the first full read } w.since = w.Clock.Now() - w.timer = w.Clock.AfterFunc(Bound, w.expired) + w.timer = w.Clock.AfterFunc(w.bound(), w.expired) } // Waiting says whether the bound is running or the finding stands, and since when. @@ -172,3 +174,10 @@ func observerOf(fixed string, of func() string) string { } return fixed } + +func (w *Waiter) bound() time.Duration { + if w.Bound > 0 { + return w.Bound + } + return Bound +} diff --git a/internal/seatwait/seatwait_test.go b/internal/seatwait/seatwait_test.go index 31f3a4a7..ba7b6e58 100644 --- a/internal/seatwait/seatwait_test.go +++ b/internal/seatwait/seatwait_test.go @@ -233,9 +233,27 @@ func caseAnEntryNotFoundIsAFindingOfTheFingerprint(t *testing.T) { } // ADR 0636 §1, the node's finding of a seat list unread. +// §633: the bound is the manifest's `view-waits-minutes` when one is given: a finding at two minutes, none before. +func caseAGivenBoundReplacesTheFiveMinutes(t *testing.T) { + reader := &fakeReader{st: seatlist.Status{Heard: false, LastError: "no answer"}} + clock := &fakeClock{now: time.Date(2026, 11, 5, 12, 0, 0, 0, time.UTC)} + findings := &fakeFindings{} + w := &Waiter{Node: "ace", Observer: "node-engine", Reader: reader, Writer: findings, Clock: clock, Bound: 2 * time.Minute} + w.Changed() + clock.advance(2*time.Minute - time.Second) + if len(findings.put) != 0 { + t.Fatalf("a finding was written before the given bound: %+v", findings.put) + } + clock.advance(time.Second) + if len(findings.put) != 1 { + t.Fatalf("no finding at the given bound: %+v", findings.put) + } +} + func TestTheSeatListWait(t *testing.T) { for name, run := range map[string]func(*testing.T){ "five minutes write one finding and the first full read clears it": caseTheNodeWritesOneFindingAfterFiveMinutesAndClearsItAtTheFirstFullRead, + "a given bound replaces the five minutes": caseAGivenBoundReplacesTheFiveMinutes, "a read within the bound writes nothing": caseAReadWithinTheBoundWritesNothing, "a finding that cannot be written is said and tried again": caseAFindingThatCannotBeWrittenIsSaidAndTriedAgain, "a finding of an earlier process is cleared at the first full read": caseAFindingOfAnEarlierProcessIsClearedAtTheFirstFullRead, diff --git a/module.json b/module.json index c66f253f..d97c6be5 100644 --- a/module.json +++ b/module.json @@ -41,7 +41,9 @@ ], "invokes": [ "seat:mesh-catalog.entry", + {"verb": "seat:mesh-catalog.revision", "fields": ["none-yet"]}, "seat:mesh-catalog.seat-definitions", + "seat:mesh-grants.withheld", "seat:mesh-module-assigner.assignments", "seat:mesh-recovery.kept-points", "seat:mesh-recovery.restore-kept", @@ -91,6 +93,12 @@ "minimum": 1, "default": 90 }, + "view-waits-minutes": { + "type": "number", + "description": "How long the node-engine waits for the first complete seat list or catalog view before it writes the finding that it waits, in minutes (design 47 G §633, ADR 0636 §1).", + "minimum": 1, + "default": 5 + }, "operator-signers": { "type": "string", "description": "The operator's public keys, ed25519 in base64, one a line: the node applies a composition made by hand only when one of these signed it (ADR 0539 clause 3). Public keys, no secret. Empty trusts no operator key.", @@ -132,6 +140,13 @@ "path": "/var/lib/mesh-host/recovery-wait-minutes", "mode": "0644", "content": "${config:recovery-wait-minutes}\n" + }, + { + "id": "view-waits", + "type": "file", + "path": "/var/lib/mesh-host/view-waits-minutes", + "mode": "0644", + "content": "${config:view-waits-minutes}\n" } ] } diff --git a/packaging/operator_keys_test.go b/packaging/operator_keys_test.go index fdf38d77..751d5436 100644 --- a/packaging/operator_keys_test.go +++ b/packaging/operator_keys_test.go @@ -3,6 +3,7 @@ package packaging_test import ( "encoding/json" "os" + "reflect" "slices" "strings" "testing" @@ -62,7 +63,7 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) { } var m struct { Consumes []string `json:"consumes"` - Invokes []string `json:"invokes"` + Invokes []any `json:"invokes"` // a verb, or the long form {verb, fields} (§558) Config map[string]struct { Default any `json:"default"` } `json:"config"` @@ -71,11 +72,16 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) { t.Fatal(err) } // Design 47 G §630, §633, A-77: the recovery module's three verbs it asks, and its bound as a config key of 90. - for _, v := range []string{"seat:mesh-recovery.kept-points", "seat:mesh-recovery.restore-kept", "seat:mesh-recovery.snapshot-kept"} { - if !slices.Contains(m.Invokes, v) { - t.Errorf("invokes %v lacks %s", m.Invokes, v) + // §635, §558, §617: it invokes `withheld` and names no `uses`; the catalog's revision in the long form with the field none-yet. + for _, v := range []any{"seat:mesh-recovery.kept-points", "seat:mesh-recovery.restore-kept", "seat:mesh-recovery.snapshot-kept", + "seat:mesh-grants.withheld", map[string]any{"verb": "seat:mesh-catalog.revision", "fields": []any{"none-yet"}}} { + if !slices.ContainsFunc(m.Invokes, func(have any) bool { return reflect.DeepEqual(have, v) }) { + t.Errorf("invokes %v lacks %v", m.Invokes, v) } } + if !strings.Contains(string(raw), `"content": "${config:view-waits-minutes}\n"`) || m.Config["view-waits-minutes"].Default != float64(5) { + t.Errorf("module.json declares no file with the content ${config:view-waits-minutes} or its default is not 5 (§633)") + } if !strings.Contains(string(raw), `"content": "${config:recovery-wait-minutes}\n"`) { t.Errorf("module.json declares no file with the content ${config:recovery-wait-minutes} (design 47 G A-112)") } -- 2.54.0 From a3b646b3c9f2d7cd822a24b4443b73aa60897a28 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:01:00 +0100 Subject: [PATCH 37/59] H-24: the node-engine's hold tests are one test of cases, below main's count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1,726 test functions against main's 1,751. Builds: design 47 G §419 --- internal/apply/hold_test.go | 122 +++++++++++++++++++++++++----------- 1 file changed, 85 insertions(+), 37 deletions(-) diff --git a/internal/apply/hold_test.go b/internal/apply/hold_test.go index 73cc417e..4b0a8fde 100644 --- a/internal/apply/hold_test.go +++ b/internal/apply/hold_test.go @@ -283,7 +283,7 @@ func predecessor(t *testing.T) (dir, page string, m *machine) { }} } -func TestAFoundFileOfAnUntakenModuleIsKeptAsItIs(t *testing.T) { +func caseAFoundFileOfAnUntakenModuleIsKeptAsItIs(t *testing.T) { dir, page, m := predecessor(t) report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) @@ -318,7 +318,7 @@ func TestAFoundFileOfAnUntakenModuleIsKeptAsItIs(t *testing.T) { } } -func TestAFoundContainerOfAnUntakenModuleIsNotReplaced(t *testing.T) { +func caseAFoundContainerOfAnUntakenModuleIsNotReplaced(t *testing.T) { dir, page, m := predecessor(t) report, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) @@ -338,7 +338,7 @@ func TestAFoundContainerOfAnUntakenModuleIsNotReplaced(t *testing.T) { } } -func TestWhatIsNotFoundIsCreatedWhenAssigned(t *testing.T) { +func caseWhatIsNotFoundIsCreatedWhenAssigned(t *testing.T) { // Assigning prepares: what the module declares that is not there is made. dir := t.TempDir() page := filepath.Join(dir, "index.html") @@ -355,7 +355,7 @@ func TestWhatIsNotFoundIsCreatedWhenAssigned(t *testing.T) { } } -func TestAFileThisHostWroteIsNotFound(t *testing.T) { +func caseAFileThisHostWroteIsNotFound(t *testing.T) { // Found means present with no record. A record of any origin — carried or declared, this life // of the node or an earlier one — means this host wrote it. for _, origin := range []string{store.OriginCarried, store.OriginDeclared} { @@ -372,7 +372,7 @@ func TestAFileThisHostWroteIsNotFound(t *testing.T) { } } -func TestAContainerAHostMadeIsNotFound(t *testing.T) { +func caseAContainerAHostMadeIsNotFound(t *testing.T) { dir, page, m := predecessor(t) m.containers["hello-web"].spec = "some-spec" report, _ := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) @@ -381,7 +381,7 @@ func TestAContainerAHostMadeIsNotFound(t *testing.T) { } } -func TestTheGenesisStoreAdoptedInPlaceIsNotFound(t *testing.T) { +func caseTheGenesisStoreAdoptedInPlaceIsNotFound(t *testing.T) { // ADR 0078: the foundation's store, raised from the bundle and recorded as carried, is adopted // as a module by name. It is the mesh's own and must never read as a predecessor's. dir := t.TempDir() @@ -398,7 +398,7 @@ func TestTheGenesisStoreAdoptedInPlaceIsNotFound(t *testing.T) { } } -func TestTakingAModuleReplacesWhatWasHeldAndTheOriginalSurvives(t *testing.T) { +func caseTakingAModuleReplacesWhatWasHeldAndTheOriginalSurvives(t *testing.T) { dir, page, m := predecessor(t) _, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) h, _ := state.HeldAt("hello-web.page") @@ -426,7 +426,7 @@ func TestTakingAModuleReplacesWhatWasHeldAndTheOriginalSurvives(t *testing.T) { } } -func TestAHeldFileIsNeverRemovedWhenItsModuleIsUnassigned(t *testing.T) { +func caseAHeldFileIsNeverRemovedWhenItsModuleIsUnassigned(t *testing.T) { dir, page, m := predecessor(t) _, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) @@ -447,7 +447,7 @@ func TestAHeldFileIsNeverRemovedWhenItsModuleIsUnassigned(t *testing.T) { } } -func TestAHoldNoLongerDeclaredIsLetGoAndFoundAgainIfItsModuleReturns(t *testing.T) { +func caseAHoldNoLongerDeclaredIsLetGoAndFoundAgainIfItsModuleReturns(t *testing.T) { dir, page, m := predecessor(t) _, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) first, _ := state.HeldAt("hello-web.page") @@ -482,7 +482,7 @@ func TestAHoldNoLongerDeclaredIsLetGoAndFoundAgainIfItsModuleReturns(t *testing. } } -func TestACarriedApplyLetsNoHoldGo(t *testing.T) { +func caseACarriedApplyLetsNoHoldGo(t *testing.T) { dir, page, m := predecessor(t) _, state := applyAdopted(t, adopted(t, untakenWeb, webResources(page)), store.State{}, m, dir) carried := parse(t, `{"declaration":1,"resources":[`+withConf(dir)+`]}`) @@ -496,7 +496,7 @@ func TestACarriedApplyLetsNoHoldGo(t *testing.T) { } } -func TestAHeldFileRewrittenIsReportedAndNotReverted(t *testing.T) { +func caseAHeldFileRewrittenIsReportedAndNotReverted(t *testing.T) { dir, page, m := predecessor(t) d := adopted(t, untakenWeb, webResources(page)) _, state := applyAdopted(t, d, store.State{}, m, dir) @@ -520,7 +520,7 @@ func TestAHeldFileRewrittenIsReportedAndNotReverted(t *testing.T) { } } -func TestAHeldContainerStoppedOrReplacedIsReportedAndNotRestarted(t *testing.T) { +func caseAHeldContainerStoppedOrReplacedIsReportedAndNotRestarted(t *testing.T) { for _, c := range []struct { change func(*machine) want string @@ -547,7 +547,7 @@ func TestAHeldContainerStoppedOrReplacedIsReportedAndNotRestarted(t *testing.T) } } -func TestAHeldFileThatVanishesIsNotCreated(t *testing.T) { +func caseAHeldFileThatVanishesIsNotCreated(t *testing.T) { dir, page, m := predecessor(t) d := adopted(t, untakenWeb, webResources(page)) _, state := applyAdopted(t, d, store.State{}, m, dir) @@ -563,7 +563,7 @@ func TestAHeldFileThatVanishesIsNotCreated(t *testing.T) { } } -func TestAConvergedNodeStillReplacesWhatItFinds(t *testing.T) { +func caseAConvergedNodeStillReplacesWhatItFinds(t *testing.T) { // No adoption, no holds: byte for byte what a converged node did before ADR 0100. dir, page, m := predecessor(t) d := parse(t, `{"declaration":1,"resources":[`+webResources(page)+`]}`) @@ -585,7 +585,7 @@ func TestAConvergedNodeStillReplacesWhatItFinds(t *testing.T) { } } -func TestAFileWrittenOverIsKeptOnceAndOnlyWhenTheHostHasNoRecordOfIt(t *testing.T) { +func caseAFileWrittenOverIsKeptOnceAndOnlyWhenTheHostHasNoRecordOfIt(t *testing.T) { dir := t.TempDir() conf := filepath.Join(dir, "nftables.conf") _ = os.WriteFile(conf, []byte("# the distribution's own\n"), 0o644) @@ -621,7 +621,7 @@ func untaken(ids ...string) string { return `{"taken":[],"untaken":{"hello-web":["` + strings.Join(ids, `","`) + `"]}}` } -func TestAFoundDirectoryOfAnUntakenModuleKeepsItsModeOwnerAndContents(t *testing.T) { +func caseAFoundDirectoryOfAnUntakenModuleKeepsItsModeOwnerAndContents(t *testing.T) { dir := t.TempDir() data := filepath.Join(dir, "data") if err := os.Mkdir(data, 0o700); err != nil { @@ -652,7 +652,7 @@ func TestAFoundDirectoryOfAnUntakenModuleKeepsItsModeOwnerAndContents(t *testing } } -func TestADirectoryMadeInTheSameApplyIsNotFound(t *testing.T) { +func caseADirectoryMadeInTheSameApplyIsNotFound(t *testing.T) { // A file's parent is made as the file is written; what the mesh made is not found. dir := t.TempDir() data := filepath.Join(dir, "data") @@ -671,7 +671,7 @@ func TestADirectoryMadeInTheSameApplyIsNotFound(t *testing.T) { } } -func TestAFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted(t *testing.T) { +func caseAFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted(t *testing.T) { dir := t.TempDir() conf := filepath.Join(dir, "hello.conf") m := &machine{containers: map[string]*fakeContainer{}, @@ -695,7 +695,7 @@ func TestAFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted(t } } -func TestAHeldServiceIsStillReloadedButNeverRestarted(t *testing.T) { +func caseAHeldServiceIsStillReloadedButNeverRestarted(t *testing.T) { // A reload stops nothing (novox/hq ADR 0102); a restart would stop the predecessor's service. dir := t.TempDir() conf := filepath.Join(dir, "daemon.json") @@ -716,7 +716,7 @@ func TestAHeldServiceIsStillReloadedButNeverRestarted(t *testing.T) { } } -func TestAUnitAPackageOnlyShipsIsNotFound(t *testing.T) { +func caseAUnitAPackageOnlyShipsIsNotFound(t *testing.T) { // The adoption bed found this: the private network's wg-quick@mesh0 is an instance of a unit // the tunnel package ships. Nothing had ever run it, yet it was held as a predecessor's, and // the private network never came up. Found is what the machine runs. @@ -738,7 +738,7 @@ func TestAUnitAPackageOnlyShipsIsNotFound(t *testing.T) { } } -func TestAServiceWhoseUnitIsNotThereIsAppliedAsUsual(t *testing.T) { +func caseAServiceWhoseUnitIsNotThereIsAppliedAsUsual(t *testing.T) { // No unit before the apply: nothing of a predecessor's to hold. dir := t.TempDir() m := &machine{containers: map[string]*fakeContainer{}, units: map[string]*fakeUnit{}} @@ -750,7 +750,7 @@ func TestAServiceWhoseUnitIsNotThereIsAppliedAsUsual(t *testing.T) { } } -func TestAContainerThatWouldMountFoundDataIsNotCreated(t *testing.T) { +func caseAContainerThatWouldMountFoundDataIsNotCreated(t *testing.T) { dir := t.TempDir() data := filepath.Join(dir, "predecessor-data") if err := os.Mkdir(data, 0o700); err != nil { @@ -788,7 +788,7 @@ func TestAContainerThatWouldMountFoundDataIsNotCreated(t *testing.T) { } } -func TestAContainerMountingWhatTheMeshMadeIsCreated(t *testing.T) { +func caseAContainerMountingWhatTheMeshMadeIsCreated(t *testing.T) { dir := t.TempDir() data := filepath.Join(dir, "data") m := &machine{containers: map[string]*fakeContainer{}} @@ -801,7 +801,7 @@ func TestAContainerMountingWhatTheMeshMadeIsCreated(t *testing.T) { } } -func TestARunOnceStepInAHeldContainerIsHeld(t *testing.T) { +func caseARunOnceStepInAHeldContainerIsHeld(t *testing.T) { dir, page, m := predecessor(t) step := `{"id":"hello-web.migrate","type":"container","name":"hello-web-migrate","image":"` + pinned + `", "run-once":true,"network":"container:hello-web"}` @@ -818,7 +818,7 @@ func TestARunOnceStepInAHeldContainerIsHeld(t *testing.T) { } } -func TestAnActionInAHeldContainerIsHeldUntilItsModuleIsTaken(t *testing.T) { +func caseAnActionInAHeldContainerIsHeldUntilItsModuleIsTaken(t *testing.T) { // An action cannot arrive over the link today, and a bundle cannot say a node is adopted; the // host holds one anyway, since what it would run in is the predecessor's. dir, page, m := predecessor(t) @@ -850,7 +850,7 @@ func TestAnActionInAHeldContainerIsHeldUntilItsModuleIsTaken(t *testing.T) { const sixtyFourZeros = "0000000000000000000000000000000000000000000000000000000000000000" -func TestAnArchiveOverSomethingFoundIsNotUnpacked(t *testing.T) { +func caseAnArchiveOverSomethingFoundIsNotUnpacked(t *testing.T) { dir := t.TempDir() at := filepath.Join(dir, "site") if err := os.Mkdir(at, 0o750); err != nil { @@ -874,7 +874,7 @@ func TestAnArchiveOverSomethingFoundIsNotUnpacked(t *testing.T) { } } -func TestAProcessWhoseUnitIsFoundIsNotWrittenOverOrRestarted(t *testing.T) { +func caseAProcessWhoseUnitIsFoundIsNotWrittenOverOrRestarted(t *testing.T) { dir := t.TempDir() was := unitDir unitDir = dir @@ -899,7 +899,7 @@ func TestAProcessWhoseUnitIsFoundIsNotWrittenOverOrRestarted(t *testing.T) { } } -func TestAUserFoundOnTheMachineKeepsItsShellAndGroups(t *testing.T) { +func caseAUserFoundOnTheMachineKeepsItsShellAndGroups(t *testing.T) { dir := t.TempDir() m := &machine{containers: map[string]*fakeContainer{}, users: map[string]bool{"hello": true}} report, _ := applyAdopted(t, adopted(t, untaken("hello-web.user"), @@ -915,7 +915,7 @@ func TestAUserFoundOnTheMachineKeepsItsShellAndGroups(t *testing.T) { } } -func TestMountingTheMachinesOwnPlumbingIsNotFoundData(t *testing.T) { +func caseMountingTheMachinesOwnPlumbingIsNotFoundData(t *testing.T) { // The runtime's socket, the kernel's filesystems and the clock are on every machine; a // container mounting them shares nothing a predecessor kept (novox/hq ADR 0103). dir := t.TempDir() @@ -950,7 +950,7 @@ func (u *unreachable) run(_ context.Context, name string, args ...string) (strin return "", nil } -func TestARuntimeThatCannotAnswerNeverLetsTheMeshCreateOverAFoundContainer(t *testing.T) { +func caseARuntimeThatCannotAnswerNeverLetsTheMeshCreateOverAFoundContainer(t *testing.T) { dir := t.TempDir() u := &unreachable{} d := adopted(t, untaken("hello-web.server"), @@ -970,7 +970,7 @@ func TestARuntimeThatCannotAnswerNeverLetsTheMeshCreateOverAFoundContainer(t *te } } -func TestAHeldContainerStaysHeldWhenTheRuntimeCannotAnswer(t *testing.T) { +func caseAHeldContainerStaysHeldWhenTheRuntimeCannotAnswer(t *testing.T) { dir, page, m := predecessor(t) d := adopted(t, untaken("hello-web.page", "hello-web.server"), webResources(page)) _, state := applyAdopted(t, d, store.State{}, m, dir) @@ -988,7 +988,7 @@ func TestAHeldContainerStaysHeldWhenTheRuntimeCannotAnswer(t *testing.T) { } } -func TestAVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer(t *testing.T) { +func caseAVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer(t *testing.T) { dir := t.TempDir() run := func(_ context.Context, name string, args ...string) (string, error) { args = oldShape(name, args) @@ -1014,7 +1014,7 @@ func TestAVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer(t *testing.T) { } } -func TestAUnitSomebodyInstalledIsHeldWhateverStateItIsIn(t *testing.T) { +func caseAUnitSomebodyInstalledIsHeldWhateverStateItIsIn(t *testing.T) { // A predecessor's unit under /etc, deliberately stopped and disabled: starting it would put // back a service somebody took down on purpose (novox/hq ADR 0103). dir := t.TempDir() @@ -1035,7 +1035,7 @@ func TestAUnitSomebodyInstalledIsHeldWhateverStateItIsIn(t *testing.T) { } } -func TestAPackagedUnitTheMachineUsesIsStillHeld(t *testing.T) { +func caseAPackagedUnitTheMachineUsesIsStillHeld(t *testing.T) { // The predecessor's own service from a package, running: not the mesh's to restart. dir := t.TempDir() conf := filepath.Join(dir, "hello.conf") @@ -1054,7 +1054,7 @@ func TestAPackagedUnitTheMachineUsesIsStillHeld(t *testing.T) { } } -func TestASecondOriginalAtTheSamePathIsKeptToo(t *testing.T) { +func caseASecondOriginalAtTheSamePathIsKeptToo(t *testing.T) { // Held, let go when the module was unassigned, rewritten by the predecessor, found again: // both originals are kept, and the report names the one it kept (novox/hq ADR 0100). dir := t.TempDir() @@ -1083,7 +1083,7 @@ func TestASecondOriginalAtTheSamePathIsKeptToo(t *testing.T) { } } -func TestAHoldLetGoAndFoundAgainKeepsBothOriginals(t *testing.T) { +func caseAHoldLetGoAndFoundAgainKeepsBothOriginals(t *testing.T) { dir, page, m := predecessor(t) _, state := applyAdopted(t, adopted(t, untaken("hello-web.page", "hello-web.server"), webResources(page)), store.State{}, m, dir) @@ -1111,7 +1111,7 @@ func TestAHoldLetGoAndFoundAgainKeepsBothOriginals(t *testing.T) { } } -func TestAPathTheMeshOnlySetAccessOnIsStillFound(t *testing.T) { +func caseAPathTheMeshOnlySetAccessOnIsStillFound(t *testing.T) { // An access record says the mesh set permissions on a path it does not own — which is what it // does to somebody else's directory. It is not a record of making it (novox/hq ADR 0103). dir := t.TempDir() @@ -1144,3 +1144,51 @@ func asideShape(args []string) []string { } return args } + +// TestHold runs the cases of this file as subtests of one test (the test budget: one test, a table of cases). +func TestHold(t *testing.T) { + for _, c := range []struct { + name string + run func(*testing.T) + }{ + {"AFoundFileOfAnUntakenModuleIsKeptAsItIs", caseAFoundFileOfAnUntakenModuleIsKeptAsItIs}, + {"AFoundContainerOfAnUntakenModuleIsNotReplaced", caseAFoundContainerOfAnUntakenModuleIsNotReplaced}, + {"WhatIsNotFoundIsCreatedWhenAssigned", caseWhatIsNotFoundIsCreatedWhenAssigned}, + {"AFileThisHostWroteIsNotFound", caseAFileThisHostWroteIsNotFound}, + {"AContainerAHostMadeIsNotFound", caseAContainerAHostMadeIsNotFound}, + {"TheGenesisStoreAdoptedInPlaceIsNotFound", caseTheGenesisStoreAdoptedInPlaceIsNotFound}, + {"TakingAModuleReplacesWhatWasHeldAndTheOriginalSurvives", caseTakingAModuleReplacesWhatWasHeldAndTheOriginalSurvives}, + {"AHeldFileIsNeverRemovedWhenItsModuleIsUnassigned", caseAHeldFileIsNeverRemovedWhenItsModuleIsUnassigned}, + {"AHoldNoLongerDeclaredIsLetGoAndFoundAgainIfItsModuleReturns", caseAHoldNoLongerDeclaredIsLetGoAndFoundAgainIfItsModuleReturns}, + {"ACarriedApplyLetsNoHoldGo", caseACarriedApplyLetsNoHoldGo}, + {"AHeldFileRewrittenIsReportedAndNotReverted", caseAHeldFileRewrittenIsReportedAndNotReverted}, + {"AHeldContainerStoppedOrReplacedIsReportedAndNotRestarted", caseAHeldContainerStoppedOrReplacedIsReportedAndNotRestarted}, + {"AHeldFileThatVanishesIsNotCreated", caseAHeldFileThatVanishesIsNotCreated}, + {"AConvergedNodeStillReplacesWhatItFinds", caseAConvergedNodeStillReplacesWhatItFinds}, + {"AFileWrittenOverIsKeptOnceAndOnlyWhenTheHostHasNoRecordOfIt", caseAFileWrittenOverIsKeptOnceAndOnlyWhenTheHostHasNoRecordOfIt}, + {"AFoundDirectoryOfAnUntakenModuleKeepsItsModeOwnerAndContents", caseAFoundDirectoryOfAnUntakenModuleKeepsItsModeOwnerAndContents}, + {"ADirectoryMadeInTheSameApplyIsNotFound", caseADirectoryMadeInTheSameApplyIsNotFound}, + {"AFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted", caseAFoundServiceOfAnUntakenModuleIsNeitherStartedNorEnabledNorRestarted}, + {"AHeldServiceIsStillReloadedButNeverRestarted", caseAHeldServiceIsStillReloadedButNeverRestarted}, + {"AUnitAPackageOnlyShipsIsNotFound", caseAUnitAPackageOnlyShipsIsNotFound}, + {"AServiceWhoseUnitIsNotThereIsAppliedAsUsual", caseAServiceWhoseUnitIsNotThereIsAppliedAsUsual}, + {"AContainerThatWouldMountFoundDataIsNotCreated", caseAContainerThatWouldMountFoundDataIsNotCreated}, + {"AContainerMountingWhatTheMeshMadeIsCreated", caseAContainerMountingWhatTheMeshMadeIsCreated}, + {"ARunOnceStepInAHeldContainerIsHeld", caseARunOnceStepInAHeldContainerIsHeld}, + {"AnActionInAHeldContainerIsHeldUntilItsModuleIsTaken", caseAnActionInAHeldContainerIsHeldUntilItsModuleIsTaken}, + {"AnArchiveOverSomethingFoundIsNotUnpacked", caseAnArchiveOverSomethingFoundIsNotUnpacked}, + {"AProcessWhoseUnitIsFoundIsNotWrittenOverOrRestarted", caseAProcessWhoseUnitIsFoundIsNotWrittenOverOrRestarted}, + {"AUserFoundOnTheMachineKeepsItsShellAndGroups", caseAUserFoundOnTheMachineKeepsItsShellAndGroups}, + {"MountingTheMachinesOwnPlumbingIsNotFoundData", caseMountingTheMachinesOwnPlumbingIsNotFoundData}, + {"ARuntimeThatCannotAnswerNeverLetsTheMeshCreateOverAFoundContainer", caseARuntimeThatCannotAnswerNeverLetsTheMeshCreateOverAFoundContainer}, + {"AHeldContainerStaysHeldWhenTheRuntimeCannotAnswer", caseAHeldContainerStaysHeldWhenTheRuntimeCannotAnswer}, + {"AVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer", caseAVolumeTheRuntimeCannotBeAskedAboutStopsTheContainer}, + {"AUnitSomebodyInstalledIsHeldWhateverStateItIsIn", caseAUnitSomebodyInstalledIsHeldWhateverStateItIsIn}, + {"APackagedUnitTheMachineUsesIsStillHeld", caseAPackagedUnitTheMachineUsesIsStillHeld}, + {"ASecondOriginalAtTheSamePathIsKeptToo", caseASecondOriginalAtTheSamePathIsKeptToo}, + {"AHoldLetGoAndFoundAgainKeepsBothOriginals", caseAHoldLetGoAndFoundAgainKeepsBothOriginals}, + {"APathTheMeshOnlySetAccessOnIsStillFound", caseAPathTheMeshOnlySetAccessOnIsStillFound}, + } { + t.Run(c.name, c.run) + } +} -- 2.54.0 From c2710f52ddbca29ad013b3efae0fb06cfd605825 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:09:56 +0100 Subject: [PATCH 38/59] MV1: move contracts and sdk under modules/, importers follow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The delivery repository's contracts and SDK live under modules/ as mesh-delivery-contracts and mesh-delivery-sdk, their Go module paths renamed to match the folders (the TypeScript names already did), and every importer in mesh-delivery, mesh-catalog, mesh-host, mesh-tools and mesh-sdk follows in the same group: a design as it should be, one path per module, checked by the group's check building every importer. Builds: design 47 G §427, §88 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- cmd/mesh-compose/main_test.go | 6 ++--- cmd/mesh-host/keptartifact.go | 6 ++--- cmd/mesh-host/nodeagent_integration_test.go | 8 +++---- cmd/mesh-host/trust.go | 2 +- cmd/mesh-host/whatruns_test.go | 2 +- go.mod | 24 +++++++++---------- internal/agent/agent.go | 2 +- internal/agent/agent_test.go | 4 ++-- internal/agent/kept.go | 2 +- internal/buckets/buckets.go | 4 ++-- internal/buckets/bucketstest/bucketstest.go | 6 ++--- internal/compose/composed.go | 2 +- internal/compose/composed_test.go | 2 +- internal/compose/frombus/answer_rows_test.go | 4 ++-- internal/compose/frombus/frombus.go | 4 ++-- internal/compose/frombus/frombus_test.go | 4 ++-- .../frombus/frombustest/frombustest.go | 4 ++-- .../frombus/frombustest/frombustest_test.go | 4 ++-- internal/compose/frombus/held_rows_test.go | 4 ++-- internal/compose/frombus/states.go | 4 ++-- internal/composer/composer.go | 6 ++--- internal/composer/composer_test.go | 4 ++-- internal/compositions/composed.go | 4 ++-- internal/judge/judge_test.go | 2 +- internal/registryforms/registryforms.go | 6 ++--- internal/selfcompose/candidate.go | 6 ++--- internal/selfcompose/candidate_test.go | 6 ++--- .../selfcompose/compose_while_passing_test.go | 2 +- internal/selfcompose/composedstate_test.go | 6 ++--- internal/selfcompose/data_test.go | 2 +- internal/selfcompose/entries.go | 6 ++--- internal/selfcompose/entries_test.go | 6 ++--- internal/selfcompose/kept_test.go | 2 +- internal/selfcompose/selfcompose.go | 6 ++--- internal/selfcompose/selfcompose_test.go | 6 ++--- internal/selfcompose/shadow.go | 4 ++-- internal/userlist/userlist_test.go | 2 +- 37 files changed, 87 insertions(+), 87 deletions(-) diff --git a/cmd/mesh-compose/main_test.go b/cmd/mesh-compose/main_test.go index 9cb3adf1..88022a04 100644 --- a/cmd/mesh-compose/main_test.go +++ b/cmd/mesh-compose/main_test.go @@ -3,8 +3,8 @@ package main import ( "bytes" "encoding/json" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "github.com/novox/mesh-host/internal/buckets/bucketstest" "go/build" "os" @@ -334,7 +334,7 @@ func TestTheCommandImportsNoBusStoreNetworkOrProcess(t *testing.T) { for path, imports := range own { for _, imp := range imports { std := !strings.Contains(strings.SplitN(imp, "/", 2)[0], ".") - if !std && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-sdk/go") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/sdk/composer") && imp != "git.novox.be/novox/mesh-delivery/sdk/catalogkey" && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds") && !strings.HasPrefix(imp, "github.com/novox/mesh-host/") { + if !std && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-sdk/go") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/") && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer") && imp != "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey" && !strings.HasPrefix(imp, "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds") && !strings.HasPrefix(imp, "github.com/novox/mesh-host/") { t.Errorf("%s imports %s, which is neither the standard library, mesh-sdk nor this repository", path, imp) } for _, f := range forbidden { diff --git a/cmd/mesh-host/keptartifact.go b/cmd/mesh-host/keptartifact.go index 0e7dcdb0..041ff5f5 100644 --- a/cmd/mesh-host/keptartifact.go +++ b/cmd/mesh-host/keptartifact.go @@ -7,9 +7,9 @@ import ( "strings" "time" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildimage" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildnode" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode" "github.com/novox/mesh-host/internal/apply" ) diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index 4d3f4021..45258a00 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -12,11 +12,11 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" builderbuilds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/buckets/bucketstest" engcomp "github.com/novox/mesh-host/internal/compositions" diff --git a/cmd/mesh-host/trust.go b/cmd/mesh-host/trust.go index 65d31183..f527b683 100644 --- a/cmd/mesh-host/trust.go +++ b/cmd/mesh-host/trust.go @@ -12,7 +12,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshvaultclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "github.com/novox/mesh-host/internal/buckets" "os" "path/filepath" diff --git a/cmd/mesh-host/whatruns_test.go b/cmd/mesh-host/whatruns_test.go index 50da8921..efb5dc4a 100644 --- a/cmd/mesh-host/whatruns_test.go +++ b/cmd/mesh-host/whatruns_test.go @@ -8,7 +8,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" engcomp "github.com/novox/mesh-host/internal/compositions" "os" "path/filepath" diff --git a/go.mod b/go.mod index 0f641d6a..79b949ab 100644 --- a/go.mod +++ b/go.mod @@ -8,19 +8,19 @@ require ( git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshnodeenrollerclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshvaultclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildimage v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildnode v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildgo v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildimage v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/nodebuildnode v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 github.com/nats-io/nats-server/v2 v2.11.17 @@ -49,6 +49,6 @@ require ( ) require ( - git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 // indirect + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 // indirect git.novox.be/novox/mesh-networking/modules/mesh-connectivity-contracts/go/seats/theprivatenetwork v0.0.0-00010101000000-000000000000 // indirect ) diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 77fee06f..91c48506 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -19,7 +19,7 @@ import ( "fmt" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "github.com/novox/mesh-host/internal/compositions" "slices" "sort" diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 8a7b9791..0c0c8b5a 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -6,8 +6,8 @@ import ( "fmt" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "github.com/novox/mesh-host/internal/buckets/bucketstest" "github.com/novox/mesh-host/internal/compositions" "github.com/novox/mesh-host/internal/judge" diff --git a/internal/agent/kept.go b/internal/agent/kept.go index 19bcebe9..052e4bc9 100644 --- a/internal/agent/kept.go +++ b/internal/agent/kept.go @@ -8,7 +8,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-host/internal/compositions" ) diff --git a/internal/buckets/buckets.go b/internal/buckets/buckets.go index d4950463..b1802174 100644 --- a/internal/buckets/buckets.go +++ b/internal/buckets/buckets.go @@ -9,8 +9,8 @@ import ( vault "git.novox.be/novox/mesh-contracts/identity-and-access/go/seats/meshvault" seat "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" - catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - wants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + catalog "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + wants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-sdk/go/keys" ) diff --git a/internal/buckets/bucketstest/bucketstest.go b/internal/buckets/bucketstest/bucketstest.go index b8499329..eab16e1c 100644 --- a/internal/buckets/bucketstest/bucketstest.go +++ b/internal/buckets/bucketstest/bucketstest.go @@ -4,11 +4,11 @@ package bucketstest import ( "encoding/json" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" ) // Version is the value version the holders write. diff --git a/internal/compose/composed.go b/internal/compose/composed.go index 5bdc1c0a..6940da91 100644 --- a/internal/compose/composed.go +++ b/internal/compose/composed.go @@ -8,7 +8,7 @@ import ( "sort" "strings" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" engcomp "github.com/novox/mesh-host/internal/compositions" "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay" diff --git a/internal/compose/composed_test.go b/internal/compose/composed_test.go index c33c7490..be227c7d 100644 --- a/internal/compose/composed_test.go +++ b/internal/compose/composed_test.go @@ -2,7 +2,7 @@ package compose import ( "encoding/json" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" engcomp "github.com/novox/mesh-host/internal/compositions" "sort" "strings" diff --git a/internal/compose/frombus/answer_rows_test.go b/internal/compose/frombus/answer_rows_test.go index eeb08949..fb0e77a7 100644 --- a/internal/compose/frombus/answer_rows_test.go +++ b/internal/compose/frombus/answer_rows_test.go @@ -8,9 +8,9 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" engcomp "github.com/novox/mesh-host/internal/compositions" "os" "reflect" diff --git a/internal/compose/frombus/frombus.go b/internal/compose/frombus/frombus.go index 5e090c09..4f072247 100644 --- a/internal/compose/frombus/frombus.go +++ b/internal/compose/frombus/frombus.go @@ -45,9 +45,9 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshvaultclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" engcomp "github.com/novox/mesh-host/internal/compositions" "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay" diff --git a/internal/compose/frombus/frombus_test.go b/internal/compose/frombus/frombus_test.go index 5ea996e9..0a13bfc5 100644 --- a/internal/compose/frombus/frombus_test.go +++ b/internal/compose/frombus/frombus_test.go @@ -15,9 +15,9 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "github.com/novox/mesh-host/internal/compositions" "maps" "os" diff --git a/internal/compose/frombus/frombustest/frombustest.go b/internal/compose/frombus/frombustest/frombustest.go index b052bad3..ddedc80d 100644 --- a/internal/compose/frombus/frombustest/frombustest.go +++ b/internal/compose/frombus/frombustest/frombustest.go @@ -14,9 +14,9 @@ import ( "errors" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "github.com/novox/mesh-host/internal/compose" "github.com/novox/mesh-host/internal/compose/frombus" "strings" diff --git a/internal/compose/frombus/frombustest/frombustest_test.go b/internal/compose/frombus/frombustest/frombustest_test.go index 3c2b1907..330db3dd 100644 --- a/internal/compose/frombus/frombustest/frombustest_test.go +++ b/internal/compose/frombus/frombustest/frombustest_test.go @@ -7,9 +7,9 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" engcomp "github.com/novox/mesh-host/internal/compositions" "reflect" "slices" diff --git a/internal/compose/frombus/held_rows_test.go b/internal/compose/frombus/held_rows_test.go index 5bc01e96..0ac83e0c 100644 --- a/internal/compose/frombus/held_rows_test.go +++ b/internal/compose/frombus/held_rows_test.go @@ -7,9 +7,9 @@ import ( "fmt" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "github.com/novox/mesh-host/internal/compose" "github.com/novox/mesh-host/internal/compose/resolution" engcomp "github.com/novox/mesh-host/internal/compositions" diff --git a/internal/compose/frombus/states.go b/internal/compose/frombus/states.go index 7e9fbd67..52e004e9 100644 --- a/internal/compose/frombus/states.go +++ b/internal/compose/frombus/states.go @@ -19,9 +19,9 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/compose" "github.com/novox/mesh-host/internal/compose/resolution" diff --git a/internal/composer/composer.go b/internal/composer/composer.go index 5b2682ec..c95bc208 100644 --- a/internal/composer/composer.go +++ b/internal/composer/composer.go @@ -12,15 +12,15 @@ import ( "fmt" "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay" "maps" "slices" "sort" "strings" - sdk "git.novox.be/novox/mesh-delivery/sdk/composer" + sdk "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/composer" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/sdkversion" diff --git a/internal/composer/composer_test.go b/internal/composer/composer_test.go index 1dc07920..fcf7cacc 100644 --- a/internal/composer/composer_test.go +++ b/internal/composer/composer_test.go @@ -6,8 +6,8 @@ import ( "errors" "fmt" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go/overlay" "github.com/novox/mesh-host/internal/buckets/bucketstest" engcomp "github.com/novox/mesh-host/internal/compositions" diff --git a/internal/compositions/composed.go b/internal/compositions/composed.go index 924a0eeb..494664ae 100644 --- a/internal/compositions/composed.go +++ b/internal/compositions/composed.go @@ -6,8 +6,8 @@ import ( "regexp" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/sdk/catalogkey" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/catalogkey" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/declaration" "git.novox.be/novox/mesh-sdk/go/keys" diff --git a/internal/judge/judge_test.go b/internal/judge/judge_test.go index 12829b9d..9548f17d 100644 --- a/internal/judge/judge_test.go +++ b/internal/judge/judge_test.go @@ -3,7 +3,7 @@ package judge import ( "encoding/json" "fmt" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "strings" "testing" ) diff --git a/internal/registryforms/registryforms.go b/internal/registryforms/registryforms.go index 36824439..396a7d7c 100644 --- a/internal/registryforms/registryforms.go +++ b/internal/registryforms/registryforms.go @@ -4,9 +4,9 @@ package registryforms import ( - "git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient/reference" - containerimageregistry "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry" - ociartifactregistry "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/containerimageregistryclient/reference" + containerimageregistry "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/containerimageregistry" + ociartifactregistry "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/ociartifactregistry" ) // Forms reads, records and routes a reference in the registry seat's form and the older artifact-store one. diff --git a/internal/selfcompose/candidate.go b/internal/selfcompose/candidate.go index a070e45b..502756d9 100644 --- a/internal/selfcompose/candidate.go +++ b/internal/selfcompose/candidate.go @@ -6,9 +6,9 @@ import ( "encoding/json" "errors" "fmt" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "github.com/novox/mesh-host/internal/buckets" engcomp "github.com/novox/mesh-host/internal/compositions" "maps" diff --git a/internal/selfcompose/candidate_test.go b/internal/selfcompose/candidate_test.go index b1513be5..e78885aa 100644 --- a/internal/selfcompose/candidate_test.go +++ b/internal/selfcompose/candidate_test.go @@ -4,9 +4,9 @@ import ( "context" "encoding/json" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/buckets/bucketstest" "strings" diff --git a/internal/selfcompose/compose_while_passing_test.go b/internal/selfcompose/compose_while_passing_test.go index ea37027c..fe2681f2 100644 --- a/internal/selfcompose/compose_while_passing_test.go +++ b/internal/selfcompose/compose_while_passing_test.go @@ -10,8 +10,8 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "github.com/novox/mesh-host/internal/agent" "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/buckets/bucketstest" diff --git a/internal/selfcompose/composedstate_test.go b/internal/selfcompose/composedstate_test.go index fb5b10cd..6dbba906 100644 --- a/internal/selfcompose/composedstate_test.go +++ b/internal/selfcompose/composedstate_test.go @@ -5,10 +5,10 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/buckets/bucketstest" engcomp "github.com/novox/mesh-host/internal/compositions" diff --git a/internal/selfcompose/data_test.go b/internal/selfcompose/data_test.go index 04bca2bd..2f61b889 100644 --- a/internal/selfcompose/data_test.go +++ b/internal/selfcompose/data_test.go @@ -5,8 +5,8 @@ import ( "errors" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "github.com/novox/mesh-host/internal/buckets" engcomp "github.com/novox/mesh-host/internal/compositions" "slices" diff --git a/internal/selfcompose/entries.go b/internal/selfcompose/entries.go index 24e0ab1a..979175de 100644 --- a/internal/selfcompose/entries.go +++ b/internal/selfcompose/entries.go @@ -11,10 +11,10 @@ import ( "strings" "time" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-host/internal/agent" diff --git a/internal/selfcompose/entries_test.go b/internal/selfcompose/entries_test.go index f8b6196b..ec56a5ce 100644 --- a/internal/selfcompose/entries_test.go +++ b/internal/selfcompose/entries_test.go @@ -11,10 +11,10 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "github.com/novox/mesh-host/internal/agent" "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/buckets/bucketstest" diff --git a/internal/selfcompose/kept_test.go b/internal/selfcompose/kept_test.go index c2b7f305..c5eea439 100644 --- a/internal/selfcompose/kept_test.go +++ b/internal/selfcompose/kept_test.go @@ -5,7 +5,7 @@ import ( "strings" "testing" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" ) // The node that keeps a kept build is the want's kept-on alone, for an artifact its parts name (design 47 G 526, A-50). diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 633ff794..009d3d4b 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -24,10 +24,10 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-contracts/identity-and-access/go/clients/meshvaultclient" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-sdk/go/declaration" "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/compositions" diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index af655ddb..cf9eff83 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -12,10 +12,10 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-sdk/go/declaration" "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/buckets/bucketstest" diff --git a/internal/selfcompose/shadow.go b/internal/selfcompose/shadow.go index 4e8e466f..eaf7ab1e 100644 --- a/internal/selfcompose/shadow.go +++ b/internal/selfcompose/shadow.go @@ -8,8 +8,8 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" - "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/compose/frombus" engcomp "github.com/novox/mesh-host/internal/compositions" diff --git a/internal/userlist/userlist_test.go b/internal/userlist/userlist_test.go index f07ed618..69e509b7 100644 --- a/internal/userlist/userlist_test.go +++ b/internal/userlist/userlist_test.go @@ -4,7 +4,7 @@ import ( "context" "encoding/json" "errors" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate" "git.novox.be/novox/mesh-sdk/go/seal" "github.com/novox/mesh-host/internal/agent" -- 2.54.0 From 4021de1d4271cd9703afe08fe3763bd278f9266d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 05:44:53 +0100 Subject: [PATCH 39/59] HRUN: mesh-compose reads its pipeline input from MESH_RUN_INPUT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The composer answers the assigner's composition step: the request is read from the file named by MESH_RUN_INPUT and the verdict is the last line of standard output, exit 0; the compose-job path stays. Builds: design 47 G §354, §63, §22; lead answer A-167 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- cmd/mesh-compose/main.go | 14 +++++++++++--- cmd/mesh-compose/main_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/cmd/mesh-compose/main.go b/cmd/mesh-compose/main.go index bd30708d..d8f5d8a6 100644 --- a/cmd/mesh-compose/main.go +++ b/cmd/mesh-compose/main.go @@ -11,9 +11,12 @@ // checkout or `git archive` of the catalogue repository at a ref: the command fetches nothing). -node adds a machine to // compose; -declarations answers each machine's declared paths with their owners and modes. // -// It opens no network connection, bus, store or file beyond its standard input, the -request file and the -catalogue -// directory, and reads -// no environment. Exit codes: 0 the verdict is written (a machine that does not compose is a verdict, not an error); +// Run as a pipeline step it reads its input, a composer.Request at depth compose, from the file named by the environment +// variable MESH_RUN_INPUT and answers the run's output as its composer.Verdict on the last line of standard output +// (design 47 G A-167). +// +// It opens no network connection, bus, store or file beyond its standard input, the -request file, the file +// MESH_RUN_INPUT names and the -catalogue directory, and reads no other environment. Exit codes: 0 the verdict is written (a machine that does not compose is a verdict, not an error); // 1 the request cannot be read or run, said on standard error; 2 the flags are wrong. package main @@ -65,6 +68,11 @@ func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int { } var raw []byte var err error + // A pipeline step hands its input as a file named by MESH_RUN_INPUT (design 47 G A-167); -request, when given, wins. + runInput := os.Getenv("MESH_RUN_INPUT") + if *requestFile == "" && runInput != "" { + *requestFile = runInput + } if *requestFile != "" { raw, err = os.ReadFile(*requestFile) } else { diff --git a/cmd/mesh-compose/main_test.go b/cmd/mesh-compose/main_test.go index 88022a04..eeef56c4 100644 --- a/cmd/mesh-compose/main_test.go +++ b/cmd/mesh-compose/main_test.go @@ -267,6 +267,31 @@ func TestTheRequestIsReadFromAFile(t *testing.T) { } } +// A pipeline step names its input file in MESH_RUN_INPUT: the command reads the request there and the verdict is the last +// line of standard output (design 47 G A-167). +func TestThePipelineStepReadsMESH_RUN_INPUT(t *testing.T) { + file := filepath.Join(t.TempDir(), "input.json") + r := composer.Request{V: composer.Version, Depth: composer.DepthCompose, Cases: []composer.Case{{Name: "only", Manifests: map[string]composer.Manifest{ + "demo": {Raw: json.RawMessage(`{"module":"demo","version":"1"}`)}}}}} + if err := os.WriteFile(file, encode(t, r), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("MESH_RUN_INPUT", file) + var out, errOut bytes.Buffer + if code := run(nil, bytes.NewReader(nil), &out, &errOut); code != 0 { + t.Fatalf("exit %d: %s", code, errOut.String()) + } + lines := strings.Split(strings.TrimRight(out.String(), "\n"), "\n") + var v composer.Verdict + if err := json.Unmarshal([]byte(lines[len(lines)-1]), &v); err != nil || len(v.Cases) != 1 || !v.Cases[0].Manifests["demo"].Accepted { + t.Fatalf("last line is no verdict: %v\n%s", err, out.String()) + } + t.Setenv("MESH_RUN_INPUT", file+".gone") + if code := run(nil, bytes.NewReader(nil), &out, &errOut); code != 1 || !strings.Contains(errOut.String(), "cannot be read") { + t.Errorf("a missing input file: exit %d, %q", code, errOut.String()) + } +} + // The exit codes: 1 for a request that cannot be read or run, 2 for wrong flags, each said on standard error. func TestTheExitCodesSayWhatFailed(t *testing.T) { unfilled := encode(t, composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "c", Manifests: map[string]composer.Manifest{"x": {Digest: "abc"}}}}}) -- 2.54.0 From 4b215ba6aee42e3ee0644ddd2d8bf75b33637039 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 06:00:42 +0100 Subject: [PATCH 40/59] H-find: the node-engine raises what only it knows of itself as its own findings The have's failed phase (apply-failing), unhealthy health (module-unhealthy) and kept words (kept-as-it-runs), and each part of its health report (machine-network, network-rewritten, network-unreachable, machine-units, relogin-needed, directory-used-as-found, node-account-fallback), raised and cleared by the held writer where the engine knows the fact. Builds: design 47 G A-197 (ADR 0596, A-192, A-149) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- cmd/mesh-host/main.go | 10 +- cmd/mesh-host/nodeagent.go | 11 + cmd/mesh-host/seatlist.go | 2 +- internal/link/have.go | 26 ++ internal/link/have_test.go | 11 + internal/link/session.go | 4 + internal/seatwait/nodefindings.go | 568 +++++++++++++++++++++++++ internal/seatwait/nodefindings_test.go | 104 +++++ 8 files changed, 734 insertions(+), 2 deletions(-) create mode 100644 internal/seatwait/nodefindings.go create mode 100644 internal/seatwait/nodefindings_test.go diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index eb47ba29..863ced65 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1402,7 +1402,15 @@ func writeHealthReport(ctx context.Context, queue *link.Session, h link.Health, if c := machinesNow.Load(); c != nil { machines.ByAddress, machines.Hub = c.Machines() } - _, err := queue.WriteNodeHealth(ctx, link.NodeHealthOf(queue.Membership.Node, h, peers, machines)) + report := link.NodeHealthOf(queue.Membership.Node, h, peers, machines) + if queue.Fallback != nil && report.Fallback == nil { + report.Fallback = queue.Fallback() // node-account-fallback (ADR 0583 §6), as the written report carries it + } + _, err := queue.WriteNodeHealth(ctx, report) + // The findings the report says are raised by the engine here, at every look, written to the bus or not (design 47 G A-197). + if keeper := ownFindingsNow.Load(); keeper != nil { + keeper.Report(report, machines.ByAddress) + } switch { case err != nil && err.Error() != *unwritten: *unwritten = err.Error() diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index d81de981..09d85816 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -616,6 +616,11 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus // The marks of the haves run off the composition pass, each bounded (see marker). marks := newMarker(run) heldFindings := &seatwait.HeldKeeper{Node: node, ObserverOf: findings.observer, Writer: findings, Ctx: run, Say: say} + // And what only this engine knows of itself, raised where it knows it (design 47 G A-197): its haves' failed phase, health + // and kept words as they are written, and each part of its health report at every look. + ownFindings := &seatwait.NodeKeeper{Node: node, ObserverOf: findings.observer, Writer: findings, Ctx: run, Say: say} + session.SetHaveSaid(ownFindings.Have, ownFindings.HaveGone) + ownFindingsNow.Store(ownFindings) hooks := composerHooks{ entryNotFound: func(fp string, notFound bool) { if err := seatwait.SayEntryNotFound(run, findings, node, findings.observer(), fp, notFound, time.Now); err != nil { @@ -707,6 +712,8 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus cancel() current.set(nil) composerNow.Store(nil) + ownFindingsNow.Store(nil) + session.SetHaveSaid(nil, nil) session.SetManifestOf(nil) apply.UseEngineModule(nil) shadowNow.Store(nil) @@ -909,6 +916,10 @@ func busyModules() []string { // (writeHealthReport); nil on a node that does not compose its own, whose parts then name no target. var machinesNow atomic.Pointer[selfcompose.Composer] +// ownFindingsNow is the keeper of the findings this engine raises of itself in the run that composes the node's own now, for the +// health loop (writeHealthReport); nil while no run holds one. +var ownFindingsNow atomic.Pointer[seatwait.NodeKeeper] + // composerNow is the composer of the run that composes the node's own now, nil while the node applies the controller's // compositions or no run is going: what the verb compose asks a pass of (issue 774). var composerNow atomic.Pointer[selfcompose.Composer] diff --git a/cmd/mesh-host/seatlist.go b/cmd/mesh-host/seatlist.go index 9eb2aa56..c23ceef8 100644 --- a/cmd/mesh-host/seatlist.go +++ b/cmd/mesh-host/seatlist.go @@ -124,7 +124,7 @@ func newFindingsOfTheNode(session *link.Session, node, module string, say func(s Module: module, Node: node, State: link.FindingsState{Session: session}, - Kinds: []string{seatwait.Kind, seatwait.HeldKind, seatwait.EntryNotFoundKind}, + Kinds: append([]string{seatwait.Kind, seatwait.HeldKind, seatwait.EntryNotFoundKind}, seatwait.NodeKinds...), Asked: condseat.SubjectFindingsAsked, Logf: func(format string, args ...any) { say(fmt.Sprintf(format, args...)) }, Publish: session.PublishEvent, diff --git a/internal/link/have.go b/internal/link/have.go index f1a5a696..75e31972 100644 --- a/internal/link/have.go +++ b/internal/link/have.go @@ -440,6 +440,9 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) { s.written = map[string]string{} } s.written[module] = f.Build + if say, _ := q.haveSaid(); say != nil { + said = append(said, func() { say(module, h) }) + } if ev, ok := s.runningToSay(module, h, q.manifestOf()); ok { said = append(said, func() { q.sayRunning(ev) }) } @@ -555,6 +558,9 @@ func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string, s switch { case errors.Is(err, ErrHaveAbsent): delete(s.removed, module) // nothing to say: the key is not there + if _, gone := q.haveSaid(); gone != nil { + *said = append(*said, func() { gone(module) }) + } continue case err != nil: q.sayHave(fmt.Sprintf("could not read %s in the have bucket, and wrote nothing more there: %v", key, err)) @@ -563,6 +569,9 @@ func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string, s } if c, err := nodeengineclient.DecodeHave(current); err == nil && c.Phase == nodeengineclient.PhaseRemoved { delete(s.removed, module) + if _, gone := q.haveSaid(); gone != nil { + *said = append(*said, func() { gone(module) }) + } continue } if err := b.Set(ctx, key, raw, revision); err != nil { @@ -571,6 +580,9 @@ func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string, s return false } delete(s.removed, module) + if _, gone := q.haveSaid(); gone != nil { + *said = append(*said, func() { gone(module) }) + } // The module no longer runs here: running says stopped, with no fingerprint, on its own subject (design 47 G §365, §404). delete(s.runningSaid, module) *said = append(*said, func() { q.sayRunning(stoppedRunning(node, module)) }) @@ -983,6 +995,20 @@ func (q *Session) sayRunning(ev nodeengine.Running) { q.sayEngineOn(nodeengine.SubjectRunning(ev.Node, ev.Module), "running", ev) } +// SetHaveSaid gives the session what it tells of each module's have as it writes it (said), and of a module whose have says +// it is gone (gone): the node-engine's findings of itself are made of them (design 47 G A-197). Either may be nil. +func (q *Session) SetHaveSaid(said func(module string, h nodeengine.HaveByNodeModuleValue), gone func(module string)) { + q.haveSay.Lock() + q.haveSaidFn, q.haveGoneFn = said, gone + q.haveSay.Unlock() +} + +func (q *Session) haveSaid() (func(string, nodeengine.HaveByNodeModuleValue), func(string)) { + q.haveSay.Lock() + defer q.haveSay.Unlock() + return q.haveSaidFn, q.haveGoneFn +} + // SetManifestOf gives the session what names the manifest of the build a module runs (the composer's picture of the // catalog half): its digest, or "" when the node does not know it. func (q *Session) SetManifestOf(f func(module, fp string) string) { diff --git a/internal/link/have_test.go b/internal/link/have_test.go index 132fa730..23b5fa5a 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -230,7 +230,13 @@ func aHaveQueue(l *haveLink) (*Session, *[]string) { func TestAnAppliedModuleIsWrittenRunningAndAFailedOneFailed(t *testing.T) { l := newHaveLink() q, _ := aHaveQueue(l) + // Design 47 G A-197: each have as written is told to the node-engine's findings of itself, the failed one with its detail. + told := map[string]nodeengine.HaveByNodeModuleValue{} + q.SetHaveSaid(func(m string, h nodeengine.HaveByNodeModuleValue) { told[m] = h }, nil) q.SayFacts(context.Background(), appliedModules()) + if told["b"].Phase != nodeengineclient.PhaseFailed || !strings.Contains(told["b"].Detail, "no space") || told["a"].Phase != nodeengineclient.PhaseRunning { + t.Errorf("the haves told were %+v", told) + } a, b := l.have(t, "n1.a"), l.have(t, "n1.b") if a.Phase != nodeengineclient.PhaseRunning || a.Build != "b-1" || a.Health != nodeengineclient.HealthOK { t.Errorf("a: %+v", a) @@ -563,9 +569,14 @@ func onlyA() Report { func TestAModuleLeavingTheDeclarationIsWrittenRemoved(t *testing.T) { l := newHaveLink() q, _ := aHaveQueue(l) + var gone []string + q.SetHaveSaid(nil, func(m string) { gone = append(gone, m) }) q.SayFacts(context.Background(), appliedModules()) revA, revB := l.kv["n1.a"].rev, l.kv["n1.b"].rev q.SayFacts(context.Background(), onlyA()) + if len(gone) != 1 || gone[0] != "b" { // design 47 G A-197: a module that left ends its findings + t.Errorf("the modules told gone were %v, want b", gone) + } b := l.have(t, "n1.b") if b.Phase != nodeengineclient.PhaseRemoved || b.Build != "b-2" { t.Errorf("b: %+v", b) diff --git a/internal/link/session.go b/internal/link/session.go index 4c2e3704..dc8b3911 100644 --- a/internal/link/session.go +++ b/internal/link/session.go @@ -29,6 +29,10 @@ type Session struct { // Own is the module that is this node-engine (§527, §634): its discovery, finding writer and vault login names read it. // Nil names none, as a test's session does. Own *OwnName + // haveSay guards haveSaidFn and haveGoneFn (SetHaveSaid). + haveSay sync.Mutex + haveSaidFn func(module string, h nodeengine.HaveByNodeModuleValue) + haveGoneFn func(module string) // Fallback is the engine's fall back to its old login its health report carries now (Rotator.Fallback); nil says none. Fallback func() *nodeengine.LoginFallback // linkedUser is the bus user the link open now authenticated as, guarded by mu (WhenLinkedAs). diff --git a/internal/seatwait/nodefindings.go b/internal/seatwait/nodefindings.go new file mode 100644 index 00000000..395815cf --- /dev/null +++ b/internal/seatwait/nodefindings.go @@ -0,0 +1,568 @@ +// NodeKeeper keeps the findings only the node-engine knows of itself (design 47 G A-197, ADR 0596, A-149): a have's failed +// phase (apply-failing) and its health (module-unhealthy), what it keeps of a module as it runs (kept-as-it-runs), and each +// part of its own health report (machine-network, network-rewritten, network-unreachable, machine-units, relogin-needed, +// directory-used-as-found, node-account-fallback), under the kind words the watchman raised them under. Each is raised and +// cleared where the engine knows the fact, through its finding writer, never read back from the have or the report by +// another part. +// +// The keeper holds the facts it was told (the haves, the last report) and, at every telling, makes the findings of this +// node those the facts say: what the writer holds of its kinds for this node and no fact says is cleared, what a fact says +// and the writer does not hold, or holds in other words, is written. Nothing runs on a timer: a bound (the apply interval, +// the minute before a failing health is said, the four hours before it is urgent) is judged at each telling, and the engine +// tells at every pass over its haves and every look at its health. +package seatwait + +import ( + "context" + "fmt" + "maps" + "slices" + "sort" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" +) + +// The kinds the node-engine raises of itself, the words the watchman raised them under. +const ( + KindApplyFailing = "apply-failing" + KindModuleUnhealthy = "module-unhealthy" + KindKeptAsRuns = "kept-as-it-runs" + KindMachineNetwork = "machine-network" + KindNetworkRewritten = "network-rewritten" + KindNetworkUnreachable = "network-unreachable" + KindMachineUnits = "machine-units" + KindReloginNeeded = "relogin-needed" + KindUsedAsFound = "directory-used-as-found" + KindAccountFallback = "node-account-fallback" +) + +// NodeKinds is every kind the keeper writes, for the finding writer's list of kinds. +var NodeKinds = []string{KindApplyFailing, KindModuleUnhealthy, KindKeptAsRuns, KindMachineNetwork, KindNetworkRewritten, + KindNetworkUnreachable, KindMachineUnits, KindReloginNeeded, KindUsedAsFound, KindAccountFallback} + +// The bounds the facts are judged by. +const ( + // ApplyEvery is how long a have may keep saying failed before the node says it does not apply its update. + ApplyEvery = 5 * time.Minute + // UnhealthyConfirmAfter is how long a have must keep saying its health failed before the module is said unhealthy. + UnhealthyConfirmAfter = time.Minute + // UnhealthyUrgentAfter is how long a module stays unhealthy before it is urgent. + UnhealthyUrgentAfter = 4 * time.Hour + // ControlNodeModule is the module whose have names the control node: a failing network there is urgent. + ControlNodeModule = "nats" + // PackageManagerKey is how a node-engine names a package manager that does not answer among what failed. + PackageManagerKey = "the package manager (" +) + +const ( + partResolvConf = "resolv-conf" + partNames = "names" + partTunnel = "tunnel" + partBus = "bus" + stateUnhealthy = "unhealthy" + phaseFailed = nodeengineclient.PhaseFailed +) + +// NodeKeeper keeps the findings of one node-engine. Tell it with Have, HaveGone and Report; every telling reconciles. +type NodeKeeper struct { + Node string + Observer string + // ObserverOf, when set, answers the observer at each write (see Waiter). + ObserverOf func() string + Writer Writer + // Ctx is the run's context for the writer's acts; nil is the background. + Ctx context.Context + // Now is the clock; nil is time.Now. + Now func() time.Time + // Say tells the node's journal; nil says nothing. + Say func(string) + + mu sync.Mutex + haves map[string]nodeengine.HaveByNodeModuleValue + failedSince map[string]time.Time + sickSince map[string]time.Time + keptSince map[string]time.Time + report *nodeengine.HealthByNodeValue + byAddress map[string]string +} + +// Have is a module's have as the node-engine wrote it. +func (k *NodeKeeper) Have(module string, h nodeengine.HaveByNodeModuleValue) { + k.mu.Lock() + defer k.mu.Unlock() + if k.haves == nil { + k.haves, k.failedSince, k.sickSince, k.keptSince = map[string]nodeengine.HaveByNodeModuleValue{}, map[string]time.Time{}, + map[string]time.Time{}, map[string]time.Time{} + } + if h.Phase == nodeengineclient.PhaseRemoved { + k.forget(module) + k.reconcile() + return + } + k.haves[module] = h + now := k.now() + setSince(k.failedSince, module, h.Phase == phaseFailed, now) + setSince(k.sickSince, module, h.Health == nodeengineclient.HealthFailed, now) + setSince(k.keptSince, module, len(h.Kept) > 0, now) + k.reconcile() +} + +// HaveGone is a module no longer held by the node. +func (k *NodeKeeper) HaveGone(module string) { + k.mu.Lock() + defer k.mu.Unlock() + k.forget(module) + k.reconcile() +} + +// Report is the node's health report as it stands now (written or not), and the machines by the address of each, for the +// peers whose tunnel it does not see. Told at every look, so a bound is judged then. +func (k *NodeKeeper) Report(h nodeengine.HealthByNodeValue, byAddress map[string]string) { + k.mu.Lock() + defer k.mu.Unlock() + if h.At.IsZero() { + h.At = k.now() + } + k.report, k.byAddress = &h, byAddress + k.reconcile() +} + +func (k *NodeKeeper) forget(module string) { + delete(k.haves, module) + delete(k.failedSince, module) + delete(k.sickSince, module) + delete(k.keptSince, module) +} + +func setSince(m map[string]time.Time, module string, on bool, now time.Time) { + if !on { + delete(m, module) + } else if _, ok := m[module]; !ok { + m[module] = now + } +} + +func (k *NodeKeeper) now() time.Time { + if k.Now != nil { + return k.Now() + } + return time.Now() +} + +func (k *NodeKeeper) say(s string) { + if k.Say != nil { + k.Say(s) + } +} + +func stampSec(t time.Time) string { return t.UTC().Format("2006-01-02 15:04:05 MST") } + +func firstLine(s string) string { + line, _, _ := strings.Cut(s, "\n") + return line +} + +func orWords(s, or string) string { + if strings.TrimSpace(s) == "" { + return or + } + return s +} + +// ownKey is the identity of a finding the keeper writes. +func ownKey(scope, id, kind string) string { return scope + "." + id + "." + kind } + +// reconcile makes this node's findings of the keeper's kinds those the facts say. The caller holds the lock. +func (k *NodeKeeper) reconcile() { + ctx := k.Ctx + if ctx == nil { + ctx = context.Background() + } + want := map[string]condseat.Finding{} + add := func(f condseat.Finding) { want[ownKey(f.Scope, f.ID, f.Kind)] = f } + k.haveFindings(add) + k.reportFindings(add) + held := map[string]condseat.Finding{} + for _, f := range k.Writer.Held() { + if k.owns(f) { + held[ownKey(f.Scope, f.ID, f.Kind)] = f + } + } + for _, key := range slices.Sorted(maps.Keys(held)) { + if _, still := want[key]; still { + continue + } + f := held[key] + if err := k.Writer.Clear(ctx, f.Scope, f.ID, f.Kind, "the node-engine no longer finds it so"); err != nil { + k.say(fmt.Sprintf("the finding %s of %s cannot be cleared: %v", f.Kind, k.Node, err)) + } + } + for _, key := range slices.Sorted(maps.Keys(want)) { + f := want[key] + if was, ok := held[key]; ok { + f.Since, f.At = was.Since, was.At // the same thing seen again writes the same words + if sameFinding(was, f) { + continue + } + } + if err := k.Writer.Write(ctx, f); err != nil { + k.say(fmt.Sprintf("the finding %s of %s cannot be written: %v", f.Kind, k.Node, err)) + continue + } + k.say(f.Summary) + } +} + +// owns says a held finding is one the keeper writes for this node. +func (k *NodeKeeper) owns(f condseat.Finding) bool { + if !slices.Contains(NodeKinds, f.Kind) { + return false + } + if f.Kind == KindNetworkUnreachable { + return strings.HasSuffix(f.ID, "."+k.Node) + } + return f.Subject.Machine == k.Node +} + +func sameFinding(a, b condseat.Finding) bool { + if a.Summary != b.Summary || a.Severity != b.Severity || a.Headline != b.Headline || a.Needs != b.Needs || + a.Explanation != b.Explanation || !slices.Equal(a.Subject.Also, b.Subject.Also) || len(a.Evidence) != len(b.Evidence) { + return false + } + for i := range a.Evidence { + if a.Evidence[i].Said != b.Evidence[i].Said { + return false + } + } + return true +} + +// finding is a finding of this node's: observed by the engine's own module, about a machine of the scope. +func (k *NodeKeeper) finding(scope, id, kind, machine string, severity string, since time.Time, summary, said string) condseat.Finding { + since = since.UTC() + f := condseat.Finding{Observer: observerOf(k.Observer, k.ObserverOf), Kind: kind, Scope: scope, ID: id, Severity: severity, + Source: kind, Summary: summary, At: since, Since: since, + Subject: condseat.Subject{Scope: scope, ID: id, Machine: machine}} + if said != "" { + f.Evidence = []condseat.Evidence{{At: since, Said: said}} + } + return f +} + +// haveFindings are the findings the haves say: apply-failing, module-unhealthy and kept-as-it-runs. +func (k *NodeKeeper) haveFindings(add func(condseat.Finding)) { + now := k.now() + type failure struct{ module, words string } + var failed []failure + var since time.Time + for _, m := range slices.Sorted(maps.Keys(k.failedSince)) { + failed = append(failed, failure{m, firstLine(orWords(k.haves[m].Detail, "no reason given"))}) + if at := k.failedSince[m]; since.IsZero() || at.Before(since) { + since = at + } + } + if len(failed) > 0 && now.Sub(since) > ApplyEvery { + var names, words []string + tool := "" + for _, f := range failed { + names = append(names, f.module) + words = append(words, f.module+": "+f.words) + w := strings.TrimPrefix(f.words, "refused: ") + if i := strings.Index(w, PackageManagerKey); i >= 0 && tool == "" { + tool = strings.TrimSuffix(strings.SplitN(w[i+len(PackageManagerKey):], ")", 2)[0], ")") + } + } + f := k.finding(scopeMachine, k.Node, KindApplyFailing, k.Node, condseat.FindingSeverityWarning, since, "", strings.Join(words, "; ")) + f.Headline = k.Node + " is not applying its update" + if tool != "" { + f.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent is applied, since %s (bound %s)", + k.Node, tool, stampMin(since), ApplyEvery) + f.Headline = k.Node + " cannot install packages" + } else { + f.Summary = fmt.Sprintf("%s has failed to apply %s since %s (bound %s)", k.Node, strings.Join(names, ", "), stampMin(since), ApplyEvery) + f.Explanation = fmt.Sprintf("%s has failed to apply its updates, and trying again has not helped. The details say what fails.", k.Node) + } + add(f) + } + for _, m := range slices.Sorted(maps.Keys(k.sickSince)) { + at := k.sickSince[m] + if now.Sub(at) < UnhealthyConfirmAfter { + continue + } + severity := condseat.FindingSeverityWarning + if now.Sub(at) >= UnhealthyUrgentAfter { + severity = condseat.FindingSeverityUrgent + } + h := k.haves[m] + f := k.finding(scopeModule, m+"."+k.Node, KindModuleUnhealthy, k.Node, severity, at, + fmt.Sprintf("%s on %s is not healthy: %s", m, k.Node, orWords(h.Detail, "its check failed")), firstLine(orWords(h.Detail, "failed"))) + f.Headline = fmt.Sprintf("%s not working on %s", m, k.Node) + add(f) + } + for _, m := range slices.Sorted(maps.Keys(k.keptSince)) { + words := strings.Join(k.haves[m].Kept, "; ") + f := k.finding(scopeModule, m+"."+k.Node, KindKeptAsRuns, k.Node, condseat.FindingSeverityWarning, k.keptSince[m], + fmt.Sprintf("%s on %s is kept as it runs: %s", m, k.Node, words), firstLine(words)) + f.Headline = fmt.Sprintf("%s kept as it runs on %s", m, k.Node) + f.Needs = "a build of " + m + " whose manifest says it (tool-runner, core-credentials)" + add(f) + } +} + +func stampMin(t time.Time) string { return t.UTC().Format("2006-01-02 15:04 MST") } + +// control says the node runs the module the control node is named by. +func (k *NodeKeeper) control() bool { + h, ok := k.haves[ControlNodeModule] + return ok && h.Phase != nodeengineclient.PhaseRemoved +} + +// reportFindings are the findings the node's health report says. +func (k *NodeKeeper) reportFindings(add func(condseat.Finding)) { + h := k.report + if h == nil { + return + } + node := k.Node + var own []nodeengine.NetworkPart + var rewritten *nodeengine.NetworkPart + var parts []nodeengine.NetworkPart + if h.Network != nil { + for _, p := range h.Network.Parts { + if p.State == stateUnhealthy { + parts = append(parts, p) + } + } + } + toward := map[string][]nodeengine.NetworkPart{} + for i, p := range parts { + switch { + case p.Part == partResolvConf: + rewritten = &parts[i] + case len(p.Targets) > 0: + for _, x := range p.Targets { + toward[x] = append(toward[x], p) + } + default: + own = append(own, p) + } + } + if rewritten != nil { + add(k.rewritten(*rewritten, parts)) + kept := own[:0] + for _, p := range own { + if p.Part != partNames { + kept = append(kept, p) + } + } + own = kept + } + if len(own) > 0 { + var words, said []string + severity := condseat.FindingSeverityWarning + for _, p := range own { + words = append(words, p.Reason) + said = append(said, fmt.Sprintf("%s since %s: %s", p.Part, stampSec(p.Since), p.Said)) + if p.Part == partBus { + severity = condseat.FindingSeverityUrgent + } + } + if k.control() { + severity = condseat.FindingSeverityUrgent + } + add(k.finding(scopeMachine, node, KindMachineNetwork, node, severity, h.Network.Since, + fmt.Sprintf("%s's network is not healthy: %s", node, strings.Join(words, "; ")), strings.Join(said, " | "))) + } + // A tunnel peer this node does not see points at that peer's machine as a tunnel part toward it would. + for _, p := range h.Peers { + if p.Seen { + continue + } + for _, a := range p.Addresses { + addr, _, _ := strings.Cut(a, "/") + if x := k.byAddress[addr]; x != "" && x != node { + toward[x] = append(toward[x], nodeengine.NetworkPart{Part: partTunnel, State: stateUnhealthy, Since: h.At, + Said: "no handshake with " + x + " within five minutes", Targets: []string{x}}) + } + } + } + for _, x := range slices.Sorted(maps.Keys(toward)) { + var what, said []string + since := toward[x][0].Since + severity := condseat.FindingSeverityWarning + for _, p := range toward[x] { + wd := map[string]string{partTunnel: "the tunnel to it", partBus: "the bus on it", partNames: "its resolver"}[p.Part] + if wd == "" { + wd = p.Part + } + if !slices.Contains(what, wd) { + what = append(what, wd) + } + said = append(said, fmt.Sprintf("%s: %s: %s", node, p.Part, p.Said)) + if p.Since.Before(since) { + since = p.Since + } + if p.Part == partBus { + severity = condseat.FindingSeverityUrgent + } + } + f := k.finding(scopeMachine, x+"."+node, KindNetworkUnreachable, x, severity, since, + fmt.Sprintf("%s cannot be reached from %s: %s", x, node, strings.Join(what, ", ")), strings.Join(said, " | ")) + f.Subject.Also = []string{node} + add(f) + } + if f, ok := k.units(h.Units); ok { + add(f) + } + for _, f := range k.relogin(h.Relogin) { + add(f) + } + for _, f := range k.found(h.Found) { + add(f) + } + if f := h.Fallback; f != nil && f.Node == node { + said := fmt.Sprintf("%s fell back from its new login %s to its old one during a rotation of its bus account: %s", node, f.From, f.Why) + g := k.finding(scopeMachine, node, KindAccountFallback, node, condseat.FindingSeverityWarning, f.At, said, said) + g.Headline = fmt.Sprintf("%s fell back to its old bus login", node) + g.Needs = fmt.Sprintf("abandon the rotation of %s's account, or rotate it again once its new login authenticates.", node) + add(g) + } +} + +func (k *NodeKeeper) rewritten(p nodeengine.NetworkPart, all []nodeengine.NetworkPart) condseat.Finding { + m := k.Node + writer := "" + if p.Writer != "" { + writer = " (" + p.Writer + ")" + } + cost := "the names still resolve through what it wrote" + said := []string{p.Part + ": " + p.Said} + for _, q := range all { + if q.Part == partNames { + cost = strings.TrimSuffix(q.Reason, ".") + said = append(said, q.Part+": "+q.Said) + } + } + id := m + summary := fmt.Sprintf("the resolver file on %s was rewritten by another program%s — %s until the node-engine writes it "+ + "back at its next reconcile, or that program gives it back", m, writer, cost) + if p.Owner != "" { + id = m + "." + p.Owner + summary = fmt.Sprintf("the resolver file %s writes on %s was rewritten by another program%s — %s until the node-engine "+ + "writes it back at its next reconcile, or that program gives it back", p.Owner, m, writer, cost) + } + severity := condseat.FindingSeverityWarning + if k.control() { + severity = condseat.FindingSeverityUrgent + } + return k.finding(scopeMachine, id, KindNetworkRewritten, m, severity, p.Since, summary, + fmt.Sprintf("since %s: %s", stampSec(p.Since), strings.Join(said, " | "))) +} + +func (k *NodeKeeper) units(u *nodeengine.UnitsHealth) (condseat.Finding, bool) { + if u == nil || len(u.Failed) == 0 { + return condseat.Finding{}, false + } + node := k.Node + var names, said []string + since := u.Failed[0].Since + for _, f := range u.Failed { + where := "" + if f.Scope == "user" { + where = " (the account's own manager)" + } + names = append(names, f.Unit+where) + scope, result := f.Scope, f.Result + if scope == "" { + scope = "system" + } + if result == "" { + result = "no reason said" + } + line := fmt.Sprintf("%s in the %s manager: %s", f.Unit, scope, result) + if f.Load != "" && f.Load != "loaded" { + line += ", its unit " + f.Load + } + if f.Resource != "" { + line += ", named by the mesh's own " + f.Resource + } + said = append(said, line+", since "+stampSec(f.Since)) + if f.Since.Before(since) { + since = f.Since + } + } + failed := "1 failed service" + if len(u.Failed) > 1 { + failed = fmt.Sprintf("%d failed services", len(u.Failed)) + } + f := k.finding(scopeMachine, node, KindMachineUnits, node, condseat.FindingSeverityWarning, since, + fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. Each is the machine's own: "+ + "mend or remove it there, or have a module place it", node, len(u.Failed), strings.Join(names, ", ")), + strings.Join(said, "; ")) + f.Headline = strings.ToUpper(failed[:1]) + failed[1:] + " on " + node + f.Needs = fmt.Sprintf("mend or remove them on %s, or silence this if they do not matter.", node) + return f, true +} + +func orModule(m string) string { + if m == "" { + return "_" + } + return m +} + +func (k *NodeKeeper) relogin(accounts []nodeengine.NodeAccount) []condseat.Finding { + by := map[string][]nodeengine.NodeAccount{} + for _, a := range accounts { + by[a.Module] = append(by[a.Module], a) + } + var out []condseat.Finding + for _, module := range slices.Sorted(maps.Keys(by)) { + var names []string + since := by[module][0].Since + for _, a := range by[module] { + names = append(names, a.Account) + if a.Since.Before(since) { + since = a.Since + } + } + sort.Strings(names) + said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose login began "+ + "before it was; log out of %[1]s completely and log in again, or restart it", k.Node, orModule(module), strings.Join(names, ", ")) + out = append(out, k.finding(scopeModule, orModule(module)+"."+k.Node, KindReloginNeeded, k.Node, condseat.FindingSeverityWarning, since, said, said)) + } + return out +} + +func (k *NodeKeeper) found(dirs []nodeengine.NodeDirectory) []condseat.Finding { + by := map[string][]nodeengine.NodeDirectory{} + for _, d := range dirs { + by[d.Module] = append(by[d.Module], d) + } + var out []condseat.Finding + for _, module := range slices.Sorted(maps.Keys(by)) { + var ids, why []string + since := by[module][0].Since + for _, d := range by[module] { + ids = append(ids, d.Resource) + why = append(why, d.Reason) + if d.Since.Before(since) { + since = d.Since + } + } + m := orModule(module) + said := fmt.Sprintf("%s on %s uses %s as found: %s", m, k.Node, strings.Join(ids, ", "), strings.Join(why, "; ")) + f := k.finding(scopeModule, m+"."+k.Node, KindUsedAsFound, k.Node, condseat.FindingSeverityWarning, since, said, said) + f.Headline = fmt.Sprintf("%s waits for a directory on %s", m, k.Node) + f.Explanation = fmt.Sprintf("Needs you: hand the directory over from the control-node, as the operator; the details name it "+ + "and the line to type. A directory of %s was already on %s, with another owner or mode than %s declares.", m, k.Node, m) + f.Needs = "hand the directory over from the control-node, as the operator; the details name it and the line to type." + out = append(out, f) + } + return out +} diff --git a/internal/seatwait/nodefindings_test.go b/internal/seatwait/nodefindings_test.go new file mode 100644 index 00000000..8324bc7e --- /dev/null +++ b/internal/seatwait/nodefindings_test.go @@ -0,0 +1,104 @@ +package seatwait + +import ( + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" +) + +// Design 47 G A-197: what only the node-engine knows of itself it raises as its own findings, under the kind words the watchman +// raised them under, and clears where it knows the fact no longer holds. +func TestTheNodeEngineRaisesWhatOnlyItKnows(t *testing.T) { + at := time.Date(2026, 11, 12, 8, 0, 0, 0, time.UTC) + type step struct { + do func(k *NodeKeeper, advance func(time.Duration)) + want []string // "scope.id.kind" of what stands after it, sorted + } + failed := nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseFailed, Detail: "refused: the package manager (pacman) does not answer"} + sick := nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthFailed, Detail: "port closed"} + report := func(h nodeengine.HealthByNodeValue) func(*NodeKeeper, func(time.Duration)) { + return func(k *NodeKeeper, _ func(time.Duration)) { k.Report(h, map[string]string{"10.0.0.2": "g14"}) } + } + for name, steps := range map[string][]step{ + "a have failed past the apply interval is apply-failing, cleared by the clean apply": { + {func(k *NodeKeeper, _ func(time.Duration)) { k.Have("b", failed) }, nil}, + {func(k *NodeKeeper, adv func(time.Duration)) { adv(ApplyEvery + time.Second); k.Have("b", failed) }, []string{"machine.ace.apply-failing"}}, + {func(k *NodeKeeper, _ func(time.Duration)) { + k.Have("b", nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning}) + }, nil}, + }, + "a have unhealthy past a minute is module-unhealthy, cleared when it is healthy or removed": { + {func(k *NodeKeeper, _ func(time.Duration)) { k.Have("b", sick) }, nil}, + {func(k *NodeKeeper, adv func(time.Duration)) { adv(time.Minute); k.Have("b", sick) }, []string{"module.b.ace.module-unhealthy"}}, + {func(k *NodeKeeper, _ func(time.Duration)) { + k.Have("b", nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRemoved}) + }, nil}, + }, + "what a module keeps as it runs is kept-as-it-runs until its have says none": { + {func(k *NodeKeeper, _ func(time.Duration)) { + k.Have("b", nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning, Kept: []string{"tool-runner"}}) + }, []string{"module.b.ace.kept-as-it-runs"}}, + {func(k *NodeKeeper, _ func(time.Duration)) { k.HaveGone("b") }, nil}, + }, + "each part of the report is its own finding and each ends with the report": { + {report(nodeengine.HealthByNodeValue{ + Network: &nodeengine.NetworkHealth{Since: at, Parts: []nodeengine.NetworkPart{ + {Part: "bus", State: "unhealthy", Reason: "no bus", Since: at}, + {Part: "tunnel", State: "unhealthy", Since: at, Targets: []string{"g14"}}, + {Part: "resolv-conf", State: "unhealthy", Owner: "dnsmasq", Since: at}}}, + Units: &nodeengine.UnitsHealth{Failed: []nodeengine.FailedUnit{{Unit: "x.service", Since: at}}}, + Relogin: []nodeengine.NodeAccount{{Account: "u", Module: "m", Since: at}}, + Found: []nodeengine.NodeDirectory{{Module: "m", Resource: "m.dir", Since: at}}, + Fallback: &nodeengine.LoginFallback{Node: "ace", From: "new", At: at}}), + []string{"machine.ace.dnsmasq.network-rewritten", "machine.ace.machine-network", "machine.ace.machine-units", + "machine.ace.node-account-fallback", "machine.g14.ace.network-unreachable", "module.m.ace.directory-used-as-found", + "module.m.ace.relogin-needed"}}, + {report(nodeengine.HealthByNodeValue{}), nil}, + }, + "a tunnel peer not seen is the machine unreachable": { + {report(nodeengine.HealthByNodeValue{Peers: []nodeengine.PeerSeen{{Key: "k", Addresses: []string{"10.0.0.2/32"}}}}), + []string{"machine.g14.ace.network-unreachable"}}, + {report(nodeengine.HealthByNodeValue{Peers: []nodeengine.PeerSeen{{Key: "k", Addresses: []string{"10.0.0.2/32"}, Seen: true}}}), nil}, + }, + } { + t.Run(name, func(t *testing.T) { + now := at + w := &fakeFindings{} + // Another node's finding and an unrelated kind stand in the state and are never touched. + w.held = []condseat.Finding{{Scope: "machine", ID: "g14", Kind: KindMachineUnits, Subject: condseat.Subject{Machine: "g14"}}, + {Scope: "machine", ID: "ace", Kind: HeldKind, Subject: condseat.Subject{Machine: "ace"}}} + k := &NodeKeeper{Node: "ace", Observer: "mesh-host", Writer: w, Now: func() time.Time { return now }} + for i, s := range steps { + s.do(k, func(d time.Duration) { now = now.Add(d) }) + var got []string + for _, f := range w.held { + if f.Observer == "mesh-host" { + got = append(got, f.Scope+"."+f.ID+"."+f.Kind) + } + } + if strings.Join(sorted(got), ",") != strings.Join(s.want, ",") { + t.Fatalf("step %d: stands %v, want %v", i, sorted(got), s.want) + } + } + if len(w.held) != 2 { + t.Fatalf("what is not the keeper's was touched: %+v", w.held) + } + }) + } +} + +func sorted(in []string) []string { + out := append([]string(nil), in...) + for i := range out { + for j := i + 1; j < len(out); j++ { + if out[j] < out[i] { + out[i], out[j] = out[j], out[i] + } + } + } + return out +} -- 2.54.0 From a3fea9106a2a607f79dd439c151bd07a30a340ec Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 06:02:07 +0100 Subject: [PATCH 41/59] H-find: the findings keep the keys the watchman raised them under Each finding's id carries its token as a ref (A-14), so machine..network, .rewritten, .unreachable, .units and module...unhealthy and .kept read as before. Builds: design 47 G A-197 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- internal/seatwait/nodefindings.go | 32 +++++++++++++++----------- internal/seatwait/nodefindings_test.go | 10 ++++---- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/internal/seatwait/nodefindings.go b/internal/seatwait/nodefindings.go index 395815cf..f3000aa4 100644 --- a/internal/seatwait/nodefindings.go +++ b/internal/seatwait/nodefindings.go @@ -224,7 +224,7 @@ func (k *NodeKeeper) owns(f condseat.Finding) bool { return false } if f.Kind == KindNetworkUnreachable { - return strings.HasSuffix(f.ID, "."+k.Node) + return strings.HasSuffix(f.ID, "."+k.Node+".unreachable") } return f.Subject.Machine == k.Node } @@ -243,10 +243,16 @@ func sameFinding(a, b condseat.Finding) bool { } // finding is a finding of this node's: observed by the engine's own module, about a machine of the scope. -func (k *NodeKeeper) finding(scope, id, kind, machine string, severity string, since time.Time, summary, said string) condseat.Finding { +// +// The id and ref are the condition's key the watchman raised it under, `..` (A-14): a token other than the kind +// goes into the id as `.` and is kept in ref, as the watchman's findings were written. +func (k *NodeKeeper) finding(scope, id, kind, ref, machine string, severity string, since time.Time, summary, said string) condseat.Finding { since = since.UTC() + if ref != "" { + id += "." + ref + } f := condseat.Finding{Observer: observerOf(k.Observer, k.ObserverOf), Kind: kind, Scope: scope, ID: id, Severity: severity, - Source: kind, Summary: summary, At: since, Since: since, + Ref: ref, Source: kind, Summary: summary, At: since, Since: since, Subject: condseat.Subject{Scope: scope, ID: id, Machine: machine}} if said != "" { f.Evidence = []condseat.Evidence{{At: since, Said: said}} @@ -277,7 +283,7 @@ func (k *NodeKeeper) haveFindings(add func(condseat.Finding)) { tool = strings.TrimSuffix(strings.SplitN(w[i+len(PackageManagerKey):], ")", 2)[0], ")") } } - f := k.finding(scopeMachine, k.Node, KindApplyFailing, k.Node, condseat.FindingSeverityWarning, since, "", strings.Join(words, "; ")) + f := k.finding(scopeMachine, k.Node, KindApplyFailing, "", k.Node, condseat.FindingSeverityWarning, since, "", strings.Join(words, "; ")) f.Headline = k.Node + " is not applying its update" if tool != "" { f.Summary = fmt.Sprintf("%s: %s does not answer, so no package reaches it; everything else it was sent is applied, since %s (bound %s)", @@ -299,14 +305,14 @@ func (k *NodeKeeper) haveFindings(add func(condseat.Finding)) { severity = condseat.FindingSeverityUrgent } h := k.haves[m] - f := k.finding(scopeModule, m+"."+k.Node, KindModuleUnhealthy, k.Node, severity, at, + f := k.finding(scopeModule, m+"."+k.Node, KindModuleUnhealthy, "unhealthy", k.Node, severity, at, fmt.Sprintf("%s on %s is not healthy: %s", m, k.Node, orWords(h.Detail, "its check failed")), firstLine(orWords(h.Detail, "failed"))) f.Headline = fmt.Sprintf("%s not working on %s", m, k.Node) add(f) } for _, m := range slices.Sorted(maps.Keys(k.keptSince)) { words := strings.Join(k.haves[m].Kept, "; ") - f := k.finding(scopeModule, m+"."+k.Node, KindKeptAsRuns, k.Node, condseat.FindingSeverityWarning, k.keptSince[m], + f := k.finding(scopeModule, m+"."+k.Node, KindKeptAsRuns, "kept", k.Node, condseat.FindingSeverityWarning, k.keptSince[m], fmt.Sprintf("%s on %s is kept as it runs: %s", m, k.Node, words), firstLine(words)) f.Headline = fmt.Sprintf("%s kept as it runs on %s", m, k.Node) f.Needs = "a build of " + m + " whose manifest says it (tool-runner, core-credentials)" @@ -375,7 +381,7 @@ func (k *NodeKeeper) reportFindings(add func(condseat.Finding)) { if k.control() { severity = condseat.FindingSeverityUrgent } - add(k.finding(scopeMachine, node, KindMachineNetwork, node, severity, h.Network.Since, + add(k.finding(scopeMachine, node, KindMachineNetwork, "network", node, severity, h.Network.Since, fmt.Sprintf("%s's network is not healthy: %s", node, strings.Join(words, "; ")), strings.Join(said, " | "))) } // A tunnel peer this node does not see points at that peer's machine as a tunnel part toward it would. @@ -411,7 +417,7 @@ func (k *NodeKeeper) reportFindings(add func(condseat.Finding)) { severity = condseat.FindingSeverityUrgent } } - f := k.finding(scopeMachine, x+"."+node, KindNetworkUnreachable, x, severity, since, + f := k.finding(scopeMachine, x+"."+node, KindNetworkUnreachable, "unreachable", x, severity, since, fmt.Sprintf("%s cannot be reached from %s: %s", x, node, strings.Join(what, ", ")), strings.Join(said, " | ")) f.Subject.Also = []string{node} add(f) @@ -427,7 +433,7 @@ func (k *NodeKeeper) reportFindings(add func(condseat.Finding)) { } if f := h.Fallback; f != nil && f.Node == node { said := fmt.Sprintf("%s fell back from its new login %s to its old one during a rotation of its bus account: %s", node, f.From, f.Why) - g := k.finding(scopeMachine, node, KindAccountFallback, node, condseat.FindingSeverityWarning, f.At, said, said) + g := k.finding(scopeMachine, node, KindAccountFallback, "", node, condseat.FindingSeverityWarning, f.At, said, said) g.Headline = fmt.Sprintf("%s fell back to its old bus login", node) g.Needs = fmt.Sprintf("abandon the rotation of %s's account, or rotate it again once its new login authenticates.", node) add(g) @@ -460,7 +466,7 @@ func (k *NodeKeeper) rewritten(p nodeengine.NetworkPart, all []nodeengine.Networ if k.control() { severity = condseat.FindingSeverityUrgent } - return k.finding(scopeMachine, id, KindNetworkRewritten, m, severity, p.Since, summary, + return k.finding(scopeMachine, id, KindNetworkRewritten, "rewritten", m, severity, p.Since, summary, fmt.Sprintf("since %s: %s", stampSec(p.Since), strings.Join(said, " | "))) } @@ -500,7 +506,7 @@ func (k *NodeKeeper) units(u *nodeengine.UnitsHealth) (condseat.Finding, bool) { if len(u.Failed) > 1 { failed = fmt.Sprintf("%d failed services", len(u.Failed)) } - f := k.finding(scopeMachine, node, KindMachineUnits, node, condseat.FindingSeverityWarning, since, + f := k.finding(scopeMachine, node, KindMachineUnits, "units", node, condseat.FindingSeverityWarning, since, fmt.Sprintf("%s's service manager is degraded: %d failed unit(s) no module places — %s. Each is the machine's own: "+ "mend or remove it there, or have a module place it", node, len(u.Failed), strings.Join(names, ", ")), strings.Join(said, "; ")) @@ -534,7 +540,7 @@ func (k *NodeKeeper) relogin(accounts []nodeengine.NodeAccount) []condseat.Findi sort.Strings(names) said := fmt.Sprintf("relogin needed on %s: %s waits for a new login of %s, which is in its group and whose login began "+ "before it was; log out of %[1]s completely and log in again, or restart it", k.Node, orModule(module), strings.Join(names, ", ")) - out = append(out, k.finding(scopeModule, orModule(module)+"."+k.Node, KindReloginNeeded, k.Node, condseat.FindingSeverityWarning, since, said, said)) + out = append(out, k.finding(scopeModule, orModule(module)+"."+k.Node, KindReloginNeeded, "", k.Node, condseat.FindingSeverityWarning, since, said, said)) } return out } @@ -557,7 +563,7 @@ func (k *NodeKeeper) found(dirs []nodeengine.NodeDirectory) []condseat.Finding { } m := orModule(module) said := fmt.Sprintf("%s on %s uses %s as found: %s", m, k.Node, strings.Join(ids, ", "), strings.Join(why, "; ")) - f := k.finding(scopeModule, m+"."+k.Node, KindUsedAsFound, k.Node, condseat.FindingSeverityWarning, since, said, said) + f := k.finding(scopeModule, m+"."+k.Node, KindUsedAsFound, "", k.Node, condseat.FindingSeverityWarning, since, said, said) f.Headline = fmt.Sprintf("%s waits for a directory on %s", m, k.Node) f.Explanation = fmt.Sprintf("Needs you: hand the directory over from the control-node, as the operator; the details name it "+ "and the line to type. A directory of %s was already on %s, with another owner or mode than %s declares.", m, k.Node, m) diff --git a/internal/seatwait/nodefindings_test.go b/internal/seatwait/nodefindings_test.go index 8324bc7e..78611246 100644 --- a/internal/seatwait/nodefindings_test.go +++ b/internal/seatwait/nodefindings_test.go @@ -33,7 +33,7 @@ func TestTheNodeEngineRaisesWhatOnlyItKnows(t *testing.T) { }, "a have unhealthy past a minute is module-unhealthy, cleared when it is healthy or removed": { {func(k *NodeKeeper, _ func(time.Duration)) { k.Have("b", sick) }, nil}, - {func(k *NodeKeeper, adv func(time.Duration)) { adv(time.Minute); k.Have("b", sick) }, []string{"module.b.ace.module-unhealthy"}}, + {func(k *NodeKeeper, adv func(time.Duration)) { adv(time.Minute); k.Have("b", sick) }, []string{"module.b.ace.unhealthy.module-unhealthy"}}, {func(k *NodeKeeper, _ func(time.Duration)) { k.Have("b", nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRemoved}) }, nil}, @@ -41,7 +41,7 @@ func TestTheNodeEngineRaisesWhatOnlyItKnows(t *testing.T) { "what a module keeps as it runs is kept-as-it-runs until its have says none": { {func(k *NodeKeeper, _ func(time.Duration)) { k.Have("b", nodeengine.HaveByNodeModuleValue{Phase: nodeengineclient.PhaseRunning, Kept: []string{"tool-runner"}}) - }, []string{"module.b.ace.kept-as-it-runs"}}, + }, []string{"module.b.ace.kept.kept-as-it-runs"}}, {func(k *NodeKeeper, _ func(time.Duration)) { k.HaveGone("b") }, nil}, }, "each part of the report is its own finding and each ends with the report": { @@ -54,14 +54,14 @@ func TestTheNodeEngineRaisesWhatOnlyItKnows(t *testing.T) { Relogin: []nodeengine.NodeAccount{{Account: "u", Module: "m", Since: at}}, Found: []nodeengine.NodeDirectory{{Module: "m", Resource: "m.dir", Since: at}}, Fallback: &nodeengine.LoginFallback{Node: "ace", From: "new", At: at}}), - []string{"machine.ace.dnsmasq.network-rewritten", "machine.ace.machine-network", "machine.ace.machine-units", - "machine.ace.node-account-fallback", "machine.g14.ace.network-unreachable", "module.m.ace.directory-used-as-found", + []string{"machine.ace.dnsmasq.rewritten.network-rewritten", "machine.ace.network.machine-network", "machine.ace.node-account-fallback", + "machine.ace.units.machine-units", "machine.g14.ace.unreachable.network-unreachable", "module.m.ace.directory-used-as-found", "module.m.ace.relogin-needed"}}, {report(nodeengine.HealthByNodeValue{}), nil}, }, "a tunnel peer not seen is the machine unreachable": { {report(nodeengine.HealthByNodeValue{Peers: []nodeengine.PeerSeen{{Key: "k", Addresses: []string{"10.0.0.2/32"}}}}), - []string{"machine.g14.ace.network-unreachable"}}, + []string{"machine.g14.ace.unreachable.network-unreachable"}}, {report(nodeengine.HealthByNodeValue{Peers: []nodeengine.PeerSeen{{Key: "k", Addresses: []string{"10.0.0.2/32"}, Seen: true}}}), nil}, }, } { -- 2.54.0 From 8734d5ee1a1f8b9ab80cc866b8a530c04f1b4f90 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 06:06:16 +0100 Subject: [PATCH 42/59] AM-D3 path: the node-engine takes the runner's reading and fingerprint into the node's record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The grants part decides a node's withheld subjects from the runner's own reading tied to the runner that stated it; the merge into capabilities stays for placement. Builds: design 46 7 §6 (its node's node-engine takes both into the node's record), lead answer A-203 (runner-capabilities, runner-fingerprint) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- internal/link/have.go | 2 +- internal/link/have_test.go | 6 ++++-- internal/link/record.go | 9 +++++++++ internal/link/session.go | 4 +++- internal/link/witnessing.go | 3 +++ 5 files changed, 20 insertions(+), 4 deletions(-) diff --git a/internal/link/have.go b/internal/link/have.go index f1a5a696..bc261de0 100644 --- a/internal/link/have.go +++ b/internal/link/have.go @@ -311,7 +311,7 @@ func (q *Session) nodeFactsWithRecord(r Report, before *nodeengine.HaveByNodeNod n := nodeFacts(r) if q.Keys != nil { keys := q.Keys() - keys.Stated = q.runnerStated + keys.Stated, keys.StatedBy = q.runnerStated, q.runnerFingerprint n.Record = recordOf(q.Membership.Node, r, keys) } if before != nil && before.Record != nil { diff --git a/internal/link/have_test.go b/internal/link/have_test.go index 132fa730..a4f0bd16 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -344,7 +344,8 @@ func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) { keys := RecordKeys{TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert", Trusts: []string{"controller key sha256:aa", "vault key sha256:bb"}} // and what it trusts (L3 s6, ADR 0550 §2) q.Keys = func() RecordKeys { return keys } - l.stated = map[string]string{CapabilitiesKey: "ledger-calls, bus-state-calls"} // the tool runner's calls (ENG-2) + l.stated = map[string]string{CapabilitiesKey: "ledger-calls, bus-state-calls", // the tool runner's calls (ENG-2) + FingerprintKey: "sha256:runner"} // and the runner that stated them (design 46 approval mark §6) rep.Profile = map[string]any{"capabilities": []any{map[string]any{"name": "systemd", "present": true}, map[string]any{"name": "docker", "present": false}, map[string]any{"name": "apt", "present": true}}, "machine": map[string]any{"networks": []any{map[string]any{"link": "eth0", "network": "192.0.2.0/24"}}}} @@ -357,7 +358,8 @@ func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) { wantRecord := nodeengine.NodeRecord{V: sdkkeys.Version, Node: "n1", TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert", Trusts: keys.Trusts, Capabilities: []string{"apt", "bus-state-calls", "ledger-calls", "systemd", CapabilityWantParts}, OutwardLinks: []string{"eth0"}, NetworksSaid: true, Networks: []nodeengine.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, Contract: NodeKeyContract, - ManifestReads: catalogue.ManifestReads()} // what it reads of a manifest: its contract's (ADR 0614 §4) + ManifestReads: catalogue.ManifestReads(), // what it reads of a manifest: its contract's (ADR 0614 §4) + RunnerCapabilities: []string{"bus-state-calls", "ledger-calls"}, RunnerFingerprint: "sha256:runner"} if !nodeengineclient.SameNodeRecord(*got.Record, wantRecord) { t.Errorf("record = %+v, want %+v", *got.Record, wantRecord) } diff --git a/internal/link/record.go b/internal/link/record.go index bd93ea82..8cf26bc6 100644 --- a/internal/link/record.go +++ b/internal/link/record.go @@ -33,6 +33,9 @@ type RecordKeys struct { // Stated is the capabilities its tool runner states of the calls it answers (CapabilitiesKey), which the record says // beside the machine's own: a module whose manifest asks for one is held off a machine whose runner does not state it. Stated []string + // StatedBy is the fingerprint of the tool runner that stated Stated, as its PING carries it (FingerprintKey): the record + // takes both, so a reading is tied to the runner that made it (design 46 approval mark §6, lead answer A-203). + StatedBy string } // buildReads are the words and forms this engine reads of a manifest (novox/hq ADR 0614 §4): the reader capabilities of @@ -135,7 +138,13 @@ func recordOf(node string, r Report, keys RecordKeys) *nodeengine.NodeRecord { if c != "" && !slices.Contains(rec.Capabilities, c) { rec.Capabilities = append(rec.Capabilities, c) } + if c != "" { + rec.RunnerCapabilities = append(rec.RunnerCapabilities, c) + } } + // The runner's own reading beside the merge, which stays for placement (design 46 approval mark §6, lead answer A-203). + sort.Strings(rec.RunnerCapabilities) + rec.RunnerFingerprint = keys.StatedBy sort.Strings(rec.Capabilities) if p.Machine != nil { rec.NetworksSaid = true diff --git a/internal/link/session.go b/internal/link/session.go index 4c2e3704..986c5cc4 100644 --- a/internal/link/session.go +++ b/internal/link/session.go @@ -92,6 +92,8 @@ type Session struct { // runnerStated is the capabilities this machine's tool runner last stated of the calls it answers (askRunner), said in the // node's record. Held under saying. runnerStated []string + // runnerFingerprint is the fingerprint of the runner that stated runnerStated (FingerprintKey). Held under saying. + runnerFingerprint string // saying is held while a facts event is noted, so the haves are written in the order the facts are made. saying sync.Mutex @@ -581,5 +583,5 @@ func (q *Session) askRunner(ctx context.Context, bus Bus) { stated = append(stated, c) } } - q.runnerStated = stated + q.runnerStated, q.runnerFingerprint = stated, strings.TrimSpace(md[FingerprintKey]) } diff --git a/internal/link/witnessing.go b/internal/link/witnessing.go index 14cd1ecc..455bfd12 100644 --- a/internal/link/witnessing.go +++ b/internal/link/witnessing.go @@ -56,6 +56,9 @@ func (l *natsLink) Ping(ctx context.Context, service, id string) error { const ( ToolRunnerService = "node-tools" CapabilitiesKey = "capabilities" + // FingerprintKey is the metadata key of its PING it states its own fingerprint under, beside CapabilitiesKey (design 46 + // approval mark §6, lead answer A-203). + FingerprintKey = "fingerprint" ) // Stater is a link that can ask what a service's instance states of itself in its PING: its metadata. -- 2.54.0 From 77e6e48ecd12ad170bbaaeb9e185e9439b119aa9 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 06:26:10 +0100 Subject: [PATCH 43/59] AM-H1: the node-engine's composition names the withheld subjects, read through mesh-grants' withheld MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The node-engine never reads the grants part's state: it asks the seat's verb withheld in its own name, and says withheld unknown when nothing answers. Builds: design 46 7 §6 (the composition reads them through the verb withheld; the node-engine's manifest gains mesh-grants in uses; "withheld unknown: the grants part does not answer"), lead answer A-239 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- cmd/mesh-host/engineseat.go | 15 ++++++++-- cmd/mesh-host/main.go | 2 +- internal/agent/composition.go | 11 ++++++++ internal/link/movesecret_test.go | 25 +++++++++++++++++ internal/link/withheld.go | 47 ++++++++++++++++++++++++++++++++ module.json | 3 ++ 6 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 internal/link/withheld.go diff --git a/cmd/mesh-host/engineseat.go b/cmd/mesh-host/engineseat.go index 2b14c308..a1ccbe1d 100644 --- a/cmd/mesh-host/engineseat.go +++ b/cmd/mesh-host/engineseat.go @@ -18,6 +18,7 @@ import ( occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" stdio "git.novox.be/novox/mesh-sdk/go" "github.com/novox/mesh-host/internal/accounts" + "github.com/novox/mesh-host/internal/agent" "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/selfcompose" @@ -284,7 +285,7 @@ const compositionReadFor = 20 * time.Second // compositionFrom answers the verb composition from the node agent that runs now (issue 774); refused in words while // none does. -func compositionFrom(current *agentNow) func(module, digest string, diff bool) (any, error) { +func compositionFrom(current *agentNow, withheld func(ctx context.Context, module string) ([]link.WithheldSubject, string)) func(module, digest string, diff bool) (any, error) { return func(module, digest string, diff bool) (any, error) { ag := current.get() if ag == nil { @@ -292,7 +293,17 @@ func compositionFrom(current *agentNow) func(module, digest string, diff bool) ( } ctx, cancel := context.WithTimeout(context.Background(), compositionReadFor) defer cancel() - return ag.Composition(ctx, module, digest, diff) + view, err := ag.Composition(ctx, module, digest, diff) + if err != nil { + return view, err + } + // And what the grants part withholds of it here, through its verb withheld (design 46 approval mark §6). + subjects, unknown := withheld(ctx, module) + for _, s := range subjects { + view.Withheld = append(view.Withheld, agent.WithheldView{Subject: s.Subject, Why: s.Why}) + } + view.WithheldUnknown = unknown + return view, nil } } diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index eb47ba29..513a82b3 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1022,7 +1022,7 @@ func runLink(ctx context.Context, opts options) error { queue.Acts = engineActs(opts, mine.Node, queue, setuidSearch) go reconcileActs(ctx, queue.Acts) current := &agentNow{} - queue.Acts.Composition = compositionFrom(current) + queue.Acts.Composition = compositionFrom(current, queue.WithheldOf) queue.Acts.Compose = composeFrom(current, &composerNow) queue.Acts.Packages = packagesFrom(mine.Node, queue, &composerNow) queue.Acts.FirstViews, queue.Acts.EntryAgain = firstViewsFrom(&composerNow), entryAgainFrom(&composerNow) diff --git a/internal/agent/composition.go b/internal/agent/composition.go index 37c89d99..9539952c 100644 --- a/internal/agent/composition.go +++ b/internal/agent/composition.go @@ -44,6 +44,17 @@ type CompositionView struct { // Own is, of the node's own composition (module node), its own fields beside what runs names (issue 774: ace's // own composition dropped the provisioner client's account, and nothing read could show it). Own *OwnView `json:"own,omitempty"` + // Withheld are the module's marked subjects the grants part withholds from this node's runtime, each with why, read + // through the seat mesh-grants' verb withheld; WithheldUnknown says when that verb did not answer (design 46 approval + // mark §6, lead answer A-239). + Withheld []WithheldView `json:"withheld,omitempty"` + WithheldUnknown string `json:"withheld-unknown,omitempty"` +} + +// WithheldView is one marked subject withheld, and why. +type WithheldView struct { + Subject string `json:"subject"` + Why string `json:"why"` } // OwnView is the node's own fields of a composition (engcomp.NodeOwn) beside what runs of them, and what the node's diff --git a/internal/link/movesecret_test.go b/internal/link/movesecret_test.go index fc017566..ca6385ce 100644 --- a/internal/link/movesecret_test.go +++ b/internal/link/movesecret_test.go @@ -139,4 +139,29 @@ func TestNatsAMoveIsAskedOfTheNodeAndTheVaultIsAskedInItsName(t *testing.T) { if _, err := (&Session{Membership: Membership{Node: "nowhere"}}).AskVault(t.Context(), "move-in", nil); !errors.Is(err, ErrNotAsked) { t.Fatalf("a queue with no link asked the vault: %v", err) } + // AM-H1 (design 46 approval mark §6, lead answer A-239): the composition reads what the grants part withholds through + // its verb withheld, asked in the engine's name, and says withheld unknown when nothing answers. + grants, err := conn.Subscribe("mesh.seat.mesh-grants.call.withheld.node~"+m.Node, func(msg *nats.Msg) { + _ = msg.Respond([]byte(`{"result":{"node":"` + m.Node + `","modules":[{"module":"shell","subjects":[` + + `{"subject":"mesh.mod.shell.tool.execute","why":"its tool runner does not state approval-mark/1"}]}]}}`)) + }) + if err != nil { + t.Fatal(err) + } + if err := conn.Flush(); err != nil { + t.Fatal(err) + } + if got, unknown := q.WithheldOf(t.Context(), "shell"); unknown != "" || len(got) != 1 || got[0].Subject != "mesh.mod.shell.tool.execute" { + t.Errorf("withheld of shell = %+v (%q)", got, unknown) + } + if got, unknown := q.WithheldOf(t.Context(), "other"); unknown != "" || len(got) != 0 { + t.Errorf("withheld of a module the grants part names none of = %+v (%q)", got, unknown) + } + _ = grants.Unsubscribe() + if err := conn.Flush(); err != nil { + t.Fatal(err) + } + if got, unknown := q.WithheldOf(t.Context(), "shell"); unknown != WithheldUnknown || got != nil { + t.Errorf("with the grants part down, withheld = %+v (%q)", got, unknown) + } } diff --git a/internal/link/withheld.go b/internal/link/withheld.go new file mode 100644 index 00000000..f0d005b7 --- /dev/null +++ b/internal/link/withheld.go @@ -0,0 +1,47 @@ +package link + +import ( + "context" + "encoding/json" +) + +// **The composition names the withheld subjects** (design 46 approval mark §6, lead answer A-239): the node-engine reads them +// through the seat mesh-grants' verb withheld, never from the grants part's state; when the verb does not answer, it says so. + +// GrantsSeat is the seat of the grants part, whose verb withheld names the marked subjects withheld from a node's runtime. +const GrantsSeat = "mesh-grants" + +// WithheldUnknown is what a composition says when the grants part does not answer withheld (design 46 approval mark §6). +const WithheldUnknown = "withheld unknown: the grants part does not answer" + +// WithheldSubject is one marked subject withheld from this node's runtime, and why. +type WithheldSubject struct { + Subject string `json:"subject"` + Why string `json:"why"` +} + +// WithheldOf is what the grants part withholds of module on this node, asked through its verb withheld: the subjects, or, +// when the verb does not answer or answers what is not its answer, WithheldUnknown as unknown. +func (q *Session) WithheldOf(ctx context.Context, module string) (subjects []WithheldSubject, unknown string) { + raw, err := q.askCalled(ctx, GrantsSeat, "withheld", map[string]string{"node": q.Membership.Node}, VaultWithin, + "the grants part's withheld") + if err != nil { + return nil, WithheldUnknown + } + var answer struct { + Node string `json:"node"` + Modules []struct { + Module string `json:"module"` + Subjects []WithheldSubject `json:"subjects"` + } `json:"modules"` + } + if json.Unmarshal(raw, &answer) != nil || answer.Node != q.Membership.Node { + return nil, WithheldUnknown + } + for _, m := range answer.Modules { + if m.Module == module { + subjects = append(subjects, m.Subjects...) + } + } + return subjects, "" +} diff --git a/module.json b/module.json index d97c6be5..9e262cfe 100644 --- a/module.json +++ b/module.json @@ -39,6 +39,9 @@ ] } ], + "uses": [ + "mesh-grants" + ], "invokes": [ "seat:mesh-catalog.entry", {"verb": "seat:mesh-catalog.revision", "fields": ["none-yet"]}, -- 2.54.0 From bef467febe700cc9b25220e5c07d0c354df675d8 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 09:42:32 +0100 Subject: [PATCH 44/59] H-revert: the node agent starts a held revert on the live data when its want says revert live MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A want with `revert` `live` starts the reverted build with no restore, no aside taken and no catch-up, and applies as any want; without the field a revert with no restore point stays held. Builds: design 47 G §710 (G-21-provider-and-revert-ask.md, "On the answer"), §300, §567 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- internal/agent/agent.go | 3 ++- internal/agent/kept.go | 8 +++++++- internal/agent/kept_test.go | 30 ++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 91c48506..33c2ff60 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -1441,7 +1441,8 @@ func (a *Agent) settleOne(ctx context.Context, module string) { // be swallowed by "it runs what its want names" (issue 1585, the second failure). The operator's answer, again or // skip, in a revision after the one the hold took, is acted on whatever runs; any other new revision of the want // is the answer only for a move (ADR 0568 §4), as before. - answered := w.Rev > att.AnsweredRev && (w.Want.Backup == meshwants.WantsByWantNodeModuleValueBackupAgain || w.Want.Backup == meshwants.WantsByWantNodeModuleValueBackupSkip) + answered := w.Rev > att.AnsweredRev && (w.Want.Backup == meshwants.WantsByWantNodeModuleValueBackupAgain || w.Want.Backup == meshwants.WantsByWantNodeModuleValueBackupSkip || + w.Want.Revert == meshwants.WantsByWantNodeModuleValueRevertLive) if !answered && (runsWant || (att.AnsweredRev >= w.Rev && att.CompRev == comp.Rev)) { a.sayOnce("held:"+module, fmt.Sprintf("%s is held for a failed backup before %s: it waits for the operator's "+ "answer, and its want (revision %d) carries none since the hold (revision %d)", module, w.Want.Fp, w.Rev, att.AnsweredRev)) diff --git a/internal/agent/kept.go b/internal/agent/kept.go index 052e4bc9..1cbc3ef4 100644 --- a/internal/agent/kept.go +++ b/internal/agent/kept.go @@ -212,7 +212,7 @@ func (a *Agent) askAndWait(ctx context.Context, h Have, kind string, k KeptMove, // keptStep takes the restore points a move of a module needs before it applies and restores the ones a revert needs: it // answers whether the module is held (its have says why, and the running build is kept) and, after a restore, the catch-up -// of the point to call once the reverted build runs. A want that says to go on without a backup asks neither. +// of the point to call once the reverted build runs. A want that says to go on without a backup asks neither, and nor does one that answers a held revert live. func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp compNow, ranBefore bool, others []compositions.Composed) (held bool, catchUps []meshrecovery.CatchUp) { r := a.env.Recovery if !ranBefore || r == nil || r.FormChanges == nil || w.Want.Backup == meshwants.WantsByWantNodeModuleValueBackupSkip { @@ -248,6 +248,12 @@ func (a *Agent) keptStep(ctx context.Context, module string, w wantNow, comp com if len(take)+len(restore) == 0 { return false, nil } + // The operator's answer "live" to a held revert, read from this want and from nothing else: the reverted build starts on the live + // data as it is, with no restore, no aside taken and no catch-up, and applies as any want (design 47 G section 710). + if len(restore) > 0 && w.Want.Revert == meshwants.WantsByWantNodeModuleValueRevertLive { + a.env.Say(fmt.Sprintf("%s starts %s on the live data as it is: its want (revision %d) carries the operator's answer %q to the held revert", module, fp, w.Rev, w.Want.Revert)) + return false, nil + } hold := func(words string, interrupted bool) { if interrupted || ctx.Err() != nil { words = BackupInterrupted diff --git a/internal/agent/kept_test.go b/internal/agent/kept_test.go index 31b0272c..cd671f79 100644 --- a/internal/agent/kept_test.go +++ b/internal/agent/kept_test.go @@ -12,7 +12,9 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "git.novox.be/novox/mesh-sdk/go/catalogue" + "github.com/novox/mesh-host/internal/buckets/bucketstest" "github.com/novox/mesh-host/internal/compositions" ) @@ -242,6 +244,34 @@ func testAMoveAcrossAFormChange(t *testing.T) { } }) + t.Run("a held revert, then the want with revert live, starts the older build with no restore, no aside and no catch-up (design 47 G section 710)", func(t *testing.T) { + f := &recoveryFake{changes: down} + b := moveDB(t, f, "a", "b", heldHave) + if h := b.haves.last("db"); !contains(h.Detail, "restore point missing") { + t.Fatalf("the have %+v", h) + } + before := len(b.appliedList()) + _, _, dig2 := composition(t, "db", fpOf("b"), true, file("db", "x", "2")) + raw, err := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fpOf("b"), Composed: dig2, Revert: meshwants.WantsByWantNodeModuleValueRevertLive, + Words: "the operator answered \"live\" to the revert ask (asking 1)"}) + if err != nil { + t.Fatal(err) + } + b.wants.put(meshwants.WantsByWantNodeModuleKey(node, "db"), raw) + eventually(t, "the older build runs on the answer", func() bool { h := b.haves.last("db"); return h.FP == fpOf("b") && h.Phase != "held" }) + if got := f.seen(); !slices.Equal(got, []string{"stop", "kept-points", "apply", "apply"}) && len(b.appliedList()) <= before { + t.Fatalf("calls %v", got) + } + for _, c := range f.seen() { + if c == "restore-kept" || c == "snapshot-kept" || c == "catch-up" { + t.Fatalf("calls %v", f.seen()) + } + } + if n := strings.Count(strings.Join(f.seen(), " "), "kept-points"); n != 1 { + t.Fatalf("the points were read again on the answer: %v", f.seen()) + } + }) + t.Run("a revert whose restore fails starts the newer build again and holds the move, saying why (A-113)", func(t *testing.T) { f := &recoveryFake{changes: down, points: []meshrecovery.Point{{Module: "db", Kept: "cache", Form: 1, Fingerprint: fp("a"), Snapshot: "snap-1", At: time.Now()}}, -- 2.54.0 From d18bf8b3fd3518b6582e14fb71040293608ce827 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 11:07:38 +0100 Subject: [PATCH 45/59] The node-engine's seat reader reads invokes in both forms and refuses a form it cannot read MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G A-275, §558 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- internal/selfcompose/kept_test.go | 19 +++++++++++++ internal/selfcompose/selfcompose.go | 44 +++++++++++++++++++---------- 2 files changed, 48 insertions(+), 15 deletions(-) diff --git a/internal/selfcompose/kept_test.go b/internal/selfcompose/kept_test.go index c5eea439..4ef6f1d5 100644 --- a/internal/selfcompose/kept_test.go +++ b/internal/selfcompose/kept_test.go @@ -2,9 +2,11 @@ package selfcompose import ( "context" + "slices" "strings" "testing" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" ) @@ -33,3 +35,20 @@ func TestTheKeepingNodeIsTheWantsKeptOn(t *testing.T) { } } } + +// Design 47 G §558, A-275: a manifest's `invokes` in its long form {verb, fields} is read for its seat, and one of a form that +// cannot be read is refused, never read as no seats. +func TestSeatsOfReadsALongFormInvokeAndRefusesAnUnreadableOne(t *testing.T) { + c := New(Env{Node: "n1"}) + c.modules["m"] = meshcatalog.ModulesByModuleValue{Manifest: "d1"} + c.modules["bad"] = meshcatalog.ModulesByModuleValue{Manifest: "d2"} + c.manifests["d1"] = []byte(`{"module":"m","invokes":["seat:a.b",{"verb":"seat:mesh-catalog.revision","fields":["x"]}]}`) + c.manifests["d2"] = []byte(`{"module":"bad","invokes":[3]}`) + seats, err := c.seatsOf(context.Background(), reader{c: c}, "m") + if err != nil || !slices.Equal(seats, []string{"a", "mesh-catalog"}) { + t.Fatalf("seats %v, %v", seats, err) + } + if seats, err := c.seatsOf(context.Background(), reader{c: c}, "bad"); err == nil { + t.Fatalf("an unreadable invokes gave seats %v", seats) + } +} diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 2b60ca71..a5dcd90c 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -2035,7 +2035,7 @@ func (c *Composer) heldUntilListed(ctx context.Context, r reader, own meshmodule held := map[string]string{} for _, pm := range own.Modules { if pm.Module == "" || pm.Module == agent.NodeModule || slices.Contains(holders, pm.Module) || - c.env.SeatList.MayAct(c.seatsOf(ctx, r, pm.Module)...) { + c.mayAct(ctx, r, pm.Module) { continue } held[pm.Module] = line + composesNothingNewOfIt @@ -2094,7 +2094,7 @@ func (c *Composer) holdUntilListed(ctx context.Context, r reader, compositions m held[module] = WaitsForTheSeatListOwnPart continue } - } else if c.env.SeatList.MayAct(c.seatsOf(ctx, r, module)...) { + } else if c.mayAct(ctx, r, module) { continue } delete(compositions, key) @@ -2207,41 +2207,55 @@ func (c *Composer) claimants(ctx context.Context, r reader, seat string) []strin return out } -// seatsOf is the seats a module claims, invokes or provides, from the manifest its module record names; none when it -// cannot be read, which the reader judges as a module no bootstrapped seat names. -func (c *Composer) seatsOf(ctx context.Context, r reader, module string) []string { +// mayAct is whether the seat list lets module act: of the seats it claims or invokes (seatsOf). A manifest whose `invokes` +// is of a form that cannot be read is refused, never read as no seats (design 47 G A-275). +func (c *Composer) mayAct(ctx context.Context, r reader, module string) bool { + seats, err := c.seatsOf(ctx, r, module) + if err != nil { + return false // refused: held with the module's other waits, never judged as no seats + } + return c.env.SeatList.MayAct(seats...) +} + +// seatsOf is the seats a module claims or invokes, from the manifest its module record names; none when it cannot be +// fetched, which the reader judges as a module no bootstrapped seat names, and an error when its manifest is of a form that +// cannot be read (design 47 G §558, A-275). +func (c *Composer) seatsOf(ctx context.Context, r reader, module string) ([]string, error) { c.mu.Lock() m, ok := c.modules[module] c.mu.Unlock() if !ok || m.Manifest == "" { - return nil + return nil, nil } raw, err := r.Manifest(ctx, m.Manifest) if err != nil { - return nil + return nil, nil } var man struct { Claims []struct { Name string `json:"name"` } `json:"claims"` - Invokes []string `json:"invokes"` - Provides []struct { - Name string `json:"name"` - } `json:"provides"` + Invokes json.RawMessage `json:"invokes"` } - if json.Unmarshal(raw, &man) != nil { - return nil + if err := json.Unmarshal(raw, &man); err != nil { + return nil, fmt.Errorf("the manifest of %s cannot be read: %w", module, err) + } + var invokes []string + if len(man.Invokes) > 0 { + if invokes, err = catalogue.InvokedVerbs(man.Invokes); err != nil { + return nil, fmt.Errorf("the manifest of %s: %w", module, err) + } } var seats []string for _, cl := range man.Claims { seats = append(seats, cl.Name) } - for _, in := range man.Invokes { + for _, in := range invokes { if seat, _, ok := strings.Cut(strings.TrimPrefix(in, "seat:"), "."); ok && strings.HasPrefix(in, "seat:") { seats = append(seats, seat) } } - return seats + return seats, nil } // unmadeOf is each module wanted at a fingerprint (fps) of which the pass made no composition at it, with why, apart from -- 2.54.0 From 0c37ac378c078cd1cac95f3ba40c6b002cc7866e Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 14:13:34 +0100 Subject: [PATCH 46/59] Join 11: the node seat's discovery test names the engine's own module, not a fixed one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit H-17 made discovery read the node-engine's own name (§527) instead of the constant mesh-host, but the integration test still gave its link no own name and expected mesh-host, so every endpoint announced an empty module. It now gives the link an own name and expects that one. Builds: design 47 G §634, A-149 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- internal/link/handover_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/link/handover_test.go b/internal/link/handover_test.go index 563baf0c..1d386ef5 100644 --- a/internal/link/handover_test.go +++ b/internal/link/handover_test.go @@ -483,6 +483,10 @@ func TestNatsAHandOverIsAskedOfTheNode(t *testing.T) { }) // 774 (ADR 0563 §1): the node seat is served on this node's subjects alone, naming its caller, and said to discovery. t.Run("the node seat is served on this node and discovered", func(t *testing.T) { + // Discovery names the engine's own module as §527 reads it, no fixed name (design 47 G §634, A-149). + own := NewOwnName() + own.Set("the-engine") + l.UseOwnName(own) l.ServeEngineSeat(func(c EngineCall) (any, error) { return c.Verb + " for " + c.Caller, nil }) reply, err := conn.Request("mesh.seat.node-engine.call.acts."+m.Node+".shanks~node-tools", []byte(`{}`), 5*time.Second) if err != nil || !strings.Contains(string(reply.Data), `"result":"acts for shanks.node-tools"`) { @@ -509,7 +513,7 @@ func TestNatsAHandOverIsAskedOfTheNode(t *testing.T) { announced := 0 for _, e := range said.Endpoints { md := e.Metadata - if md["kind"] != "seat" || md["scope"] != "node" || md["seat"] != EngineSeat || md["module"] != "mesh-host" || + if md["kind"] != "seat" || md["scope"] != "node" || md["seat"] != EngineSeat || md["module"] != "the-engine" || md["node"] != m.Node || md["description"] == "" || !strings.HasPrefix(md["schema"], "{") { t.Errorf("the endpoint on %s announces %v", e.Subject, md) } -- 2.54.0 From a9b970991198ef0984cb8cf1620f04534b6befc2 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 16:57:36 +0100 Subject: [PATCH 47/59] J23 A-292: a node seat event the bus did not take is said again on the same link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After a restart, g14's engine restated its reads three seconds after its first, bootstrapped statement, and the assigner never heard that event: a say that failed past the bus's retry bound waited for the next link. Now a say the bus did not take (no responders, a deadline) is said again on the link until the bus takes it, off the caller, each failed attempt written to the engine's journal with the event's name; only a lost link, or a refusal the bus answered, keeps it for the next link. Builds: design 47 G §590, A-292 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- internal/link/have_test.go | 43 +++++++++++++++++++++++++- internal/link/session.go | 62 ++++++++++++++++++++++++++++++++++---- 2 files changed, 98 insertions(+), 7 deletions(-) diff --git a/internal/link/have_test.go b/internal/link/have_test.go index 88195a60..ed29f89c 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -45,6 +45,8 @@ type haveLink struct { readsSaid []nodeengine.ManifestReadsChangedEvent // sayFails fails that many of the node seat's says, as a publish past the bus's retry bound; said is every event said. sayFails int + // sayFault is what a failed say answers; a bus answering a refusal when nil. + sayFault error said []string // onSaid are the events said on a subject of their own (running), by subject, in order. onSaid []string @@ -77,7 +79,11 @@ func (l *haveLink) SayEngine(event string, body any) error { l.hmu.Lock() if l.sayFails > 0 { l.sayFails-- + fault := l.sayFault l.hmu.Unlock() + if fault != nil { + return fault + } return errors.New("nats: timeout") } l.said = append(l.said, event) @@ -320,7 +326,7 @@ func TestAModulesHealthFollowsTheLook(t *testing.T) { // 1132/AC6: the node's key holds its windows, busy, strays, filters and profile. func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) { l := newHaveLink() - q, _ := aHaveQueue(l) + q, journal := aHaveQueue(l) rep := appliedModules() rep.Windows = []Window{{Step: "backup", Holds: []string{"db"}}} rep.Busy = []Busy{{ID: "x", Container: "x", State: "waiting"}} @@ -465,6 +471,41 @@ func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) { catalogue.UseSeats(all) q.RestateReads(context.Background()) + // A-292 (§590, issue 1975): a restatement the bus did not take, past its retry bound, is said again on the same link + // until the bus takes it, never held for the next link, and each failed attempt is in the engine's journal by name. + saidBefore = len(l.readsSaid) + l.hmu.Lock() + l.sayFails, l.sayFault = 2, fmt.Errorf("saying: %w", nats.ErrTimeout) + l.hmu.Unlock() + catalogue.UseSeats(slices.DeleteFunc(slices.Clone(all), func(s catalogue.Seat) bool { return s.Name == "mesh-vault" })) + q.RestateReads(context.Background()) + q.retrying.Wait() + l.hmu.Lock() + again := slices.Clone(l.readsSaid[saidBefore:]) + l.sayFault = nil + l.hmu.Unlock() + if len(again) != 1 || slices.Contains(again[0].ManifestReads, "seat:mesh-vault") { + t.Errorf("a restatement the bus did not take, said on this link: %+v, want it said once with the new statement", again) + } + failed := 0 + for _, line := range *journal { + if strings.Contains(line, "could not say the node seat's "+nodeengine.EventManifestReadsChanged) && + strings.Contains(line, "on this link") { + failed++ + } + } + if failed != 2 { + t.Errorf("the journal names %d failed attempts of %s, want 2: %q", failed, nodeengine.EventManifestReadsChanged, *journal) + } + q.reads.Lock() + _, held := q.reads.unsaid[nodeengine.EventManifestReadsChanged] + q.reads.Unlock() + if held { + t.Errorf("a restatement said on this link is also held for the next link") + } + catalogue.UseSeats(all) + q.RestateReads(context.Background()) + // And the publish itself (sayOnce): through no responders, as the old JetStream client says it, and a deadline it is published again under one message id, // so the stream takes it once. t.Cleanup(busretry.Waits(10*time.Second, time.Millisecond, 2*time.Millisecond)) diff --git a/internal/link/session.go b/internal/link/session.go index ee9dc0ab..8e687287 100644 --- a/internal/link/session.go +++ b/internal/link/session.go @@ -16,6 +16,7 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/health" "git.novox.be/novox/mesh-sdk/go/operatorask" ) @@ -112,6 +113,8 @@ type Session struct { } // restating takes RestateReads one at a time. restating sync.Mutex + // retrying counts the says being said again on the link open (sayAgain). + retrying sync.WaitGroup // haves is what this node-engine writes beside its facts (novox/hq ADR 0429 §4, issue 1132). haves haveState @@ -524,22 +527,69 @@ func (q *Session) sayEngineOn(subject, event string, body any) { } } -// SayEngine says one of the node seat's events over the link open now. With none open, or a publish that failed past the -// bus's retry bound, it is kept and said at the next link's opening (SayReads; novox/hq ADR 0668 §4, issue 1966): an -// event a reader waits on is never only a line in this engine's log. +// SayEngine says one of the node seat's events over the link open now. A publish the bus did not take is said again on +// that link until it takes it (sayAgain; design 47 G A-292, §590); with no link open, or a link lost meanwhile, or a +// refusal the bus answered, it is kept and said at the next link's opening (SayReads; novox/hq ADR 0668 §4, issue 1966): +// an event a reader waits on is never only a line in this engine's log. func (q *Session) SayEngine(event string, body any) { q.init() q.mu.Lock() bus := q.bus q.mu.Unlock() - sayer, ok := bus.(interface{ SayEngine(string, any) error }) - var err error + sayer, ok := bus.(engineSayer) if ok { - if err = sayer.SayEngine(event, body); err == nil { + err := sayer.SayEngine(event, body) + if err == nil { + return + } + if sayableAgain(err) { + // A say the bus did not take is said again on this link until the bus takes it (design 47 G A-292, §590): + // off the caller, so a link's setup and a restatement never wait on it. + q.Say("could not say the node seat's " + event + ", said again on this link until the bus takes it: " + err.Error()) + q.retrying.Add(1) + go q.sayAgain(bus, sayer, event, body) return } q.Say("could not say the node seat's " + event + ", said again at the next link: " + err.Error()) } + q.holdUnsaid(event, body) +} + +// engineSayer is a link that says the node seat's events. +type engineSayer interface{ SayEngine(string, any) error } + +// sayableAgain is an error the SDK's bus retry asks again on: the bus took nothing (no responders, a deadline). An error +// the bus answered is not asked again on this link. +func sayableAgain(err error) bool { return busretry.Undelivered(err) || busretry.TimedOut(err) } + +// sayAgain says event again through the link's bus retry for as long as bus is the link open and the bus did not take it, +// writing each failed attempt to the engine's journal with the event's name (A-292). A link lost meanwhile keeps the +// event for the next link (holdUnsaid). +func (q *Session) sayAgain(bus Bus, sayer engineSayer, event string, body any) { + defer q.retrying.Done() + for q.linkedTo(bus) { + err := sayer.SayEngine(event, body) + if err == nil { + return + } + if !sayableAgain(err) { + q.Say("could not say the node seat's " + event + ", said again at the next link: " + err.Error()) + break + } + q.Say("could not say the node seat's " + event + " again, said again on this link until the bus takes it: " + err.Error()) + } + q.holdUnsaid(event, body) +} + +// linkedTo says bus is the link open now. +func (q *Session) linkedTo(bus Bus) bool { + q.mu.Lock() + defer q.mu.Unlock() + return q.bus != nil && q.bus == bus +} + +// holdUnsaid keeps event for the next link's SayReads: no link is open, or the bus answered it with a refusal. +func (q *Session) holdUnsaid(event string, body any) { q.reads.Lock() if q.reads.unsaid == nil { q.reads.unsaid = map[string]any{} -- 2.54.0 From 905265a56b5231c49cd9cdcca96ca22d63d3c7d8 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 17:20:30 +0100 Subject: [PATCH 48/59] J25 A-293: a placed module's failed newest build said as refused in by-entries mode, on the builder's build-ended event MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reading the catalog half by fingerprint, the node-engine no longer reads the builder's builds, so #224's refused key was never written there and a reader waited on a build that will never be done. The engine now hears the builder's build.ended stream, read whole at start then followed (as the assigner hears it); the last trunk end of a placed module that ended failed is written as the refused key with the event's words, a later done end deletes it, an end naming a branch moves nothing. The manifest consumes mesh-delivery-builder.build.ended.**. Builds: design 47 G A-293 (with §421, §628), ADR 0593 §6, issue 1966 U1/U8 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- cmd/mesh-host/nodeagent.go | 24 ++++++- cmd/mesh-host/recovery_integration_test.go | 36 ++++++++++ internal/link/running.go | 81 ++++++++++++++++++++++ internal/link/session_test.go | 27 ++++++++ internal/selfcompose/composedstate.go | 52 +++++++++++++- internal/selfcompose/composedstate_test.go | 45 ++++++++++++ internal/selfcompose/selfcompose.go | 50 +++++++++++++ module.json | 1 + packaging/operator_keys_test.go | 4 +- 9 files changed, 312 insertions(+), 8 deletions(-) diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 09d85816..75310ea0 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -694,6 +694,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone session.SetManifestOf(composer.ManifestOf) // the event running names the manifest of the build that runs go followRunning(run, session, composer, say) + go followBuildEnds(run, session, composer, say) apply.UseEngineModule(composer.EngineModule) // the guard of the engine's own trees fixes no name (§527) env.Composed, env.SelfComposed, env.Policy, env.ComposedToday = composer, true, composer.Policy, composer.ComposedToday env.Recovery = recoveryEnv(session, composer) @@ -1204,11 +1205,28 @@ func composerEnv(session *link.Session, node string, say func(string), hooks com // followRunning keeps the composer told what every node runs, from the seat node-engine's event running, for the run it is // in: the stream or the link not being there is said and tried again after a backoff, never polled. func followRunning(run context.Context, session *link.Session, composer *selfcompose.Composer, say func(string)) { + keepFollowing(run, "this node does not hear what the nodes run, and tries again: ", func(ctx context.Context) error { + return session.FollowRunning(ctx, composer.NoteRunning, composer.RunningRead) + }, composer.RunningLost, say) +} + +// followBuildEnds keeps the composer told how the builder's builds ended, from its event build.ended, for the run it is in +// (design 47 G A-293): a placed module's newest build that ended failed is said as a refusal in the composition state. +func followBuildEnds(run context.Context, session *link.Session, composer *selfcompose.Composer, say func(string)) { + keepFollowing(run, "this node does not hear how the builder's builds end, so a failed build is not said as refused, and it tries again: ", + func(ctx context.Context) error { + return session.FollowBuildEnds(ctx, composer.NoteBuildEnded, composer.BuildEndsRead) + }, composer.BuildEndsLost, say) +} + +// keepFollowing follows a stream with follow for the run, lost called each time it ends; why it ended is said once until it +// changes, and it is followed again after a backoff, never polled. +func keepFollowing(run context.Context, line string, follow func(context.Context) error, lost func(), say func(string)) { said := "" for try := 1; run.Err() == nil; try++ { began := time.Now() - err := session.FollowRunning(run, composer.NoteRunning, composer.RunningRead) - composer.RunningLost() + err := follow(run) + lost() if run.Err() != nil { return } @@ -1216,7 +1234,7 @@ func followRunning(run context.Context, session *link.Session, composer *selfcom try = 1 } if why := fmt.Sprint(err); why != said { - say("this node does not hear what the nodes run, and tries again: " + why) + say(line + why) said = why } select { diff --git a/cmd/mesh-host/recovery_integration_test.go b/cmd/mesh-host/recovery_integration_test.go index 14718427..2f665064 100644 --- a/cmd/mesh-host/recovery_integration_test.go +++ b/cmd/mesh-host/recovery_integration_test.go @@ -11,6 +11,7 @@ import ( "time" "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "github.com/novox/mesh-host/internal/link" @@ -131,6 +132,41 @@ func TestNatsTheRecoveryStepAsksHearsBoundsAndCatchesUpOverTheBus(t *testing.T) case <-time.After(10 * time.Second): t.Fatal("the bound's end, the bus's marker, was not heard") } + + // Design 47 G A-293: over the same session, the builder's build ends the composer follows: the stream read whole first (the + // last end of each module and fingerprint, in the stream's order), then the read's end, then each end said after it. + if _, err := b.js.AddStream(&nats.StreamConfig{Name: link.BuildEndsStream, Subjects: []string{link.BuildEndsSubject}, MaxMsgsPerSubject: 1}); err != nil { + t.Fatal(err) + } + ended := func(fp, outcome, why string) { + raw, _ := json.Marshal(builds.BuildEnded{Module: "db", Fingerprint: fp, Outcome: outcome, Error: why}) + if _, err := b.js.Publish("mesh.stream.mesh-delivery-builder.event."+builds.BuildEndedEventName("db", fp), raw); err != nil { + t.Fatal(err) + } + } + ended("f1", builds.BuildFailed, "first") + ended("f1", builds.BuildFailed, "no go") + ended("f0", builds.BuildDone, "") + heard := make(chan string, 8) + follow, stopFollow := context.WithCancel(ctx) + defer stopFollow() + go func() { + _ = session.FollowBuildEnds(follow, func(e builds.BuildEnded) { heard <- e.Fingerprint + ":" + e.Outcome + ":" + e.Error }, + func() { heard <- "read" }) + }() + for i, want := range []string{"f1:failed:no go", "f0:done:", "read", "f2:failed:later"} { + if want == "f2:failed:later" { + ended("f2", builds.BuildFailed, "later") + } + select { + case got := <-heard: + if got != want { + t.Fatalf("build end %d heard %q, want %q", i, got, want) + } + case <-time.After(10 * time.Second): + t.Fatalf("build end %d, %q, was not heard", i, want) + } + } } // startSession is a session on the node's own connection, with no node agent run on it. diff --git a/internal/link/running.go b/internal/link/running.go index fa24b0d1..c1d76213 100644 --- a/internal/link/running.go +++ b/internal/link/running.go @@ -6,6 +6,8 @@ import ( "fmt" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/nats-io/nats.go" ) @@ -105,3 +107,82 @@ func (b OverNATS) HearRunning(consumer string, heard func([]byte)) (func(), erro } var _ RunningBus = OverNATS{} + +// The builder's event build.ended (design 47 G A-293, as the assigner hears it, A-181, §556): the builder's stream of build +// ends keeps the last message of each module and fingerprint, and the node reads it whole at each start, then follows it, through +// one ordered consumer it makes for the run: what ended, and a failed build's words, are known from the stream however long ago +// they were said, never from the builder's builds. The engine's manifest consumes `mesh-delivery-builder.build.ended.**`. + +// BuildEndsStream is the builder's stream of build ends as the bus names it, BuildEndsSubject every event it carries. +var ( + BuildEndsStream = catalogue.StreamBusName(builds.BuildsStream) + BuildEndsSubject = "mesh.stream.mesh-delivery-builder.event." + builds.BuildEndedEvent + ".>" +) + +// BuildEndsBus is a link that hears the builder's build ends. +type BuildEndsBus interface { + // HearBuildEnds calls heard with the last message of every subject, then readEnded once they are all read, then heard with + // each after them, until ctx ends (nil) or the stream is lost (its error). + HearBuildEnds(ctx context.Context, heard func([]byte), readEnded func()) error +} + +func hearBuildEnds(ctx context.Context, js nats.JetStreamContext, heard func([]byte), readEnded func()) error { + sub, err := js.SubscribeSync(BuildEndsSubject, nats.OrderedConsumer(), nats.DeliverLastPerSubject(), nats.BindStream(BuildEndsStream)) + if err != nil { + return fmt.Errorf("the stream %s cannot be read: %w", BuildEndsStream, err) + } + defer sub.Unsubscribe() + info, err := sub.ConsumerInfo() + if err != nil { + return fmt.Errorf("the stream %s cannot be read: %w", BuildEndsStream, err) + } + total := info.Delivered.Consumer + info.NumPending + for got := uint64(0); ; got++ { + if got == total { + readEnded() + } + msg, err := sub.NextMsgWithContext(ctx) + if err != nil { + if ctx.Err() != nil { + return nil + } + return fmt.Errorf("the stream %s is not followed any more (%d read): %w", BuildEndsStream, got, err) + } + heard(msg.Data) + } +} + +func (l *natsLink) HearBuildEnds(ctx context.Context, heard func([]byte), readEnded func()) error { + return hearBuildEnds(ctx, l.js, heard, readEnded) +} + +// HearBuildEnds reads the stream as the link's session does, over its connection. +func (b OverNATS) HearBuildEnds(ctx context.Context, heard func([]byte), readEnded func()) error { + return hearBuildEnds(ctx, b.JS, heard, readEnded) +} + +var ( + _ BuildEndsBus = (*natsLink)(nil) + _ BuildEndsBus = OverNATS{} +) + +// FollowBuildEnds hands every build end the builder said to heard: the last of each module and fingerprint first, with +// readEnded called after them (until then the caller acts on none), then each after them, until ctx ends. A body that is no +// end of a module at a fingerprint is passed over. An error is the stream or the link not being there, which the caller tries +// again. +func (q *Session) FollowBuildEnds(ctx context.Context, heard func(builds.BuildEnded), readEnded func()) error { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + h, ok := bus.(BuildEndsBus) + if bus == nil || !ok { + return ErrNoLink + } + return h.HearBuildEnds(ctx, func(raw []byte) { + var e builds.BuildEnded + if json.Unmarshal(raw, &e) == nil && e.Module != "" && e.Fingerprint != "" { + heard(e) + } + }, readEnded) +} diff --git a/internal/link/session_test.go b/internal/link/session_test.go index 64783c1d..21234b77 100644 --- a/internal/link/session_test.go +++ b/internal/link/session_test.go @@ -12,6 +12,7 @@ import ( "time" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "github.com/nats-io/nats.go" ) @@ -45,6 +46,17 @@ func (l *runningLink) HearRunning(_ string, heard func([]byte)) (func(), error) return func() {}, nil } +func (l *runningLink) HearBuildEnds(_ context.Context, heard func([]byte), readEnded func()) error { + for _, raw := range l.first { + heard(raw) + } + readEnded() + for _, raw := range l.after { + heard(raw) + } + return nil +} + func aMember(t *testing.T) (Membership, ed25519.PrivateKey) { t.Helper() public, private, err := ed25519.GenerateKey(nil) @@ -84,6 +96,21 @@ func TestFactsWithNoLinkOpenAreNotSaid(t *testing.T) { if err != nil || strings.Join(order, " ") != "a/x b/y read" { t.Fatalf("followed %v: %v, want a/x b/y read", order, err) } + + // Design 47 G A-293: the builder's build ends likewise, the last of each subject first, then the read's end, then the + // rest, a body that is no end of a module at a fingerprint passed over. + if err := (&Session{Membership: m}).FollowBuildEnds(context.Background(), func(builds.BuildEnded) {}, func() {}); !errors.Is(err, ErrNoLink) { + t.Fatalf("build ends followed with no link: %v", err) + } + rl.first = [][]byte{[]byte(`{"module":"x","fingerprint":"f1","outcome":"failed","error":"no go"}`), []byte(`not json`), []byte(`{"module":"x"}`)} + rl.after = [][]byte{[]byte(`{"module":"y","fingerprint":"f2","outcome":"done","branch":"b"}`)} + order = nil + err = s.FollowBuildEnds(context.Background(), func(e builds.BuildEnded) { + order = append(order, e.Module+"@"+e.Fingerprint+":"+e.Outcome+":"+e.Error+":"+e.Branch) + }, func() { order = append(order, "read") }) + if err != nil || strings.Join(order, " ") != "x@f1:failed:no go: read y@f2:done::b" { + t.Fatalf("build ends followed %v: %v", order, err) + } } // A link opening says the node's facts, off the link's own loop. diff --git a/internal/selfcompose/composedstate.go b/internal/selfcompose/composedstate.go index 8496b967..da0eec31 100644 --- a/internal/selfcompose/composedstate.go +++ b/internal/selfcompose/composedstate.go @@ -91,8 +91,19 @@ func (c *Composer) publishComposed(ctx context.Context) { func (c *Composer) newestFPs(ctx context.Context, at *published) (out map[string]string, failed map[string][2]string) { if c.byEntries() { // The newest build is the indexer's (and the builder's done key says it is built): neither is read, and the node - // follows no branch (design 47 G §421, §628), so the state holds the wanted builds alone. - return maps.Clone(at.fps), nil + // follows no branch (design 47 G §421, §628), so the state holds the wanted builds alone. A placed module's newest + // build that ended is the last trunk end the builder said of it (NoteBuildEnded): a failed one is refused (A-293). + failed = map[string][2]string{} + c.mu.Lock() + if c.endsRead { + for _, module := range at.placed { + if e, ok := c.ends[module]; ok && e.Outcome == builds.BuildFailed { + failed[module] = [2]string{e.Fingerprint, e.Error} + } + } + } + c.mu.Unlock() + return maps.Clone(at.fps), failed } out = map[string]string{} failed = map[string][2]string{} @@ -153,7 +164,10 @@ func (c *Composer) writeComposed(ctx context.Context, at *published) (wrote, del wrote++ c.saidComposed(key, values[key]) } - for _, key := range c.pastKeeping() { + for _, key := range append(c.pastKeeping(), c.endedRefusals(values)...) { + if _, kept := c.pubKept[key]; !kept { + continue // past keeping and ended both + } if err := c.env.ComposedState.Delete(ctx, key); err != nil { return wrote, deleted, fmt.Errorf("%s cannot be deleted: %w", key, err) } @@ -271,6 +285,38 @@ func (c *Composer) composedValues(ctx context.Context, at *published) (map[strin return out, nil } +// endedRefusals is, of the keys the state holds, each refusal a failed end of a module wrote (newestFPs) once that module's +// last trunk end is done, at any fingerprint (design 47 G A-293): the state never says refused for a done build. A key this +// pass writes (values), or one a want names, is not one. Under pubWrite. +func (c *Composer) endedRefusals(values map[string][]byte) []string { + c.mu.Lock() + defer c.mu.Unlock() + if !c.byEntries() || !c.endsRead { + return nil + } + var out []string + for module, last := range c.ends { + if last.Outcome != builds.BuildDone { + continue + } + for fp := range c.endFPs[module] { + key, err := engcomp.Key(c.env.Node, module, fp) + if err != nil || values[key] != nil || c.wants[module].Fp == fp { + continue + } + kept, ok := c.pubKept[key] + if !ok { + continue + } + if comp, err := engcomp.Decode(kept.Value); err == nil && comp.Refused != "" { + out = append(out, key) + } + } + } + sort.Strings(out) + return out +} + // composedKeyOf is a composition's key in the state. func composedKeyOf(comp engcomp.Composed) (string, error) { if comp.Module == "" { diff --git a/internal/selfcompose/composedstate_test.go b/internal/selfcompose/composedstate_test.go index e87f0fc5..68f9721d 100644 --- a/internal/selfcompose/composedstate_test.go +++ b/internal/selfcompose/composedstate_test.go @@ -2,6 +2,8 @@ package selfcompose import ( "context" + "encoding/json" + "errors" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" @@ -12,6 +14,7 @@ import ( "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/buckets/bucketstest" engcomp "github.com/novox/mesh-host/internal/compositions" + "maps" "slices" "strings" "sync" @@ -258,4 +261,46 @@ func TestTheNodePublishesItsCompositions(t *testing.T) { if past := keep.pastKeeping(); len(past) != 0 { t.Errorf("past keeping with a want naming the oldest: %v, want none", past) } + + // Design 47 G A-293: reading the catalog half by fingerprint, a placed module's newest build that ended is the last trunk + // build.ended heard of it; a failed one is said refused with the event's words once the first read ended, an end naming a + // branch moves nothing, a module with no end (queued or running) stays unpublished, and a later done end, at any + // fingerprint, deletes the refused key. + byEnv := b.env() + byEnv.Entry = EntryCall(func(context.Context, string, string, any) (json.RawMessage, error) { + return nil, errors.New("nothing is asked") + }) + byState := &fakeComposed{kv: map[string]ComposedEntry{}} + byEnv.ComposedState = byState + by := New(byEnv) + byAt := &published{fps: map[string]string{}, placed: []string{"broken", "waiting", "branched"}} + words := `refused kind: package of language "go"` + by.NoteBuildEnded(builds.BuildEnded{Module: "broken", Fingerprint: bfp, Outcome: builds.BuildFailed, Error: words}) + by.NoteBuildEnded(builds.BuildEnded{Module: "branched", Fingerprint: wfp, Outcome: builds.BuildFailed, Error: "no", Branch: "feature"}) + for _, step := range []struct { + name string + do func() + holds map[string]string // by key, the refusal it says + }{ + {"before the first read ended", func() {}, map[string]string{}}, + {"once the first read ended", by.BuildEndsRead, map[string]string{brokenKey: words}}, + {"a later done end", func() { + by.NoteBuildEnded(builds.BuildEnded{Module: "broken", Fingerprint: wfp, Outcome: builds.BuildDone}) + }, map[string]string{}}, + } { + step.do() + if _, _, err := by.writeComposed(ctx, byAt); err != nil { + t.Fatalf("%s: %v", step.name, err) + } + byState.mu.Lock() + got := map[string]string{} + for k, v := range byState.kv { + comp, _ := engcomp.Decode(v.Value) + got[k] = comp.Refused + } + byState.mu.Unlock() + if !maps.Equal(got, step.holds) { + t.Errorf("%s: the state holds %v, want %v", step.name, got, step.holds) + } + } } diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 67f20f13..ea0d7af1 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -204,6 +204,12 @@ type Composer struct { // node's own consumer (NoteRunning), under mu; heardRead is whether the first read of the stream ended (RunningRead). heard map[string]map[string]heardRun heardRead bool + // ends is, by module, the last build end naming no branch the builder said of it (NoteBuildEnded), its newest build that + // ended, under mu; endsRead is whether the first read of the builder's stream ended (BuildEndsRead) (design 47 G A-293). + ends map[string]builds.BuildEnded + endsRead bool + // endFPs is, by module, every fingerprint a trunk end was heard of: the keys a failed end may have written. + endFPs map[string]map[string]bool // engine is the node-engine's own module as the last composing of the node's input found it (Scope.Engine), under mu. engine string // rehand asks for a pass that hands again what the agent waits for (Waiting). @@ -2443,6 +2449,50 @@ func (c *Composer) RunningLost() { c.mu.Unlock() } +// NoteBuildEnded takes the builder's event build.ended (design 47 G A-293): of a module, the last end naming no branch (the +// trunk's) stands as its newest build that ended, a failed one said as a refusal in the composition state, a done one ending +// that refusal; an end naming a branch moves nothing. Once the first read of the stream ended, an end of a module placed here +// asks a pass. +func (c *Composer) NoteBuildEnded(e builds.BuildEnded) { + if e.Module == "" || e.Fingerprint == "" || e.Branch != "" { + return + } + c.mu.Lock() + if c.ends == nil { + c.ends, c.endFPs = map[string]builds.BuildEnded{}, map[string]map[string]bool{} + } + if c.endFPs[e.Module] == nil { + c.endFPs[e.Module] = map[string]bool{} + } + c.endFPs[e.Module][e.Fingerprint] = true + if had, ok := c.ends[e.Module]; ok && had.Fingerprint == e.Fingerprint && had.Outcome == e.Outcome && had.Error == e.Error { + c.mu.Unlock() + return + } + c.ends[e.Module] = e + ask := c.endsRead + c.mu.Unlock() + if ask { + c.askRehand() + } +} + +// BuildEndsRead says the first read of the builder's stream ended: from now the composer acts on the ends it heard, and a +// pass is asked. +func (c *Composer) BuildEndsRead() { + c.mu.Lock() + c.endsRead = true + c.mu.Unlock() + c.askRehand() +} + +// BuildEndsLost says the builder's stream is not followed any more: what was heard is not acted on until a new read ends. +func (c *Composer) BuildEndsLost() { + c.mu.Lock() + c.endsRead = false + c.mu.Unlock() +} + func (c *Composer) askRehand() { select { case c.rehand <- struct{}{}: diff --git a/module.json b/module.json index 9e262cfe..d90fe2ba 100644 --- a/module.json +++ b/module.json @@ -56,6 +56,7 @@ "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-conditions.findings-asked", + "mesh-delivery-builder.build.ended.**", "mesh-delivery-view.settled", "mesh-recovery.ended", "mesh-recovery.restored", diff --git a/packaging/operator_keys_test.go b/packaging/operator_keys_test.go index 751d5436..72fd6773 100644 --- a/packaging/operator_keys_test.go +++ b/packaging/operator_keys_test.go @@ -88,8 +88,8 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) { if m.Config["recovery-wait-minutes"].Default != float64(90) { t.Errorf("config recovery-wait-minutes is %+v, want default 90", m.Config["recovery-wait-minutes"]) } - // The backup event, the status view's settled, and the two events of the seat that serves the seat list (ADR 0636 §4), the keeper's ask (design 47 G §387) and the seat's running (§199). - if want := []string{"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-conditions.findings-asked", "mesh-delivery-view.settled", "mesh-recovery.ended", "mesh-recovery.restored", "node-engine.running", "restic.backed-up"}; !slices.Equal(m.Consumes, want) { + // The backup event, the status view's settled, and the two events of the seat that serves the seat list (ADR 0636 §4), the keeper's ask (design 47 G §387), the seat's running (§199) and the builder's build ends (A-293). + if want := []string{"mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-conditions.findings-asked", "mesh-delivery-builder.build.ended.**", "mesh-delivery-view.settled", "mesh-recovery.ended", "mesh-recovery.restored", "node-engine.running", "restic.backed-up"}; !slices.Equal(m.Consumes, want) { t.Fatalf("consumes %v, want %v", m.Consumes, want) } // 774: the acts bucket mesh-host_acts is not declared as state for now: a declared state of a module with no account -- 2.54.0 From 3167f0104f794603446eb60299f91cc036cd17b2 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 22:16:27 +0100 Subject: [PATCH 49/59] H1 (review r2): the foundation host is the entry that claims the seat node-engine, not a module named mesh-host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §527, §634 --- internal/compose/resolution/seat_dependencies.go | 7 +++++-- .../compose/resolution/seat_dependencies_test.go | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/internal/compose/resolution/seat_dependencies.go b/internal/compose/resolution/seat_dependencies.go index a6556734..42fca172 100644 --- a/internal/compose/resolution/seat_dependencies.go +++ b/internal/compose/resolution/seat_dependencies.go @@ -29,6 +29,10 @@ var appliedThrough = map[string]string{ // A variable rather than a constant only so a test can hold both behaviours; nothing else sets it. var enforceSeatDependencies = true +// nodeEngineSeat is the seat the node-engine's own module claims (design 47 G §508, §527, §634): the +// foundation's host is the entry that claims it, never a module named here. +const nodeEngineSeat = "node-engine" + // foundationModules are the pieces genesis lays before any module exists (novox/hq ADR 0207 §5): // the host and the private network. Registered as modules so they can be assigned, but what they // declare is the installation's, not a module's, so it is never judged. The third piece, the @@ -38,7 +42,6 @@ var enforceSeatDependencies = true // The private network's module is overlay.Name, written out because the overlay package composes // on top of this one; its resources are computed, which exempts it by the rule below as well. var foundationModules = map[string]bool{ - "mesh-host": true, "mesh-wireguard": true, } @@ -46,7 +49,7 @@ var foundationModules = map[string]bool{ // module whose resources are computed is the mesh's by construction — the private network's peer // list and its tools are the controller's, written per node — so it counts whatever its name. func isFoundation(m mcat.Manifest) bool { - return foundationModules[m.Module] || m.Computed != "" + return foundationModules[m.Module] || m.ClaimsSeat(nodeEngineSeat) || m.Computed != "" } // DependsOn is every seat a module needs held on its node: derived from the resource types it diff --git a/internal/compose/resolution/seat_dependencies_test.go b/internal/compose/resolution/seat_dependencies_test.go index c514f16a..806fdf7e 100644 --- a/internal/compose/resolution/seat_dependencies_test.go +++ b/internal/compose/resolution/seat_dependencies_test.go @@ -95,3 +95,17 @@ func TestWithTheSwitchFlippedAnUnmetDependencyRefusesTheNode(t *testing.T) { t.Errorf("the first pass refused an unmet dependency: %v", err) } } + +func TestTheFoundationHostIsTheEntryThatClaimsTheNodeEngineSeat(t *testing.T) { + svc := res("service", "x") + // Whatever its name, the entry claiming the seat is the foundation's: nothing to depend on. + engine := withResources(mod("the-engine", nil, nil, nil, mcat.Claim{Name: "node-engine"}), svc) + if got := DependsOn(nil, engine); len(got) != 0 { + t.Errorf("the entry claiming node-engine depends on %v, want nothing (§527)", got) + } + // A module named as the old code named the engine, claiming no such seat, is an ordinary module. + named := withResources(mod("mesh-host", nil, nil, nil), svc) + if got := DependsOn(nil, named); !reflect.DeepEqual(got, []string{mcat.ServiceManagerSeat}) { + t.Errorf("a module merely named mesh-host depends on %v, want the service manager seat", got) + } +} -- 2.54.0 From 302dcc098f9a8f12e9255408ae693299d286e82e Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 22:16:40 +0100 Subject: [PATCH 50/59] H4 (review r2, A-307): the node-engine's manifest names no uses; its invoke of seat:mesh-grants.withheld is the edge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §635 --- module.json | 3 --- packaging/operator_keys_test.go | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/module.json b/module.json index d90fe2ba..a2d7d50d 100644 --- a/module.json +++ b/module.json @@ -39,9 +39,6 @@ ] } ], - "uses": [ - "mesh-grants" - ], "invokes": [ "seat:mesh-catalog.entry", {"verb": "seat:mesh-catalog.revision", "fields": ["none-yet"]}, diff --git a/packaging/operator_keys_test.go b/packaging/operator_keys_test.go index 72fd6773..6d07d6f6 100644 --- a/packaging/operator_keys_test.go +++ b/packaging/operator_keys_test.go @@ -63,6 +63,7 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) { } var m struct { Consumes []string `json:"consumes"` + Uses []string `json:"uses"` Invokes []any `json:"invokes"` // a verb, or the long form {verb, fields} (§558) Config map[string]struct { Default any `json:"default"` @@ -73,6 +74,9 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) { } // Design 47 G §630, §633, A-77: the recovery module's three verbs it asks, and its bound as a config key of 90. // §635, §558, §617: it invokes `withheld` and names no `uses`; the catalog's revision in the long form with the field none-yet. + if len(m.Uses) != 0 { + t.Errorf("uses is %v, want none: the invoke of seat:mesh-grants.withheld is the edge (§635, A-307)", m.Uses) + } for _, v := range []any{"seat:mesh-recovery.kept-points", "seat:mesh-recovery.restore-kept", "seat:mesh-recovery.snapshot-kept", "seat:mesh-grants.withheld", map[string]any{"verb": "seat:mesh-catalog.revision", "fields": []any{"none-yet"}}} { if !slices.ContainsFunc(m.Invokes, func(have any) bool { return reflect.DeepEqual(have, v) }) { -- 2.54.0 From 1d409cd852c8cb5108862f610d7fe497ff1fcaf3 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 22:24:38 +0100 Subject: [PATCH 51/59] H3 (review r2): the entries test waits on the refusal the composer says until the ghost's reason is entry-missing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §421, §614 (test only) --- internal/selfcompose/entries_test.go | 30 ++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/internal/selfcompose/entries_test.go b/internal/selfcompose/entries_test.go index ec56a5ce..15b6dd4e 100644 --- a/internal/selfcompose/entries_test.go +++ b/internal/selfcompose/entries_test.go @@ -7,6 +7,7 @@ import ( "strings" "sync" "testing" + "time" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" @@ -57,7 +58,16 @@ func composeFromEntries(t *testing.T) { var held map[string]string env := b.env() env.Entry, env.ComposedState = EntryCall(ask), state - env.Refused = func(_ string, h map[string]string) { mu.Lock(); held = h; mu.Unlock() } + refused := make(chan struct{}, 1) // fed at each refusal the composer says, so a wait follows what it says, not a pause + env.Refused = func(_ string, h map[string]string) { + mu.Lock() + held = h + mu.Unlock() + select { + case refused <- struct{}{}: + default: + } + } told := map[string]bool{} env.EntryNotFound = func(fp string, notFound bool) { mu.Lock(); told[fp] = notFound; mu.Unlock() } c := New(env) @@ -93,9 +103,21 @@ func composeFromEntries(t *testing.T) { } b.mu.Unlock() // A want whose entry the half holds none of is held, saying entry-missing and the fingerprint; the rest is composed. - mu.Lock() - why := held["ghost"] - mu.Unlock() + // The ghost's entry is asked first and answered not found after: wait until the refusal says entry-missing. + var why string + for deadline := time.After(10 * time.Second); !strings.HasPrefix(why, agent.EntryMissing); { + mu.Lock() + why = held["ghost"] + mu.Unlock() + if strings.HasPrefix(why, agent.EntryMissing) { + break + } + select { + case <-refused: + case <-deadline: + t.Fatalf("the ghost's reason is still %q after 10 s, want %s", why, agent.EntryMissing) + } + } if !strings.HasPrefix(why, agent.EntryMissing) || !strings.Contains(why, ghost) || held["lemurs"] != "" { t.Errorf("held %q, want ghost held with %s naming %s", held, agent.EntryMissing, ghost) } -- 2.54.0 From e558510a54ac28b0d4476bb4ac9d96132ce643d3 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 22:28:15 +0100 Subject: [PATCH 52/59] H5 (review r2, A-308, A-312): the shadow watches no indexer modules; it reads the entries its wants name and asks the verb module for a module a candidate adds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §421, §628 --- internal/selfcompose/candidate.go | 1 + internal/selfcompose/candidate_test.go | 16 +++- internal/selfcompose/entries.go | 76 +++++++++++++++++- internal/selfcompose/selfcompose.go | 9 ++- internal/selfcompose/selfcompose_test.go | 53 +++++++++---- internal/selfcompose/shadow.go | 99 +++++++++++++++++++++--- module.json | 1 + packaging/operator_keys_test.go | 2 +- 8 files changed, 226 insertions(+), 31 deletions(-) diff --git a/internal/selfcompose/candidate.go b/internal/selfcompose/candidate.go index 502756d9..53161098 100644 --- a/internal/selfcompose/candidate.go +++ b/internal/selfcompose/candidate.go @@ -328,6 +328,7 @@ func (c *Composer) scratch() *Composer { out.certified = c.certified out.assignmentKeys = c.assignmentKeys out.origin = c + out.forCandidates = c.forCandidates return out } diff --git a/internal/selfcompose/candidate_test.go b/internal/selfcompose/candidate_test.go index e78885aa..dcebd22f 100644 --- a/internal/selfcompose/candidate_test.go +++ b/internal/selfcompose/candidate_test.go @@ -7,6 +7,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/buckets/bucketstest" "strings" @@ -78,6 +79,15 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) { env.ComposedState = state var applied atomic.Int64 env.Refused = func(string, map[string]string) { applied.Add(1) } + // What the node's want names is read from the catalog half's entry (design 47 G §421, A-308): the shadow watches no modules. + var manifest map[string]any + if err := json.Unmarshal([]byte(lemurs), &manifest); err != nil { + t.Fatal(err) + } + entry := meshcatalog.CatalogEntry{Module: "lemurs", Fp: fp, Manifest: manifest, Repository: "novox/x", Commit: "c", ManifestDigest: digest} + env.Entry = EntryCall(func(_ context.Context, _, _ string, _ any) (json.RawMessage, error) { + return json.Marshal(meshcatalog.EntryAnswer{Entry: entry}) + }) var handed atomic.Int64 s := NewShadow(env, func(context.Context, map[string]nodeengine.CandidateAnswer) { handed.Add(1) }) ctx, cancel := context.WithCancel(context.Background()) @@ -89,7 +99,7 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) { place := assignedValues(t, meshmoduleassigner.Assignments{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"}, Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}}}) have, _ := nec.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}}) - mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"}) + want, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fp, Self: true}) deadline := time.Now().Add(5 * time.Second) watching := func() int { b.mu.Lock(); defer b.mu.Unlock(); return len(b.watches) } for watching() < 6 && time.Now().Before(deadline) { @@ -97,8 +107,8 @@ func TestTheNodeComposesACandidateOnAsk(t *testing.T) { } b.putAssigned(t, place) b.put(t, buckets.Have, "n1.node", have) - b.put(t, buckets.Modules, "lemurs", mod) - for _, bucket := range []string{buckets.ProposalCandidates, buckets.Assignments, buckets.Have, buckets.Modules, ResolvedBucket} { + b.put(t, buckets.Wants, "want.n1.lemurs", want) + for _, bucket := range []string{buckets.ProposalCandidates, buckets.Assignments, buckets.Have, buckets.Wants, ResolvedBucket} { b.send(bucket, agent.Update{Caught: true}) } for !s.ready.Load() && time.Now().Before(deadline) { diff --git a/internal/selfcompose/entries.go b/internal/selfcompose/entries.go index 979175de..8c427341 100644 --- a/internal/selfcompose/entries.go +++ b/internal/selfcompose/entries.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "git.novox.be/novox/mesh-sdk/go/buscalls" "reflect" "sort" "strings" @@ -257,7 +258,10 @@ func (c *Composer) syncEntries(ctx context.Context) { c.entryOf(fp) } for _, row := range c.entries.Rows() { - if _, named := need[row.Fingerprint]; !named { + c.mu.Lock() + added := c.addedFps[row.Fingerprint] + c.mu.Unlock() + if _, named := need[row.Fingerprint]; !named && !added { c.entries.Release(row.Fingerprint) c.tellNotFound(row.Fingerprint, false) } @@ -265,7 +269,7 @@ func (c *Composer) syncEntries(ctx context.Context) { c.mu.Lock() var drop []string for fp := range c.keptEnt { - if _, named := need[fp]; !named { + if _, named := need[fp]; !named && !c.addedFps[fp] { delete(c.keptEnt, fp) drop = append(drop, entryKey(fp)) } @@ -568,3 +572,71 @@ func (c *Composer) FormChanges(_ context.Context, module, runningFp, appliedFp s } return catalogue.FormChanges(running, applied), nil } + +// syncShadowEntries asks the entry of each build a want of this node names, drops the one nothing names, lays c.modules and +// c.manifests from the entries held, and says whether an entry asked is not yet answered: the shadow's picture of the catalog, +// where it once watched the indexer's modules (design 47 G §421, A-308). +func (c *Composer) syncShadowEntries(ctx context.Context) (pending bool) { + if !c.byEntries() { + return false + } + c.syncEntries(ctx) + c.mu.Lock() + defer c.mu.Unlock() + for module, w := range c.wants { + if module == agent.NodeModule || !strings.HasPrefix(w.Fp, meshcatalogclient.SourcePrefix2) { + continue + } + if _, held := c.keptEnt[w.Fp]; !held { + for _, r := range c.entries.Rows() { + if r.Fingerprint == w.Fp && r.State == meshcatalogclient.EntryAsked { + return true + } + } + } + } + return false +} + +// moduleVerb is the seat mesh-catalog's verb `module`: one module's trunk fingerprint and manifest by name. +const moduleVerb = "seat:" + meshcatalog.Seat + "." + meshcatalog.VerbModule + +// addedModule is the trunk's build of a module an assignment candidate adds to this node when no want here names it yet: the +// fingerprint the seat's verb `module` answers, then its entry by the verb `entry` as for a wanted module (design 47 G §421, A-312). +// A call, not a watch: nothing of the catalog is kept. It answers the entry when held; otherwise why not: a module with no +// fingerprint says the verb's reason, an entry asked says so (its answer wakes the shadow, which composes again). +func (c *Composer) addedModule(ctx context.Context, module string) (fp string, entry meshcatalog.CatalogEntry, held, pending bool, why string) { + if c.env.Entry == nil { + return "", entry, false, false, "this node reads no module of the catalog half" + } + raw, err := c.env.Entry(moduleVerb, meshcatalog.ModuleInput{Module: module}) + if err != nil { + return "", entry, false, false, fmt.Sprintf("the catalog half's answer for %s: %v", module, err) + } + answer, said, ok := buscalls.Unwrap(raw) + if !ok { + return "", entry, false, false, fmt.Sprintf("the catalog half holds no module %s: %s", module, said) + } + var a meshcatalog.ModuleAnswer + if err := json.Unmarshal(answer, &a); err != nil { + return "", entry, false, false, fmt.Sprintf("the catalog half's answer for %s cannot be read: %v", module, err) + } + if a.Module.Fp == "" { + return "", entry, false, false, fmt.Sprintf("the catalog half gives %s no fingerprint: %s", module, a.Module.None) + } + fp = a.Module.Fp + c.mu.Lock() + if c.addedFps == nil { + c.addedFps = map[string]bool{} + } + c.addedFps[fp] = true + c.mu.Unlock() + e, state := c.entryOf(fp) + switch state { + case meshcatalogclient.EntryAnswered: + return fp, e, true, false, "" + case meshcatalogclient.EntryNotFound: + return fp, e, false, false, fmt.Sprintf("the catalog half holds no entry of %s's build %s", module, fp) + } + return fp, e, false, true, fmt.Sprintf("the entry of %s's build %s is asked of the catalog half and not yet answered", module, fp) +} diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index ea0d7af1..07999e2c 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -198,6 +198,11 @@ type Composer struct { keptWantVal map[string]keptWant keptWants map[string]bool keptWrite sync.Mutex + // forCandidates marks the shadow's composers: they read the catalog half's entries for the modules their wants name (§421) and, for + // a candidate's own builds, the manifests and verdicts of the indexer that compose-candidate alone keeps (§628, A-308). + forCandidates bool + // addedFps are the builds of modules a candidate adds that the shadow asked the entry of (addedModule), kept beside the wants' under mu. + addedFps map[string]bool // modulesRev is the revision of the last update of the indexer's modules taken: where the picture of them stands. modulesRev uint64 // heard is, by node and module, what each node's event running last said it runs, kept from the seat's stream by this @@ -1631,7 +1636,7 @@ func (r reader) Manifest(ctx context.Context, digest string) ([]byte, error) { if ok { return raw, nil } - if r.c.byEntries() { + if r.c.byEntries() && !r.c.forCandidates { return nil, fmt.Errorf("the catalog half's entries hold no manifest %s", digest) } key, err := meshcatalogclient.ManifestsKey(digest) @@ -1655,7 +1660,7 @@ func (r reader) Manifest(ctx context.Context, digest string) ([]byte, error) { } func (r reader) Build(ctx context.Context, module, fp string) (*builds.BuilderBuild, error) { - if r.c.byEntries() { + if r.c.byEntries() && !r.c.forCandidates { return r.c.buildOf(module, fp), nil } key, err := builds.BuilderBuildKey(module, fp) diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index cf9eff83..4face352 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -1127,6 +1127,34 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { "keeper": {Fp2: fp, Manifest: kdigest, Repository: "novox/k", Commit: "c"}} readOnly := &fakeBus{kv: map[string][]byte{buckets.Manifests + "/" + key: man, buckets.Manifests + "/" + kkey: kman}} senv := readOnly.env() + // The catalog half: lemurs's entry, which its want names, and keeper's, which a candidate adding it asks by the verb module + // (design 47 G §421, A-308, A-312). The shadow watches no modules. + kfp := meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64) + entryOfManifest := func(module, fp, body string) meshcatalog.CatalogEntry { + var m map[string]any + if err := json.Unmarshal([]byte(body), &m); err != nil { + t.Fatal(err) + } + return meshcatalog.CatalogEntry{Module: module, Fp: fp, Manifest: m, Repository: "novox/x", Commit: "c", ManifestDigest: meshcatalogclient.ManifestDigest([]byte(body))} + } + catalogEntries := map[string]meshcatalog.CatalogEntry{fp: entryOfManifest("lemurs", fp, lemurs), kfp: entryOfManifest("keeper", kfp, keeper)} + senv.Entry = EntryCall(func(_ context.Context, _, verb string, args any) (json.RawMessage, error) { + switch in := args.(type) { + case meshcatalog.ModuleInput: + for _, e := range catalogEntries { + if e.Module == in.Module { + return json.Marshal(meshcatalog.ModuleAnswer{Module: meshcatalog.ModuleAnswerModule{Module: e.Module, Repository: e.Repository, Commit: e.Commit, Fp: e.Fp}}) + } + } + return nil, fmt.Errorf("the catalog holds no module %s", in.Module) + case meshcatalog.EntryInput: + if e, ok := catalogEntries[in.Fingerprint]; ok { + return json.Marshal(meshcatalog.EntryAnswer{Entry: e}) + } + return nil, errors.New("no entry of fingerprint " + in.Fingerprint + " is kept") + } + return nil, fmt.Errorf("asked %s of %T", verb, args) + }) senv.Watch = sb.watch var ownFails atomic.Bool // the vault's own on a bus that does not answer (2026-10-28, ace's answer to seed-7) senv.Own = func(context.Context, map[string]any) (json.RawMessage, error) { @@ -1177,10 +1205,6 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { tp := assignedValues(t, trunk) sb.putAssigned(tp, 1) sb.put(buckets.Have, "n1.node", rec, 1) - for name, m := range mods { - raw, _ := meshcatalogclient.EncodeModule(m) - sb.put(buckets.Modules, name, raw, 1) - } wantLemurs := meshwants.WantsByWantNodeModuleKey("n1", "lemurs") var lemursDigest string for _, c := range running { @@ -1244,12 +1268,18 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { } } waitFor("every group answered", func(a map[string]nodeengine.CandidateAnswer) bool { return len(a) == 6 }) + // keeper, which the candidate adds and no want names, is asked of the catalog half (its entry by the verb entry after the verb + // module): the answer waits for it, and is the same as before the shadow watched no modules once it comes (A-312). + waitFor("the added module's entry answered", func(a map[string]nodeengine.CandidateAnswer) bool { return len(a["adds"].Added) == 1 }) // Every watch and get the shadow makes is one the node's grant allows (issue 1126, 2026-10-27 09:21). sb.mu.Lock() for _, w := range sb.watches { if !grantAllows(t, "n1", w.bucket, w.pattern) { t.Errorf("the shadow watches %s of %s, which the node's grant does not allow", w.pattern, w.bucket) } + if w.bucket == buckets.Modules { + t.Errorf("the shadow watches the indexer's modules (%s), which design 47 G A-308 ends", w.pattern) + } } sb.mu.Unlock() readOnly.mu.Lock() @@ -1334,8 +1364,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { lemursHave("its tool lemurs_health: no answer within 10s", 11) lemursHave("its tool lemurs_health: no answer within 20s", 12) sb.putAssigned(tp, 2) - lraw, _ := meshcatalogclient.EncodeModule(mods["lemurs"]) - sb.put(buckets.Modules, "lemurs", lraw, 2) + sb.put(buckets.Wants, wantLemurs, wantRaw, 2) drain() if got := answering(); got != settled { t.Errorf("answered %d more time(s) with no candidate revision and no input of the node changed", got-settled) @@ -1393,7 +1422,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { // Review of L2b, round 2: a watch ends while the link stays; a group's key is deleted before the shadow watches // again. The new run reads afresh, so that group is no longer answered. sb.missing(buckets.ProposalCandidates, true) - sb.end(buckets.Modules) + sb.end(buckets.Wants) dropsKey := meshmoduleassigner.CandidatesByProposalNodeKey("drops", "n1") sb.del(buckets.ProposalCandidates, dropsKey) time.Sleep(30 * time.Millisecond) @@ -1428,6 +1457,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { nenv := readOnly.env() nenv.Watch = nb.watch nenv.Running = senv.Running + nenv.Entry = senv.Entry var nsaid []string var nmu sync.Mutex nenv.Say = func(l string) { nmu.Lock(); nsaid = append(nsaid, l); nmu.Unlock() } @@ -1439,10 +1469,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { nb.made(6) nb.putAssigned(tp, 1) nb.put(buckets.Have, "n1.node", rec, 1) - for name, m := range mods { - raw, _ := meshcatalogclient.EncodeModule(m) - nb.put(buckets.Modules, name, raw, 1) - } + nb.put(buckets.Wants, wantLemurs, wantRaw, 1) sameRaw, _ := bucketstest.Candidate(meshmoduleassigner.CandidatesByProposalNodeValue{Proposal: "same", Assignments: &trunk}) sameKey := meshmoduleassigner.CandidatesByProposalNodeKey("same", "n1") nb.put(buckets.ProposalCandidates, sameKey, sameRaw, 3) @@ -1450,7 +1477,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { oldRaw, _ := bucketstest.Candidate(meshmoduleassigner.CandidatesByProposalNodeValue{Proposal: "older", Assignments: &trunk}) oldKey := meshmoduleassigner.CandidatesByProposalNodeKey("older", "n1") nb.put(buckets.ProposalCandidates, oldKey, oldRaw, 2) - for _, bucket := range []string{buckets.ProposalCandidates, buckets.Assignments, buckets.Have, buckets.Modules, builds.BuilderBuildsBucket} { + for _, bucket := range []string{buckets.ProposalCandidates, buckets.Assignments, buckets.Have, buckets.Wants, builds.BuilderBuildsBucket} { nb.send(bucket, agent.Update{Caught: true}) } select { @@ -1825,7 +1852,7 @@ func TestAHaveSaysWhetherItsModuleRunsInsideTheNodesOwn(t *testing.T) { } s := &Shadow{c: c, ran: map[string]string{}} s.takeRan(agent.Update{Key: key, Value: value}) - if _, got := s.heldToday()["web"]; got != row.heldToday { + if _, got := s.heldToday(nil)["web"]; got != row.heldToday { t.Errorf("web held today: %v, want %v", got, row.heldToday) } }) diff --git a/internal/selfcompose/shadow.go b/internal/selfcompose/shadow.go index eaf7ab1e..e272cf8c 100644 --- a/internal/selfcompose/shadow.go +++ b/internal/selfcompose/shadow.go @@ -8,12 +8,14 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/compose/frombus" engcomp "github.com/novox/mesh-host/internal/compositions" "github.com/novox/mesh-host/internal/judge" + "maps" "sort" "strings" "sync" @@ -61,6 +63,8 @@ type Shadow struct { // after a bound (laterAgain), whether or not an input moves (issue 1126, 2026-10-28: ace kept "the vault's own: not // yet ... no responders" in its answer to seed-7). owed bool + // entryMoved wakes the run when an entry it asked of the catalog half is answered or found missing (§421). + entryMoved chan struct{} // ready says this run read every input once: the verb compose-candidate composes only then (ComposeCandidate). ready atomic.Bool } @@ -72,7 +76,7 @@ type answeredAt struct { } // inputWords name the shadow's inputs, in Run's order, as an answer says which of them woke it. -var inputWords = [...]string{"the candidate", "assignment", "records", "modules", "values", "haves"} +var inputWords = [...]string{"the candidate", "assignment", "records", "wants", "values", "haves"} // wokenBy says what moved between the answer a proposal stands at (had, when done) and the one about to be composed (at): // its first answer, a retry, its candidate's revisions, or the inputs that changed, by their words. @@ -106,8 +110,28 @@ type candidateKey struct { // NewShadow is the shadow composer of one node; answer writes the node's answers into its own have key. func NewShadow(env Env, answer func(ctx context.Context, answers map[string]nodeengine.CandidateAnswer)) *Shadow { - return &Shadow{c: New(env), answer: answer, candidates: map[string]map[string]candidateKey{}, - answers: map[string]nodeengine.CandidateAnswer{}, ran: map[string]string{}} + s := &Shadow{answer: answer, candidates: map[string]map[string]candidateKey{}, + answers: map[string]nodeengine.CandidateAnswer{}, ran: map[string]string{}, entryMoved: make(chan struct{}, 1)} + s.c = s.composer(env) + return s +} + +// composer is a composer for the shadow: it asks the catalog half's entry of each build the node's wants name, and wakes the +// run when one answers (design 47 G §421, A-308: the node-engine no longer watches the indexer's modules). +func (s *Shadow) composer(env Env) *Composer { + env.ComposedState = nil // the node's own composer keeps the composition state; the shadow writes nothing + c := New(env) + c.forCandidates = true + if c.entries != nil { + c.entries.Changed = func(row meshcatalogclient.EntryRow) { + c.entryChanged(row) + select { + case s.entryMoved <- struct{}{}: + default: + } + } + } + return c } // Keep runs the shadow until ctx ends, beginning again after backoff(try) whenever it stops: an input it cannot watch @@ -153,7 +177,7 @@ func (s *Shadow) Run(ctx context.Context) error { s.answered = map[string]answeredAt{} s.moved = [len(inputWords)]uint64{} s.retry = map[string]bool{} - s.c = New(s.c.env) + s.c = s.composer(s.c.env) s.mu.Unlock() s.ready.Store(false) defer s.ready.Store(false) @@ -164,7 +188,7 @@ func (s *Shadow) Run(ctx context.Context) error { {buckets.ProposalCandidates, ">", nil, nil}, {buckets.Assignments, ">", (*Composer).takeAssignments, nil}, {buckets.Have, "*." + nodeengineclient.NodeWord, nil, nil}, - {buckets.Modules, ">", (*Composer).takeModule, nil}, + {buckets.Wants, "want." + node + ".>", (*Composer).takeWant, nil}, {ResolvedBucket, node + ".>", (*Composer).takeValue, nil}, // What this node runs moved: an apply ends in its module's have. {buckets.Have, node + ".>", nil, nil}, @@ -209,6 +233,9 @@ func (s *Shadow) Run(ctx context.Context) error { case <-again: again = nil dirty = true + case <-s.entryMoved: + s.moved[3]++ + dirty = true case <-waiting.C: if !ready { s.notReady(ctx, ins, caught) @@ -245,10 +272,15 @@ func (s *Shadow) Run(ctx context.Context) error { continue } } + if !ready && !all(caught) { + continue + } + // What the wants name is read from the catalog half's entries; an answer waits until each entry asked has come (§421). + if (dirty || !ready) && s.c.syncShadowEntries(ctx) { + continue + } if !ready { - if ready = all(caught); !ready { - continue - } + ready = true s.ready.Store(true) dirty = true } @@ -386,6 +418,12 @@ func (s *Shadow) answerAll(ctx context.Context) { proposals = append(proposals, l) } // A proposal whose keys' revisions and the node's inputs are those its answer stands at is not answered again. + if len(s.candidates) == 0 { + // No proposal is left that adds a module: the entries asked for those are no longer kept (A-312). + c.mu.Lock() + c.addedFps = nil + c.mu.Unlock() + } for g := range s.answered { if _, ok := s.candidates[g]; !ok { delete(s.answered, g) @@ -554,6 +592,46 @@ func (s *Shadow) answerProposal(ctx context.Context, proposal string) (nodeengin inOwn := c.inOwnOf(*assignment) c.mu.Unlock() fps := shadowBuilds(running, *assignment, modules, inOwn) + // A module the candidate adds that no want here names: the trunk's build is asked of the catalog half's verb module, and its + // entry by the verb entry (design 47 G §421, A-312). One with no build to take is held, with the reason. + heldAdded := map[string]string{} + folded := map[string]bool{} + for _, m := range inOwn { + folded[m] = true + } + for _, pm := range assignment.Modules { + if _, has := fps[pm.Module]; has || folded[pm.Module] || pm.Module == "" { + continue + } + fp, entry, held, pending, why := c.addedModule(ctx, pm.Module) + if pending { + // Its answer wakes the shadow (entryMoved), which composes this proposal again: until then this is no answer to keep, as one + // that met a read the bus did not answer is none. + c.missed.Add(1) + return nodeengine.CandidateAnswer{Proposal: proposal, Revision: int64(rev), Verdict: nodeengine.CandidateAnswerVerdictRefused, + Refused: why + ", so this answer waits for it"}, true + } + if !held { + heldAdded[pm.Module] = why + continue + } + digest, raw, err := entryManifest(entry) + if err != nil { + heldAdded[pm.Module] = fmt.Sprintf("the entry of %s's build %s holds no manifest that can be read: %v", pm.Module, fp, err) + continue + } + m := meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: entry.Repository, Commit: entry.Commit, Path: entry.Path, + Needs: entry.Needs, Packages: entry.Packages, Verdict: entry.Verdict} + if entry.Inputs != nil { + m.Problems, m.Warnings, m.Unknown = entry.Inputs.Problems, entry.Inputs.Warnings, entry.Inputs.Unknown + } + c.mu.Lock() + c.modules[pm.Module] = m + c.manifests[digest] = raw + c.mu.Unlock() + modules[pm.Module] = m + fps[pm.Module] = fp + } kept := map[string]string{} for module, fp := range fps { if mod, ok := modules[module]; ok && mod.Fp2 == fp { @@ -571,7 +649,7 @@ func (s *Shadow) answerProposal(ctx context.Context, proposal string) (nodeengin c.wants = held c.mu.Unlock() a := frombus.Answer(ctx, reader{c: c}, c.env.Mesh, frombus.Scope{Node: node, KeptAt: kept, Assignments: given, InOwn: inOwn, - SameValue: c.sameValue, HeldToday: s.heldToday()}, fps, running, proposal, rev) + SameValue: c.sameValue, HeldToday: s.heldToday(heldAdded)}, fps, running, proposal, rev) s.mu.Lock() ran := make(map[string]string, len(s.ran)) for m, d := range s.ran { @@ -583,8 +661,9 @@ func (s *Shadow) answerProposal(ctx context.Context, proposal string) (nodeengin // heldToday is what the node holds today: each module whose have's phase is held, and what its own composer holds, the // node's own composition ("node") among them when it composes none of its own. -func (s *Shadow) heldToday() map[string]string { +func (s *Shadow) heldToday(added map[string]string) map[string]string { out := map[string]string{} + maps.Copy(out, added) s.mu.Lock() for m, how := range s.heldNow { if how != "" { diff --git a/module.json b/module.json index a2d7d50d..e749c563 100644 --- a/module.json +++ b/module.json @@ -41,6 +41,7 @@ ], "invokes": [ "seat:mesh-catalog.entry", + "seat:mesh-catalog.module", {"verb": "seat:mesh-catalog.revision", "fields": ["none-yet"]}, "seat:mesh-catalog.seat-definitions", "seat:mesh-grants.withheld", diff --git a/packaging/operator_keys_test.go b/packaging/operator_keys_test.go index 6d07d6f6..12524586 100644 --- a/packaging/operator_keys_test.go +++ b/packaging/operator_keys_test.go @@ -78,7 +78,7 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) { t.Errorf("uses is %v, want none: the invoke of seat:mesh-grants.withheld is the edge (§635, A-307)", m.Uses) } for _, v := range []any{"seat:mesh-recovery.kept-points", "seat:mesh-recovery.restore-kept", "seat:mesh-recovery.snapshot-kept", - "seat:mesh-grants.withheld", map[string]any{"verb": "seat:mesh-catalog.revision", "fields": []any{"none-yet"}}} { + "seat:mesh-grants.withheld", "seat:mesh-catalog.module", map[string]any{"verb": "seat:mesh-catalog.revision", "fields": []any{"none-yet"}}} { if !slices.ContainsFunc(m.Invokes, func(have any) bool { return reflect.DeepEqual(have, v) }) { t.Errorf("invokes %v lacks %v", m.Invokes, v) } -- 2.54.0 From 23799a30ed4df1960057738adc7fc913a0cc3431 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 23:05:06 +0100 Subject: [PATCH 53/59] R3-1, R3-2, R3-3: the node-engine reads no build of the builder; the shadow builds from the entry it holds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The composer's Build is made from the catalog half's entry it holds and the parts of the want that names it (buildOf); the inputs of the indexer's modules and the builder's builds, takeBuildDone, takeModule and the builder read are gone, and mesh-host names no BuilderBuildsBucket. The shadow takes the entries of the builds it composes from (askBuildsEntries), builds a module a candidate adds from its entry and the candidate's `added` parts (A-316), and holds, with the reason, a module whose entry has no part given, whose fingerprint the verb module does not give, or for which it errors (A-312). Its composer is given the entry call as the node's own is. Rows: a built module added with and without parts, the verb module answering no fingerprint and erroring. Builds: design 47 G §421, §628, A-308, A-312, A-316 --- cmd/mesh-host/nodeagent.go | 6 +- cmd/mesh-host/nodeagent_integration_test.go | 15 +- internal/compose/frombus/frombus.go | 2 +- internal/selfcompose/candidate.go | 4 +- .../selfcompose/compose_while_passing_test.go | 3 +- internal/selfcompose/composedstate_test.go | 90 +++----- internal/selfcompose/data_test.go | 38 ---- internal/selfcompose/entries.go | 42 +++- internal/selfcompose/entries_test.go | 69 +++++- internal/selfcompose/selfcompose.go | 128 ++---------- internal/selfcompose/selfcompose_test.go | 197 +++++++----------- internal/selfcompose/shadow.go | 44 +++- 12 files changed, 279 insertions(+), 359 deletions(-) diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 75310ea0..8a256529 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -1155,9 +1155,6 @@ func selfComposer(session *link.Session, node string, hooks composerHooks, say f env.ComposedState = link.EngineComposed{Session: session, Node: node} // And each composition it writes there said as the seat's event composed, which the assigner waits on. env.SayComposed = session.SayComposed - // And the catalog half read by the fingerprints its wants and runnings name, over its own calls of the seat mesh-catalog's - // verb entry on its link session (design 47 G §421): neither the indexer's nor the builder's states are read. - env.Entry = selfcompose.EntryCall(session.AskMeshSeat) return selfcompose.New(env) } @@ -1199,6 +1196,9 @@ func composerEnv(session *link.Session, node string, say func(string), hooks com } else if hooks.seatListFailed { env.SeatList = selfcompose.Unread{} } + // And the shadow too: the catalog half read by the fingerprints its wants and runnings name, over its own calls of the seat mesh-catalog's + // verb entry on its link session (design 47 G §421): neither the indexer's nor the builder's states are read. + env.Entry = selfcompose.EntryCall(session.AskMeshSeat) return env } diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index 05b9eed8..933bbaac 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -12,7 +12,6 @@ import ( "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" - builderbuilds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" @@ -493,7 +492,6 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { } assignment, modules, manifests := kvOf(buckets.Assignments), kvOf(buckets.Modules), kvOf(buckets.Manifests) resolved := kvOf(selfcompose.ResolvedBucket) - builds := kvOf(builderbuilds.BuilderBuildsBucket) candidates := kvOf(buckets.ProposalCandidates) packages := kvOf(link.PackagesBucket) path := filepath.Join(b.dir, "web.conf") @@ -523,10 +521,6 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { rec2, _ := nec.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "it2", Capabilities: []string{"service-manager"}}}) pkey2, _ := meshmoduleassignerclient.NodeKey("it2") hkey2, _ := nec.HaveNodeKey("it2") - buildKey, _ := builderbuilds.BuilderBuildKey("web", fp) - done, _ := builderbuilds.EncodeBuilderBuild(builderbuilds.BuilderBuild{State: builderbuilds.BuildDone, Manifest: digest, - Artifacts: []builderbuilds.BuiltArtifact{{Name: "files", Kind: "archive", Reference: "artifact-store://web/files/blobs/" + fpDigest("f"), Digest: fpDigest("f")}}}) - firstBuild := done for _, w := range []struct { kv nats.KeyValue key string @@ -534,7 +528,7 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { }{{manifests, mkey, mraw}, {modules, "web", modv}, {b.haves, hkey, rec}, {resolved, itNode + ".web.greeting", []byte(`{"value":"bonjour","source":"module"}`)}, {manifests, mlkey, mlraw}, {modules, "mailer", mlmod}, {b.haves, hkey2, rec2}, {assignment, pkey2, places2[pkey2]}, {assignment, "it2.mailer", places2["it2.mailer"]}, - {resolved, "it2.mailer.domain", []byte(`{"value":"example.org","source":"module"}`)}, {builds, buildKey, firstBuild}} { + {resolved, "it2.mailer.domain", []byte(`{"value":"example.org","source":"module"}`)}} { if _, err := w.kv.Put(w.key, w.value); err != nil { t.Fatal(err) } @@ -566,7 +560,6 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { t.Fatal(err) } if checkBuild { - _ = buildKey // The entry of the build its want names is asked of the catalog half through the node's session: one the half holds none // of holds the want (entry-missing, naming the fingerprint) and nothing is applied; once the half holds it and it is // asked again by hand, then a config event, and the node composes. @@ -855,7 +848,6 @@ func TestNatsTheBusServerIsTakenOverAndReplacedOnlyWhenItsBuildChangesIt(t *test } assignment, modules, manifests := kvOf(buckets.Assignments), kvOf(buckets.Modules), kvOf(buckets.Manifests) kvOf(selfcompose.ResolvedBucket) - builds := kvOf(builderbuilds.BuilderBuildsBucket) kvOf(buckets.ProposalCandidates) kvOf(link.PackagesBucket) // The bus module at a build, done: its manifest written and the indexer's module naming it at fp. @@ -872,11 +864,6 @@ func TestNatsTheBusServerIsTakenOverAndReplacedOnlyWhenItsBuildChangesIt(t *test if _, err := manifests.Put(mkey, mraw); err != nil { t.Fatal(err) } - bkey, _ := builderbuilds.BuilderBuildKey("nats", fp) - done, _ := builderbuilds.EncodeBuilderBuild(builderbuilds.BuilderBuild{State: builderbuilds.BuildDone, Manifest: digest}) - if _, err := builds.Put(bkey, done); err != nil { - t.Fatal(err) - } if _, err := modules.Put("nats", modv); err != nil { t.Fatal(err) } diff --git a/internal/compose/frombus/frombus.go b/internal/compose/frombus/frombus.go index 4f072247..5c85ada0 100644 --- a/internal/compose/frombus/frombus.go +++ b/internal/compose/frombus/frombus.go @@ -81,7 +81,7 @@ type Reader interface { // and each it grants as a provider, with the copy sealed to it (the vault's `own` with the keys, ADR 0550 §5). A key // the vault holds no value for is left out; the composition then stands in for it, and Compose refuses it. Credentials(ctx context.Context, node string, asConsumer, asProvider []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) - // Build is a module's build at a fingerprint, the builder's key (builds.BuilderBuildsBucket, BuilderBuildKey); nil + // Build is a module's build at a fingerprint, the builder's key (BuilderBuildKey); nil // when the builder holds none. The composing node composes each of its modules from its build (issue 1126). Build(ctx context.Context, module, fp string) (*builds.BuilderBuild, error) // Archive is the archive at url, a kept reference routed to the artifact store as the composing node reaches it diff --git a/internal/selfcompose/candidate.go b/internal/selfcompose/candidate.go index 53161098..643bed97 100644 --- a/internal/selfcompose/candidate.go +++ b/internal/selfcompose/candidate.go @@ -322,7 +322,9 @@ func (c *Composer) scratch() *Composer { out.values = maps.Clone(c.values) out.manifests = maps.Clone(c.manifests) out.archives = maps.Clone(c.archives) - out.builds = maps.Clone(c.builds) + // The entries it holds and the wants that name them give a build (buildOf), as the builder's builds once did. + out.keptEnt = maps.Clone(c.keptEnt) + out.wants = maps.Clone(c.wants) out.sealedOwn = maps.Clone(c.sealedOwn) out.ownHanded = maps.Clone(c.ownHanded) out.certified = c.certified diff --git a/internal/selfcompose/compose_while_passing_test.go b/internal/selfcompose/compose_while_passing_test.go index fe2681f2..d755ae2d 100644 --- a/internal/selfcompose/compose_while_passing_test.go +++ b/internal/selfcompose/compose_while_passing_test.go @@ -10,7 +10,6 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "github.com/novox/mesh-host/internal/agent" "github.com/novox/mesh-host/internal/buckets" @@ -48,7 +47,7 @@ func composeAnsweredWhileAPassRuns(t *testing.T) { b.putAssigned(t, place) b.put(t, buckets.Have, "n1.node", have) b.put(t, buckets.Wants, "want.n1.node", nodeWant) - for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, buckets.Modules, ResolvedBucket, builds.BuilderBuildsBucket} { + for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, ResolvedBucket} { b.send(bucket, agent.Update{Caught: true}) } go func() { diff --git a/internal/selfcompose/composedstate_test.go b/internal/selfcompose/composedstate_test.go index 68f9721d..5de5c96f 100644 --- a/internal/selfcompose/composedstate_test.go +++ b/internal/selfcompose/composedstate_test.go @@ -63,13 +63,31 @@ func (f *fakeComposed) Delete(_ context.Context, key string) error { func (f *fakeComposed) written() []string { f.mu.Lock() defer f.mu.Unlock() - return slices.Clone(f.puts) + var out []string + for _, key := range f.puts { + // The entries and wants the node keeps are its composition state's too (§683), not compositions. + if !strings.HasPrefix(key, keptEntryPrefix) && !strings.HasPrefix(key, keptWantPrefix) { + out = append(out, key) + } + } + return out +} + +// keep holds in the state, as an earlier run of the node left them, the entry of fp and the want of module at it (§683): the +// composer then composes at its start from what it kept, and asks the catalog half of nothing first. +func (f *fakeComposed) keep(t *testing.T, half catalogHalf, module, fp string) { + t.Helper() + entry, _ := json.Marshal(meshcatalog.EntryAnswer{Entry: half[fp]}) + want, _ := json.Marshal(keptWant{V: 1, Since: time.Now(), Want: meshwants.WantsByWantNodeModuleValue{Fp: fp, Self: true}}) + f.mu.Lock() + defer f.mu.Unlock() + f.kv[entryKey(fp)] = ComposedEntry{Value: entry, Revision: 1} + f.kv[wantStateKey(module)] = ComposedEntry{Value: want, Revision: 1} } // novox/hq ADR 0593 §6: the node publishes each composition it makes in its composition state, a record never applied: -// the wanted module and the node's own at the first pass; nothing again for a pass that changes nothing; a done build -// of a module placed here that no want names, at its trunk's build, written once as its own key (before any want moves); -// the same build landing again writes nothing. And a module's keys are kept while among the five it wrote last or named +// the wanted module and the node's own at the first pass, from the entry and want it kept; nothing again for a pass that +// changes nothing; a failed build of a placed module said refused by its build-ended event. And a module's keys are kept while among the five it wrote last or named // by a want, and deleted otherwise. func TestTheNodePublishesItsCompositions(t *testing.T) { t.Run("the verb compose is answered while a pass runs", composeAnsweredWhileAPassRuns) @@ -90,6 +108,12 @@ func TestTheNodePublishesItsCompositions(t *testing.T) { state := &fakeComposed{kv: map[string]ComposedEntry{}} env := b.env() env.ComposedState = state + ofp0 := meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64) + half := catalogHalf{} + half.add(t, "lemurs", fp, lemurs) + half.add(t, "oldie", ofp0, oldie) + env.Entry = half.call() + state.keep(t, half, "lemurs", fp) // Design 47, *What each step says*: each module composition written says composed, with its fingerprint and digest. var saidMu sync.Mutex var saidComposed []string @@ -124,13 +148,12 @@ func TestTheNodePublishesItsCompositions(t *testing.T) { } b.put(t, buckets.Wants, "want.n1.lemurs", want) b.put(t, buckets.Wants, "want.n1.node", nodeWant) - for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, buckets.Modules, ResolvedBucket, builds.BuilderBuildsBucket} { + for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, ResolvedBucket} { b.send(bucket, agent.Update{Caught: true}) } for u := next(t, out); !u.Caught; u = next(t, out) { } lemursKey, _ := engcomp.Key("n1", "lemurs", fp) - oldieKey, _ := engcomp.Key("n1", "oldie", ofp) waitFor := func(what string, ok func([]string) bool) []string { t.Helper() deadline := time.Now().Add(5 * time.Second) @@ -190,60 +213,6 @@ func TestTheNodePublishesItsCompositions(t *testing.T) { if got := state.written(); len(got) != 2 { t.Errorf("a pass that changed nothing wrote %v", got[2:]) } - // oldie's trunk build is done: its key at that build is written once, though no want names it. - build, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: odigest}) - bkey, _ := builds.BuilderBuildKey("oldie", ofp) - b.mu.Lock() - b.kv[builds.BuilderBuildsBucket+"/"+bkey] = build - b.mu.Unlock() - b.put(t, builds.BuilderBuildsBucket, bkey, build) - got := waitFor("oldie's done build", func(got []string) bool { return len(got) >= 3 }) - if got[2] != oldieKey { - t.Errorf("the done build of oldie wrote %v, want %s", got[2:], oldieKey) - } - b.put(t, builds.BuilderBuildsBucket, bkey, build) - time.Sleep(100 * time.Millisecond) - if got := state.written(); len(got) != 3 { - t.Errorf("the same build landing again wrote %v", got[3:]) - } - - // A placed module whose newest build failed is published as a refusal at that build with the build's words (the - // provisioner holds it and does not wait); one whose build is queued or running stays unpublished. - for m, st := range map[string]builds.BuilderBuild{ - "broken": {State: builds.BuildFailed, Error: `refused kind: package of language "go"`}, - "waiting": {State: builds.BuildQueued}, - } { - f := map[string]string{"broken": bfp, "waiting": wfp}[m] - raw, _ := builds.EncodeBuilderBuild(st) - k, _ := builds.BuilderBuildKey(m, f) - b.mu.Lock() - b.kv[builds.BuilderBuildsBucket+"/"+k] = raw - b.mu.Unlock() - b.put(t, builds.BuilderBuildsBucket, k, raw) - } - brokenKey, _ := engcomp.Key("n1", "broken", bfp) - got = waitFor("broken's failed build", func(got []string) bool { return len(got) >= 4 }) - // The pass that wrote it says so ("1 key(s) written" was said once before, for oldie); then only brokenKey is new. - for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) { - n := 0 - b.mu.Lock() - for _, l := range b.said { - if strings.Contains(l, "this node's composition state: 1 key(s) written") { - n++ - } - } - b.mu.Unlock() - if n >= 2 { - break - } - } - if got := state.written(); len(got) != 4 || got[3] != brokenKey { - t.Errorf("a failed build and a queued one wrote %v, want only %s", got[3:], brokenKey) - } - if comp, err := engcomp.Decode(state.kv[brokenKey].Value); err != nil || comp.Refused != `refused kind: package of language "go"` || len(comp.Resources) != 0 { - t.Errorf("the failed build's key: %+v, %v", comp, err) - } - // What is kept: the five keys of a module written last, and one a want names; the rest is deleted. keep := New(b.env()) keep.pubKept = map[string]ComposedEntry{} @@ -262,6 +231,7 @@ func TestTheNodePublishesItsCompositions(t *testing.T) { t.Errorf("past keeping with a want naming the oldest: %v, want none", past) } + brokenKey, _ := engcomp.Key("n1", "broken", bfp) // Design 47 G A-293: reading the catalog half by fingerprint, a placed module's newest build that ended is the last trunk // build.ended heard of it; a failed one is said refused with the event's words once the first read ended, an end naming a // branch moves nothing, a module with no end (queued or running) stays unpublished, and a later done end, at any diff --git a/internal/selfcompose/data_test.go b/internal/selfcompose/data_test.go index 2f61b889..41c90dfe 100644 --- a/internal/selfcompose/data_test.go +++ b/internal/selfcompose/data_test.go @@ -5,7 +5,6 @@ import ( "errors" "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "github.com/novox/mesh-host/internal/buckets" engcomp "github.com/novox/mesh-host/internal/compositions" @@ -17,8 +16,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-host/internal/compose/frombus" - - "github.com/novox/mesh-host/internal/agent" ) // fakeData is the engine's state data as a map, counting the writes and deletes made. @@ -231,39 +228,4 @@ func TestTheEngineKeepsEachPlacedModulesDataItems(t *testing.T) { if wrote, _ := pc.WritePackages(context.Background()); wrote { t.Error("the verb's write of a key holding what was composed wrote it again") } - - // The items are worked out from what the pass read: a done build is read over the bus once, not again for the data - // items after the pass (a module in a running build is read again, since it may land). - b := newFakeBus() - done, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: meshcatalogclient.ManifestDigest([]byte(lemurs))}) - running, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: "building"}) - fp := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64) - doneKey, _ := builds.BuilderBuildKey("web", fp) - runningKey, _ := builds.BuilderBuildKey("pg", fp) - b.kv[builds.BuilderBuildsBucket+"/"+doneKey], b.kv[builds.BuilderBuildsBucket+"/"+runningKey] = done, running - r := reader{c: New(b.env())} - for range 3 { - if got, err := r.Build(context.Background(), "web", fp); err != nil || got.State != builds.BuildDone { - t.Fatalf("the done build: %+v, %v", got, err) - } - if got, err := r.Build(context.Background(), "pg", fp); err != nil || got.State != "building" { - t.Fatalf("the running build: %+v, %v", got, err) - } - } - reads := map[string]int{} - for _, read := range b.reads { - reads[read[1]]++ - } - if reads[doneKey] != 1 || reads[runningKey] != 3 { - t.Errorf("the builds were read %v times over the bus; want the done one once, the running one each time", reads) - } - // A done key the builder claims again (a gone step, issue 1937) and lands done again is read again, not served kept. - again, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: meshcatalogclient.ManifestDigest([]byte("{}"))}) - b.mu.Lock() - b.kv[builds.BuilderBuildsBucket+"/"+doneKey] = again - b.mu.Unlock() - r.c.takeBuildDone(agent.Update{Key: doneKey, Value: running}) - if got, err := r.Build(context.Background(), "web", fp); err != nil || got.Manifest != meshcatalogclient.ManifestDigest([]byte("{}")) { - t.Errorf("a done build written again is served as %+v (%v), not read again", got, err) - } } diff --git a/internal/selfcompose/entries.go b/internal/selfcompose/entries.go index 8c427341..c45daa2e 100644 --- a/internal/selfcompose/entries.go +++ b/internal/selfcompose/entries.go @@ -7,7 +7,9 @@ import ( "encoding/json" "fmt" "git.novox.be/novox/mesh-sdk/go/buscalls" + "maps" "reflect" + "slices" "sort" "strings" "time" @@ -441,9 +443,10 @@ func referenceOf(module, artifact, digest string) string { return "artifact-store://" + module + "/" + artifact + "/blobs/" + digest } -// buildOf is the build a want names, made from its entry and its parts alone (the builder's done key is not read): done, with -// each artifact of the entry at the digest its part names and the reference made from it. Nil where the entry is not held, the -// want names another fingerprint, or a part is missing. +// buildOf is the build a want names, made from the entry the composer holds and the parts of the want (the builder's done key is +// not read, §421, §628): done, with each artifact of the entry at the digest its part names and the reference made from it. Nil +// where the entry is not held, the want names another fingerprint, or a part is missing. The shadow's wants for the modules a +// candidate adds carry the parts the candidate gives (A-316). func (c *Composer) buildOf(module, fp string) *builds.BuilderBuild { c.mu.Lock() w, wanted := c.wants[module] @@ -640,3 +643,36 @@ func (c *Composer) addedModule(ctx context.Context, module string) (fp string, e } return fp, e, false, true, fmt.Sprintf("the entry of %s's build %s is asked of the catalog half and not yet answered", module, fp) } + +// askBuildsEntries asks the entry of each build in fps, the builds the node runs or its wants name, that the composer does not hold, +// and says the first not yet answered: the shadow composes a candidate from entries alone (§421, §628), and an answer wakes it. The +// fingerprints are kept beside the candidates' added ones while a proposal stands (addedFps). +func (c *Composer) askBuildsEntries(fps map[string]string) (pending string) { + if !c.byEntries() { + return "" + } + for _, module := range slices.Sorted(maps.Keys(fps)) { + fp := fps[module] + if !strings.HasPrefix(fp, meshcatalogclient.SourcePrefix2) { + continue + } + c.mu.Lock() + if c.addedFps == nil { + c.addedFps = map[string]bool{} + } + c.addedFps[fp] = true + c.mu.Unlock() + e, state := c.entryOf(fp) + switch { + case state == meshcatalogclient.EntryAnswered: + c.mu.Lock() + if _, kept := c.keptEnt[fp]; !kept && e.Module != "" { + c.keptEnt[fp] = keptEntry{entry: e, since: c.now()} + } + c.mu.Unlock() + case state == meshcatalogclient.EntryAsked && pending == "": + pending = fmt.Sprintf("the entry of %s's build %s is asked of the catalog half and not yet answered", module, fp) + } + } + return pending +} diff --git a/internal/selfcompose/entries_test.go b/internal/selfcompose/entries_test.go index 15b6dd4e..b7fb2ac1 100644 --- a/internal/selfcompose/entries_test.go +++ b/internal/selfcompose/entries_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "strings" "sync" "testing" @@ -12,7 +13,6 @@ import ( "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" @@ -97,7 +97,7 @@ func composeFromEntries(t *testing.T) { // Neither the indexer's states nor the builder's are watched or read. b.mu.Lock() for _, r := range b.reads { - if r[0] == buckets.Modules || r[0] == buckets.Manifests || r[0] == buckets.Verdicts || r[0] == builds.BuilderBuildsBucket { + if r[0] == buckets.Modules || r[0] == buckets.Manifests || r[0] == buckets.Verdicts || strings.HasSuffix(r[0], "_builds") { t.Errorf("the node-engine read %s of %s", r[1], r[0]) } } @@ -183,3 +183,68 @@ func composeFromEntries(t *testing.T) { t.Errorf("a removed want left its kept want %v and entry %v", hasWant, hasEntry) } } + +// takeModule lays one module of the catalog half's picture as the composer's own lay of an entry does (layModules): the tests +// that compose from a picture they set read no indexer. +func (c *Composer) takeModule(u agent.Update) { + name, err := meshcatalogclient.ParseModulesKey(u.Key) + if err != nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + m, err := meshcatalogclient.DecodeModule(u.Value) + if u.Deleted || err != nil { + delete(c.modules, name) + return + } + c.modules[name] = m +} + +// catalogHalf is the catalog half's entries a test composes from, by fingerprint, answered through the seat mesh-catalog's verbs +// entry and module (design 47 G §421, A-312): the node-engine reads no indexer. +type catalogHalf map[string]meshcatalog.CatalogEntry + +// add holds the entry of module at fp, made of the manifest body. +func (h catalogHalf) add(t *testing.T, module, fp, body string) { + t.Helper() + var m map[string]any + if err := json.Unmarshal([]byte(body), &m); err != nil { + t.Fatal(err) + } + h[fp] = meshcatalog.CatalogEntry{Module: module, Fp: fp, Manifest: m, Repository: "novox/" + module, Commit: "c", ManifestDigest: meshcatalogclient.ManifestDigest([]byte(body))} +} + +func (h catalogHalf) call() meshcatalogclient.Call { + return EntryCall(func(_ context.Context, _, verb string, args any) (json.RawMessage, error) { + switch in := args.(type) { + case meshcatalog.ModuleInput: + for _, e := range h { + if e.Module == in.Module { + return json.Marshal(meshcatalog.ModuleAnswer{Module: meshcatalog.ModuleAnswerModule{Module: e.Module, Repository: e.Repository, Commit: e.Commit, Fp: e.Fp}}) + } + } + return nil, fmt.Errorf("the catalog holds no module %s", in.Module) + case meshcatalog.EntryInput: + if e, ok := h[in.Fingerprint]; ok { + return json.Marshal(meshcatalog.EntryAnswer{Entry: e}) + } + return nil, errors.New("no entry of fingerprint " + in.Fingerprint + " is kept") + } + return nil, fmt.Errorf("asked %s of %T", verb, args) + }) +} + +// holdEntry makes the composer hold the entry of module at fp, as its call of the verb entry would have answered it (§421), for the +// tests that set the rest of its picture directly. +func (c *Composer) holdEntry(t *testing.T, module, fp, body string) { + t.Helper() + half := catalogHalf{} + half.add(t, module, fp, body) + c.mu.Lock() + defer c.mu.Unlock() + if c.keptEnt == nil { + c.keptEnt = map[string]keptEntry{} + } + c.keptEnt[fp] = keptEntry{entry: half[fp]} +} diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 07999e2c..02b63622 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -180,10 +180,7 @@ type Composer struct { values map[string]map[string]any // by module, this node's manifests map[string][]byte // by digest, read once archives map[string][]byte // templates archives by digest, read once - // builds are the builder's done builds by key, read once: a done build at a fingerprint never changes, so a pass and - // the data items after it (publishData) do not read each placed module's build again over the bus (ADR 0577). - builds map[string]builds.BuilderBuild - handed map[string]string // composed key to the digest handed + handed map[string]string // composed key to the digest handed // notTaken is, by module, why a want of this node's was not taken (it could not be decoded, or names no fingerprint). notTaken map[string]string // unmadeSaid is, by module, the stable part of the reason its last journal line said (without where the picture stands). @@ -203,8 +200,6 @@ type Composer struct { forCandidates bool // addedFps are the builds of modules a candidate adds that the shadow asked the entry of (addedModule), kept beside the wants' under mu. addedFps map[string]bool - // modulesRev is the revision of the last update of the indexer's modules taken: where the picture of them stands. - modulesRev uint64 // heard is, by node and module, what each node's event running last said it runs, kept from the seat's stream by this // node's own consumer (NoteRunning), under mu; heardRead is whether the first read of the stream ended (RunningRead). heard map[string]map[string]heardRun @@ -418,7 +413,7 @@ func New(env Env) *Composer { func newComposer(env Env) *Composer { return &Composer{env: env, wants: map[string]meshwants.WantsByWantNodeModuleValue{}, assignments: map[string]meshmoduleassigner.Assignments{}, assignmentKeys: meshmoduleassignerclient.NewKeys(), records: map[string]nodeengine.NodeRecord{}, inOwn: map[string]bool{}, modules: map[string]meshcatalog.ModulesByModuleValue{}, values: map[string]map[string]any{}, - manifests: map[string][]byte{}, archives: map[string][]byte{}, builds: map[string]builds.BuilderBuild{}, handed: map[string]string{}, notTaken: map[string]string{}, unmadeSaid: map[string]string{}, rehand: make(chan struct{}, 1), sealedOwn: map[string]ownSealed{}, + manifests: map[string][]byte{}, archives: map[string][]byte{}, handed: map[string]string{}, notTaken: map[string]string{}, unmadeSaid: map[string]string{}, rehand: make(chan struct{}, 1), sealedOwn: map[string]ownSealed{}, pkgs: map[string]compositions.Composed{}, asked: make(chan chan Pass, 8)} } @@ -478,22 +473,9 @@ type input struct { changed func(c *Composer, u agent.Update) bool } +// inputs are the buckets the composer watches: it watches neither the indexer's modules nor the builder's builds, but reads the +// catalog half's entry of each build its wants name (§421, §628). func (c *Composer) inputs() []input { - ins := c.allInputs() - if !c.byEntries() { - return ins - } - // Reading the catalog half by fingerprint, it watches neither the indexer's modules nor the builder's builds (§421, §628). - out := ins[:0:0] - for _, in := range ins { - if in.bucket != buckets.Modules && in.bucket != builds.BuilderBuildsBucket { - out = append(out, in) - } - } - return out -} - -func (c *Composer) allInputs() []input { node := c.env.Node return []input{ {buckets.Wants, "want." + node + ".>", (*Composer).takeWant, nil}, @@ -503,50 +485,10 @@ func (c *Composer) allInputs() []input { // A watch of every key is refused by the bus (issue 1126, 2026-10-27 09:21: the fold's `>` stopped every agent). {buckets.Have, "*." + nodeengineclient.NodeWord, nil, (*Composer).takeRecordChanged}, {buckets.Have, node + ".>", nil, (*Composer).takeModuleHave}, - {buckets.Modules, ">", (*Composer).takeModule, nil}, {ResolvedBucket, node + ".>", (*Composer).takeValue, nil}, - // The builder's builds: a pass that held a module for a build not yet done is made again when it lands (issue - // 774, 2026-10-30 19:30: node-tools' build landed and ace, g14 and shanks stayed refused until another input moved). - {builds.BuilderBuildsBucket, ">", nil, (*Composer).takeBuildDone}, } } -// takeBuildDone says whether a build that landed is one this node composes from: done, of a module it wants at that -// build, or wanted or placed here whose trunk's build it is; or failed, of a module placed here whose trunk's build it -// is, which the node publishes as a refusal (ADR 0593 §6). Every other build moves nothing. -func (c *Composer) takeBuildDone(u agent.Update) bool { - // Any write or delete of a build's key forgets the build kept as done: the builder claims a done key again as a new - // revision (a gone step, issue 1937), which lands done with other artifacts. - c.mu.Lock() - delete(c.builds, u.Key) - c.mu.Unlock() - if u.Deleted { - return false - } - module, fp, err := builds.ParseBuilderBuildKey(u.Key) - if err != nil { - return false - } - c.mu.Lock() - w, wanted := c.wants[module] - m := c.modules[module] - placedHere := false - for _, pm := range c.assignments[c.env.Node].Modules { - placedHere = placedHere || pm.Module == module - } - c.mu.Unlock() - // A module placed here is composed at its trunk's build once that is done, for the composition state, before its - // want moves (ADR 0593 §6). - b, err := builds.DecodeBuilderBuild(u.Value) - if err != nil { - return false - } - if b.State == builds.BuildFailed { - return placedHere && m.Fp2 == fp - } - return b.State == builds.BuildDone && (wanted && w.Fp == fp || (wanted || placedHere) && m.Fp2 == fp) -} - // Watch starts the composer and answers the compositions it makes: each as a key of `composed-` would be, then a // Caught update once every input was read and the node composed once, then every change. It ends when ctx ends or an // input's watch ends. @@ -794,13 +736,6 @@ func (c *Composer) entry(bucket, key string) (v any, has, known bool) { } p, ok := c.assignments[node] return p, ok, true - case buckets.Modules: - name, err := meshcatalogclient.ParseModulesKey(key) - if err != nil { - return nil, false, true - } - m, ok := c.modules[name] - return m, ok, true case ResolvedBucket: rest, ok := strings.CutPrefix(key, c.env.Node+".") if !ok { @@ -969,22 +904,6 @@ func (c *Composer) inOwnOf(p meshmoduleassigner.Assignments) []string { return out } -func (c *Composer) takeModule(u agent.Update) { - name, err := meshcatalogclient.ParseModulesKey(u.Key) - if err != nil { - return - } - c.mu.Lock() - defer c.mu.Unlock() - m, err := meshcatalogclient.DecodeModule(u.Value) - if u.Deleted || err != nil { - delete(c.modules, name) - return - } - c.modules[name] = m - c.modulesRev = u.Revision -} - func (c *Composer) takeValue(u agent.Update) { rest, ok := strings.CutPrefix(u.Key, c.env.Node+".") if !ok { @@ -1659,33 +1578,10 @@ func (r reader) Manifest(ctx context.Context, digest string) ([]byte, error) { return []byte(m.Manifest), nil } -func (r reader) Build(ctx context.Context, module, fp string) (*builds.BuilderBuild, error) { - if r.c.byEntries() && !r.c.forCandidates { - return r.c.buildOf(module, fp), nil - } - key, err := builds.BuilderBuildKey(module, fp) - if err != nil { - return nil, err - } - r.c.mu.Lock() - done, ok := r.c.builds[key] - r.c.mu.Unlock() - if ok { - return &done, nil - } - defer r.c.waiting("the build of " + module + " at " + fp + " (" + builds.BuilderBuildsBucket + ")")() - raw, err := r.c.get(ctx, builds.BuilderBuildsBucket, key) - if err != nil || raw == nil { - return nil, err - } - b, err := builds.DecodeBuilderBuild(raw) - if err != nil { - return nil, fmt.Errorf("the builder's %s: %w", key, err) - } - if b.State == builds.BuildDone { - r.c.keepRead(func(k *Composer) { k.builds[key] = b }) - } - return &b, nil +// Build is the build a want names, made from the catalog half's entry the composer holds (§421, §628): the builder's builds are +// not read. +func (r reader) Build(_ context.Context, module, fp string) (*builds.BuilderBuild, error) { + return r.c.buildOf(module, fp), nil } // archiveWithin bounds one read of a templates archive in the artifact store. @@ -2313,18 +2209,18 @@ func (c *Composer) unmadeOf(fps map[string]string, comps map[string]engcomp.Comp why = fmt.Sprintf("the seat's assignment places no %s on this node (%d modules placed)", module, len(own.Modules)) key = "not placed" case !in: - why = fmt.Sprintf("no manifest in the composer's picture of the indexer's modules (%d modules, last update at revision %d)", - len(c.modules), c.modulesRev) + why = fmt.Sprintf("no manifest in the composer's picture of the indexer's modules (%d modules)", + len(c.modules)) key = "no manifest" case slices.Contains(skipped, module): why = "the composition skipped it: the declaration gives it no fingerprint (indexer's picture: " + m.Fp2 + ")" key = why default: if at := made[module]; at != "" { - why = fmt.Sprintf("the pass composed it at %s and the want names %s (indexer's picture: %s, revision %d)", at, fp, m.Fp2, c.modulesRev) + why = fmt.Sprintf("the pass composed it at %s and the want names %s (indexer's picture: %s)", at, fp, m.Fp2) key = "composed at " + at } else { - why = fmt.Sprintf("the composition made nothing of it at %s (indexer's picture: %s, revision %d)", fp, m.Fp2, c.modulesRev) + why = fmt.Sprintf("the composition made nothing of it at %s (indexer's picture: %s)", fp, m.Fp2) key = "made nothing at " + m.Fp2 } } diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index 4face352..0620df94 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -210,11 +210,14 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { b.put(t, buckets.Have, "n1.node", have) oldHave, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Build: "build-1", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) b.put(t, buckets.Have, "n1.oldie", oldHave) - b.put(t, buckets.Modules, "lemurs", mod) - b.put(t, buckets.Modules, "oldie", omod) + // The picture of the catalog half is set directly (no indexer is watched): lemurs's entry, which its want names, and oldie, + // which the node runs inside its own composition and no want names. + c.takeModule(agent.Update{Key: "lemurs", Value: mod}) + c.takeModule(agent.Update{Key: "oldie", Value: omod}) + c.holdEntry(t, "lemurs", fp, lemurs) b.put(t, buckets.Wants, "want.n1.lemurs", want) b.put(t, buckets.Wants, "want.n1.node", nodeWant) - for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, buckets.Modules, ResolvedBucket, builds.BuilderBuildsBucket} { + for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, ResolvedBucket} { b.send(bucket, agent.Update{Caught: true}) } got := map[string]bool{} @@ -636,38 +639,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { b.put(t, ResolvedBucket, "n1.other.k", []byte(`{"value":1,"source":"node"}`)) quiet(t, out) - // Issue 774, 2026-10-30 19:30: node-tools' want moved to a build not yet done, and the node stayed refused after it - // landed until another input moved. A done build of a module the node wants, at its want's or the trunk's build, - // makes a pass, and so does one of a module placed here at its trunk's build though no want names it (novox/hq ADR 0593 - // §6: the composition state holds it at its newest done build before its want moves); a build not done, of another - // build or of a module neither wanted nor placed here, makes none. passes := func() int64 { return composed.Load() } - for _, row := range []struct { - name, module, fp, state string - pass bool - }{ - {"lemurs's wanted build, not done", "lemurs", fp, "building", false}, - {"lemurs's wanted build lands", "lemurs", fp, builds.BuildDone, true}, - {"another build of lemurs", "lemurs", meshcatalogclient.SourcePrefix2 + strings.Repeat("e", 64), builds.BuildDone, false}, - {"a placed module no want names, at its trunk's build", "oldie", meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64), builds.BuildDone, true}, - {"a module neither wanted nor placed here", "ghost", meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64), builds.BuildDone, false}, - } { - before := passes() - key, _ := builds.BuilderBuildKey(row.module, row.fp) - build := builds.BuilderBuild{State: row.state} - if row.state == builds.BuildDone { - build.Manifest = digest - } - raw, err := builds.EncodeBuilderBuild(build) - if err != nil { - t.Fatal(err) - } - b.put(t, builds.BuilderBuildsBucket, key, raw) - quiet(t, out) - if got := passes() > before; got != row.pass { - t.Errorf("%s: a pass made %v, want %v", row.name, got, row.pass) - } - } // Issue 774, 2026-10-31: every put under want.n1.> made a pass, and each pass asked the vault for the node's own // secrets. A key written again with what it held, or a want whose words moved and not its build, makes none. worded, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fp, Self: true, Words: "lemurs is held at aaaaaaaaaaaa"}) @@ -726,7 +698,6 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { {"the node's own want again", buckets.Wants, "want.n1.node", nodeWant, false}, {"lemurs's want with other words", buckets.Wants, "want.n1.lemurs", worded, false}, {"the assignment again", buckets.Assignments, "n1.node", place["n1.node"], false}, - {"lemurs's module again", buckets.Modules, "lemurs", mod, false}, {"lemurs's want at another build", buckets.Wants, "want.n1.lemurs", elsewhere, true}, {"lemurs's want back at its build", buckets.Wants, "want.n1.lemurs", want, true}, }, assignmentRows...) { @@ -747,72 +718,6 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { t.Errorf("%s: a pass made %v, want %v", row.name, got, row.pass) } } - // And today's sequence whole: tool runs at its old build; its want and the trunk move to a build not yet done, and - // the node holds tool, keeping what runs, naming why; the build lands, and with no other change tool is handed at it. - { - tb := newFakeBus() - tool := `{"module":"tool","version":"1","capabilities":["systemd"],"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},` + - `"resources":[{"id":"service","type":"service","unit":"tool.service","state":"running","boot":"enabled"}]}` - tdigest := meshcatalogclient.ManifestDigest([]byte(tool)) - tman, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "tool", Manifest: tool}) - tkey, _ := meshcatalogclient.ManifestsKey(tdigest) - tb.kv[buckets.Manifests+"/"+tkey] = tman - old, next2 := meshcatalogclient.SourcePrefix2+strings.Repeat("1", 64), meshcatalogclient.SourcePrefix2+strings.Repeat("2", 64) - tenv := tb.env() - var held []string - var hmu sync.Mutex - tenv.Refused = func(why string, h map[string]string) { - hmu.Lock() - defer hmu.Unlock() - held = append(held, why+fmt.Sprint(h)) - } - tenv.Running = func() []compositions.Composed { - return []compositions.Composed{{Node: "n1", Module: "tool", FP: old, Resources: []json.RawMessage{ - json.RawMessage(`{"id":"service","module":"tool","type":"service","unit":"tool.service","state":"running","boot":"enabled"}`)}}} - } - tc := New(tenv) - tout, err := tc.Watch(ctx, "n1.>") - if err != nil { - t.Fatal(err) - } - tplace := assignedValues(t, meshmoduleassigner.Assignments{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"}, - Modules: []meshmoduleassigner.PlacedModule{{Module: "tool"}}}) - tmod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: next2, Manifest: tdigest, Repository: "novox/t", Commit: "c"}) - twant, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: next2, Self: true}) - building, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: "building"}) - bkey, _ := builds.BuilderBuildKey("tool", next2) - tb.kv[builds.BuilderBuildsBucket+"/"+bkey] = building - tb.putAssigned(t, tplace) - tb.put(t, buckets.Have, "n1.node", have) - tb.put(t, buckets.Modules, "tool", tmod) - tb.put(t, buckets.Wants, "want.n1.tool", twant) - for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, buckets.Modules, - ResolvedBucket, builds.BuilderBuildsBucket} { - tb.send(bucket, agent.Update{Caught: true}) - } - toolKey, _ := compositions.Key("n1", "tool", next2) - for u := next(t, tout); !u.Caught; u = next(t, tout) { - if u.Key == toolKey { - t.Errorf("tool is handed at a build not yet done") - } - } - hmu.Lock() - if len(held) == 0 || !strings.Contains(held[len(held)-1], "no done build yet") { - t.Errorf("the node was told %q, want tool held for its build not yet done", held) - } - hmu.Unlock() - done, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: tdigest, - Artifacts: []builds.BuiltArtifact{{Name: "files", Kind: "archive", Reference: "artifact-store://tool/files/blobs/sha256:" + strings.Repeat("9", 64), - Digest: "sha256:" + strings.Repeat("9", 64)}}}) - tb.mu.Lock() - tb.kv[builds.BuilderBuildsBucket+"/"+bkey] = done - tb.mu.Unlock() - tb.put(t, builds.BuilderBuildsBucket, bkey, done) - if u := next(t, tout); u.Key != toolKey || u.Deleted { - t.Errorf("the build landed and the node handed %s, want tool at its new build", u.Key) - } - } - // A want removed takes its composition back. removed, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Removed: true}) b.put(t, buckets.Wants, "want.n1.lemurs", removed) @@ -853,10 +758,12 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { b.mu.Lock() b.kv[buckets.Manifests+"/"+kkey] = kman b.mu.Unlock() - kmod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: kdigest, Repository: "novox/k", Commit: "c"}) + kfp9 := meshcatalogclient.SourcePrefix2 + strings.Repeat("9", 64) + kmod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: kfp9, Manifest: kdigest, Repository: "novox/k", Commit: "c"}) both := assignedValues(t, meshmoduleassigner.Assignments{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"}, Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}, {Module: "keeper"}}}) - b.put(t, buckets.Modules, "keeper", kmod) + c.takeModule(agent.Update{Key: "keeper", Value: kmod}) + c.holdEntry(t, "keeper", kfp9, keeper) // n1 is held in keys per module and node, which the writes below move. bothPlaced, _ := meshmoduleassignerclient.AssignmentsOf(both) bothNode := bothPlaced["n1"] @@ -869,8 +776,9 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { b.put(t, buckets.Assignments, w.Key, w.Value) } } - b.put(t, buckets.Wants, "want.n1.keeper", want) - keeperKey, _ := compositions.Key("n1", "keeper", fp) + keeperWant, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: kfp9, Self: true}) + b.put(t, buckets.Wants, "want.n1.keeper", keeperWant) + keeperKey, _ := compositions.Key("n1", "keeper", kfp9) var handed string for handed == "" { if u := next(t, out); u.Key == keeperKey { @@ -1138,9 +1046,19 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { return meshcatalog.CatalogEntry{Module: module, Fp: fp, Manifest: m, Repository: "novox/x", Commit: "c", ManifestDigest: meshcatalogclient.ManifestDigest([]byte(body))} } catalogEntries := map[string]meshcatalog.CatalogEntry{fp: entryOfManifest("lemurs", fp, lemurs), kfp: entryOfManifest("keeper", kfp, keeper)} + // A module with an artifact: its build is made from its entry and the parts the candidate gives (design 47 G §628, A-316). + artFp := meshcatalogclient.SourcePrefix2 + strings.Repeat("7", 64) + artManifest := `{"module":"art","version":"1","capabilities":["systemd"],"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},` + + `"resources":[{"id":"service","type":"service","unit":"art.service","state":"running","boot":"enabled"}]}` + artEntry := entryOfManifest("art", artFp, artManifest) + artEntry.Artifacts = []meshcatalog.Artifact{{Kind: "archive", Language: "none", Ref: "art:files"}} + catalogEntries[artFp] = artEntry senv.Entry = EntryCall(func(_ context.Context, _, verb string, args any) (json.RawMessage, error) { switch in := args.(type) { case meshcatalog.ModuleInput: + if in.Module == "nofp" { // a module the catalog holds and gives no fingerprint (A-312) + return json.Marshal(meshcatalog.ModuleAnswer{Module: meshcatalog.ModuleAnswerModule{Module: "nofp", None: "its build is not made yet"}}) + } for _, e := range catalogEntries { if e.Module == in.Module { return json.Marshal(meshcatalog.ModuleAnswer{Module: meshcatalog.ModuleAnswerModule{Module: e.Module, Repository: e.Repository, Commit: e.Commit, Fp: e.Fp}}) @@ -1237,6 +1155,20 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { candidate("adds", 4, "lemurs", "keeper") candidate("drops", 5) candidate("ghost", 6, "lemurs", "ghost") + candidate("nofp", 11, "lemurs", "nofp") + // A module with an artifact: held when the candidate gives no part for it, built from its entry and the parts when it does. + for group, added := range map[string]map[string]meshmoduleassigner.AddedModule{ + "art-without": nil, + "art-with": {"art": {Version: "1", Parts: map[string]string{"art:files": "sha256:" + strings.Repeat("9", 64)}}}, + } { + p := trunk + p.Modules = []meshmoduleassigner.PlacedModule{{Module: "lemurs"}, {Module: "art"}} + raw, err := bucketstest.Candidate(meshmoduleassigner.CandidatesByProposalNodeValue{Proposal: group, Assignments: &p, Added: added}) + if err != nil { + t.Fatal(err) + } + sb.put(buckets.ProposalCandidates, meshmoduleassigner.CandidatesByProposalNodeKey(group, "n1"), raw, 10) + } // A head deleting n1's assignment file (the seed's revert, mesh-catalog #526): n1 returns to the controller's // compositions, answered as such, not as every module removed. if raw, err := bucketstest.Candidate(meshmoduleassigner.CandidatesByProposalNodeValue{Proposal: "deletes"}); err != nil { @@ -1297,11 +1229,22 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { return a.Verdict == nodeengine.CandidateAnswerVerdictDiffers && len(a.Added) == 1 && a.Added[0].Module == "keeper" && len(a.Added[0].Unissued) == 1 && len(a.Refusals) == 0 }, + "art-without": func(a nodeengine.CandidateAnswer) bool { + return a.Verdict == nodeengine.CandidateAnswerVerdictSame && len(a.Held) == 1 && a.Held[0].Module == "art" && + strings.Contains(a.Held[0].Why, "the candidate gives no part for art:files of art's build "+artFp) + }, + "art-with": func(a nodeengine.CandidateAnswer) bool { + return len(a.Added) == 1 && a.Added[0].Module == "art" && len(a.Held) == 0 + }, "drops": func(a nodeengine.CandidateAnswer) bool { return a.Verdict == nodeengine.CandidateAnswerVerdictDiffers && len(a.Removed) == 1 && a.Removed[0] == "lemurs" }, + // A module the verb module answers an error for, or no fingerprint, is held with the verb's reason (A-312), the rest composed. "ghost": func(a nodeengine.CandidateAnswer) bool { - return a.Verdict == nodeengine.CandidateAnswerVerdictRefused && a.Refused != "" + return len(a.Held) == 1 && a.Held[0].Module == "ghost" && strings.Contains(a.Held[0].Why, "the catalog holds no module ghost") && a.Refused == "" + }, + "nofp": func(a nodeengine.CandidateAnswer) bool { + return len(a.Held) == 1 && a.Held[0].Module == "nofp" && strings.Contains(a.Held[0].Why, "gives nofp no fingerprint: its build is not made yet") }, "reach": func(a nodeengine.CandidateAnswer) bool { return a.Verdict == nodeengine.CandidateAnswerVerdictSame && a.Returns == "" && a.Revision == 8 && len(a.Removed) == 0 && @@ -1417,7 +1360,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { } k := meshmoduleassigner.CandidatesByProposalNodeKey("same", "n1") sb.del(buckets.ProposalCandidates, k) - waitFor("the deleted group's answer stays", func(a map[string]nodeengine.CandidateAnswer) bool { _, ok := a["same"]; return !ok && len(a) == 5 }) + waitFor("the deleted group's answer stays", func(a map[string]nodeengine.CandidateAnswer) bool { _, ok := a["same"]; return !ok && len(a) == 8 }) // Review of L2b, round 2: a watch ends while the link stays; a group's key is deleted before the shadow watches // again. The new run reads afresh, so that group is no longer answered. @@ -1429,7 +1372,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { sb.missing(buckets.ProposalCandidates, false) waitFor("a group deleted while the shadow was down is still answered", func(a map[string]nodeengine.CandidateAnswer) bool { _, ok := a["drops"] - return !ok && len(a) == 4 + return !ok && len(a) == 7 }) // Issue 1126, A7 AC1: a assignment that cannot be read composes nothing and says why; the node never applies a @@ -1477,7 +1420,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { oldRaw, _ := bucketstest.Candidate(meshmoduleassigner.CandidatesByProposalNodeValue{Proposal: "older", Assignments: &trunk}) oldKey := meshmoduleassigner.CandidatesByProposalNodeKey("older", "n1") nb.put(buckets.ProposalCandidates, oldKey, oldRaw, 2) - for _, bucket := range []string{buckets.ProposalCandidates, buckets.Assignments, buckets.Have, buckets.Wants, builds.BuilderBuildsBucket} { + for _, bucket := range []string{buckets.ProposalCandidates, buckets.Assignments, buckets.Have, buckets.Wants} { nb.send(bucket, agent.Update{Caught: true}) } select { @@ -1568,10 +1511,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { // Issue 1126, the seed's refusals: what a node composes from its builds and reads of a provider elsewhere. A build // is the builder's key; a templates archive is fetched once per digest and refused when it is not that digest; another // node's served key is read alone, as the node's grant allows it (buscompose Principal.ServedReads). - built, _ := builds.EncodeBuilderBuild(builds.BuilderBuild{State: builds.BuildDone, Manifest: digest}) - buildKey, _ := builds.BuilderBuildKey("lemurs", fp) b.mu.Lock() - b.kv[builds.BuilderBuildsBucket+"/"+buildKey] = built b.kv[ResolvedBucket+"/n2.mailer.domain"] = []byte(`{"value":"example.org","source":"module"}`) b.reads = nil b.mu.Unlock() @@ -1580,7 +1520,14 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { sum := sha256.Sum256(archive) env.Fetch = func(context.Context, string) ([]byte, error) { fetched++; return archive, nil } r := reader{c: New(env)} - if got, err := r.Build(ctx, "lemurs", fp); err != nil || got == nil || got.Manifest != digest { + // A build is made from the entry of its fingerprint that the composer holds, with the parts its want names (design 47 G §421, §628). + var lemursManifest map[string]any + if err := json.Unmarshal([]byte(lemurs), &lemursManifest); err != nil { + t.Fatal(err) + } + r.c.keptEnt = map[string]keptEntry{fp: {entry: meshcatalog.CatalogEntry{Module: "lemurs", Fp: fp, Manifest: lemursManifest, ManifestDigest: digest}}} + r.c.wants["lemurs"] = meshwants.WantsByWantNodeModuleValue{Fp: fp, Self: true} + if got, err := r.Build(ctx, "lemurs", fp); err != nil || got == nil || got.State != builds.BuildDone || got.Manifest != digest { t.Errorf("the build of lemurs: %+v, %v", got, err) } if got, err := r.Build(ctx, "lemurs", other); err != nil || got != nil { @@ -1907,6 +1854,15 @@ func TestAnUnreadSeatListHoldsWhatIsNewAndHandsItAtTheRead(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() env := b.env() + afp := meshcatalogclient.SourcePrefix2 + strings.Repeat("b", 64) + half := catalogHalf{} + half.add(t, "lemurs", fp, lemurs) + half.add(t, "avahi", afp, avahi) + env.Entry = half.call() + state := &fakeComposed{kv: map[string]ComposedEntry{}} + state.keep(t, half, "lemurs", fp) // the entries asked of the catalog half are answered, as kept by an earlier run (§683) + state.keep(t, half, "avahi", afp) + env.ComposedState = state g := &namedGate{gate: &gate{}, seats: []string{"mesh-catalog", "node-engine"}} changed := make(chan struct{}, 1) env.SeatList, env.SeatListChanged = g, changed @@ -1935,11 +1891,13 @@ func TestAnUnreadSeatListHoldsWhatIsNewAndHandsItAtTheRead(t *testing.T) { b.putAssigned(t, place) b.put(t, buckets.Have, "n1.node", have) b.put(t, buckets.Modules, "lemurs", mod) - avahiMod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: avahiDigest, Repository: "novox/x", Commit: "c"}) + avahiMod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: afp, Manifest: avahiDigest, Repository: "novox/x", Commit: "c"}) b.put(t, buckets.Modules, "avahi", avahiMod) b.put(t, buckets.Wants, "want.n1.lemurs", want) + avahiWant, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: afp, Self: true}) + b.put(t, buckets.Wants, "want.n1.avahi", avahiWant) b.put(t, buckets.Wants, "want.n1.node", nodeWant) - for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, buckets.Modules, ResolvedBucket, builds.BuilderBuildsBucket} { + for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, ResolvedBucket} { b.send(bucket, agent.Update{Caught: true}) } handed := map[string]bool{} @@ -2020,6 +1978,9 @@ func TestAHeldOwnCompositionThatWasHandedIsNotDropped(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() env := b.env() + half := catalogHalf{} + half.add(t, "lemurs", fp, lemurs) + env.Entry = half.call() g := &gate{} changed := make(chan struct{}, 1) env.SeatList, env.SeatListChanged = g, changed @@ -2041,7 +2002,7 @@ func TestAHeldOwnCompositionThatWasHandedIsNotDropped(t *testing.T) { b.put(t, buckets.Modules, "lemurs", mod) b.put(t, buckets.Wants, "want.n1.lemurs", want) b.put(t, buckets.Wants, "want.n1.node", nodeWant) - for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, buckets.Modules, ResolvedBucket, builds.BuilderBuildsBucket} { + for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, ResolvedBucket} { b.send(bucket, agent.Update{Caught: true}) } // The list is read: the node's own composition is handed. Then the reader cannot act again (it restarted and has @@ -2151,7 +2112,7 @@ func TestTheHolderOfTheCatalogIsStartedFromTheOwnCompositionBeforeTheListIsReadA mk, _ := meshcatalogclient.ManifestsKey(digest) b.kv[buckets.Manifests+"/"+mk] = man rec, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64), Manifest: digest, Repository: "novox/x", Commit: "c"}) - b.put(t, buckets.Modules, m, rec) + c.takeModule(agent.Update{Key: m, Value: rec}) // a module run inside the node's own composition: no want names it, so no entry is asked hk, _ := nodeengineclient.HaveKey("n1", m) hv, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Build: "build-1", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) b.put(t, buckets.Have, hk, hv) @@ -2163,7 +2124,7 @@ func TestTheHolderOfTheCatalogIsStartedFromTheOwnCompositionBeforeTheListIsReadA b.putAssigned(t, place) b.put(t, buckets.Have, "n1.node", have) b.put(t, buckets.Wants, "want.n1.node", nodeWant) - for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, buckets.Modules, ResolvedBucket, builds.BuilderBuildsBucket} { + for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, ResolvedBucket} { b.send(bucket, agent.Update{Caught: true}) } servicesIn := func(u agent.Update) []string { diff --git a/internal/selfcompose/shadow.go b/internal/selfcompose/shadow.go index e272cf8c..1fff9c71 100644 --- a/internal/selfcompose/shadow.go +++ b/internal/selfcompose/shadow.go @@ -16,6 +16,7 @@ import ( engcomp "github.com/novox/mesh-host/internal/compositions" "github.com/novox/mesh-host/internal/judge" "maps" + "slices" "sort" "strings" "sync" @@ -592,9 +593,16 @@ func (s *Shadow) answerProposal(ctx context.Context, proposal string) (nodeengin inOwn := c.inOwnOf(*assignment) c.mu.Unlock() fps := shadowBuilds(running, *assignment, modules, inOwn) + if why := c.askBuildsEntries(fps); why != "" { + // Its answer wakes the shadow (entryMoved), which composes this proposal again. + c.missed.Add(1) + return nodeengine.CandidateAnswer{Proposal: proposal, Revision: int64(rev), Verdict: nodeengine.CandidateAnswerVerdictRefused, + Refused: why + ", so this answer waits for it"}, true + } // A module the candidate adds that no want here names: the trunk's build is asked of the catalog half's verb module, and its // entry by the verb entry (design 47 G §421, A-312). One with no build to take is held, with the reason. heldAdded := map[string]string{} + addedWants := map[string]meshwants.WantsByWantNodeModuleValue{} folded := map[string]bool{} for _, m := range inOwn { folded[m] = true @@ -620,6 +628,13 @@ func (s *Shadow) answerProposal(ctx context.Context, proposal string) (nodeengin heldAdded[pm.Module] = fmt.Sprintf("the entry of %s's build %s holds no manifest that can be read: %v", pm.Module, fp, err) continue } + // The module is built from its entry and the parts the candidate gives (A-316); an artifact it gives no part for holds it. + w := meshwants.WantsByWantNodeModuleValue{Fp: fp, Parts: own.c.Added[pm.Module].Parts, Version: own.c.Added[pm.Module].Version} + if missing := partsMissing(entry, w); missing != "" { + heldAdded[pm.Module] = fmt.Sprintf("the candidate gives no part for %s of %s's build %s, so the node composes nothing new of it", missing, pm.Module, fp) + continue + } + addedWants[pm.Module] = w m := meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: entry.Repository, Commit: entry.Commit, Path: entry.Path, Needs: entry.Needs, Packages: entry.Packages, Verdict: entry.Verdict} if entry.Inputs != nil { @@ -642,14 +657,41 @@ func (s *Shadow) answerProposal(ctx context.Context, proposal string) (nodeengin // The boxes it runs, by what each module's have and its own key say after a restart (Held), are reused for the // modules it runs: a value sealed again is no difference. held := map[string]meshwants.WantsByWantNodeModuleValue{agent.NodeModule: {}} + c.mu.Lock() for module := range fps { held[module] = meshwants.WantsByWantNodeModuleValue{Fp: fps[module]} + if w, ok := c.wants[module]; ok && w.Fp == fps[module] { + held[module] = w // the want's parts give the build's artifacts their digests (buildOf) + } + if w, ok := addedWants[module]; ok { + held[module] = w // so do the parts the candidate gives a module it adds + } } - c.mu.Lock() c.wants = held c.mu.Unlock() + // A module the candidate adds that cannot be built is held with the reason, and the rest is composed without it: composed with it, the + // assignment would be refused for a module nothing provides. + if len(heldAdded) > 0 { + without := *assignment + without.Modules = nil + for _, pm := range assignment.Modules { + if _, held := heldAdded[pm.Module]; !held { + without.Modules = append(without.Modules, pm) + } + } + given[node] = &without + } a := frombus.Answer(ctx, reader{c: c}, c.env.Mesh, frombus.Scope{Node: node, KeptAt: kept, Assignments: given, InOwn: inOwn, SameValue: c.sameValue, HeldToday: s.heldToday(heldAdded)}, fps, running, proposal, rev) + for _, module := range slices.Sorted(maps.Keys(heldAdded)) { + a.Held = append(a.Held, nodeengine.CandidateRefusal{Module: module, Why: heldAdded[module]}) + } + // What this proposal added is its own: another proposal that places the module without giving its parts must not find it composed. + c.mu.Lock() + for module := range addedWants { + delete(c.modules, module) + } + c.mu.Unlock() s.mu.Lock() ran := make(map[string]string, len(s.ran)) for m, d := range s.ran { -- 2.54.0 From 860ccd7e300a703f8456872770e9e42bee91b66a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 23:12:41 +0100 Subject: [PATCH 54/59] H9: an entry asked again by hand is asked again in the shadow's cache too, so a candidate waits no longer on a not-found it holds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §698 (entry-again), §421 (A-308: the shadow reads the entries its wants name) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- cmd/mesh-host/nodeagent.go | 1 + internal/selfcompose/entries.go | 3 +++ internal/selfcompose/selfcompose.go | 4 +++- internal/selfcompose/shadow.go | 6 ++++++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 8a256529..29a2822d 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -689,6 +689,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus // composition bucket) is never applied. While the seat's assignment holds no key for this node it composes // nothing and keeps running what it runs; a assignment it cannot watch stops the agent, said, and it begins again. composer := selfComposer(session, node, hooks, say) + composer.AlsoEntryAgain = shadow.EntryAgain composer.OnEngine = session.Own.Set // the first composing names the engine's own module on a first start (§527, §634) machinesNow.Store(composer) // the machines a failing network part points at, by the assignments (issue 774) composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone diff --git a/internal/selfcompose/entries.go b/internal/selfcompose/entries.go index c45daa2e..bcc6e610 100644 --- a/internal/selfcompose/entries.go +++ b/internal/selfcompose/entries.go @@ -505,6 +505,9 @@ func (c *Composer) FirstViews() (views []map[string]any, entries []EntryRow, wan // EntryAgain asks fp's entry of the catalog half again now (the seat's verb entry-again, §698): one the composer keeps, whatever // its state. The entry kept of it is dropped, so a pass holds the module until the answer. func (c *Composer) EntryAgain(ctx context.Context, fp string) error { + if c.AlsoEntryAgain != nil { + c.AlsoEntryAgain(ctx, fp) + } if !c.byEntries() { return fmt.Errorf("this engine reads no entry by fingerprint") } diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 02b63622..ed533851 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -163,7 +163,9 @@ type Composer struct { // OnEngine is told the node-engine's own module each time a composing finds it (design 47 G §527, §634); set before the // composer runs. OnEngine func(module string) - env Env + // AlsoEntryAgain is told each entry asked again by hand, so the shadow's own cache of entries asks it again too. + AlsoEntryAgain func(ctx context.Context, fp string) + env Env mu sync.Mutex wants map[string]meshwants.WantsByWantNodeModuleValue // by module, the node's own under agent.NodeModule diff --git a/internal/selfcompose/shadow.go b/internal/selfcompose/shadow.go index 1fff9c71..9c58e4c2 100644 --- a/internal/selfcompose/shadow.go +++ b/internal/selfcompose/shadow.go @@ -825,3 +825,9 @@ func (c *Composer) sameValue(a, b string) (same, known bool) { } return bytes.Equal(x, y), true } + +// EntryAgain asks fp's entry of the catalog half again in the shadow's own cache, where the node's own composer was asked +// again by hand: an entry the shadow holds as not found stays so until it is asked (a build the half had not yet taken in). +func (s *Shadow) EntryAgain(ctx context.Context, fp string) { + _ = s.c.EntryAgain(ctx, fp) // refuses only an entry the shadow keeps none of +} -- 2.54.0 From 321a0744d80acccc83fd2be96bde96373042de7b Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 23:13:57 +0100 Subject: [PATCH 55/59] H9 test: entry-again takes a shadow's not-found entry in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §698; lead answer A-319 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- internal/selfcompose/entries_test.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/internal/selfcompose/entries_test.go b/internal/selfcompose/entries_test.go index b7fb2ac1..24d32128 100644 --- a/internal/selfcompose/entries_test.go +++ b/internal/selfcompose/entries_test.go @@ -5,8 +5,10 @@ import ( "encoding/json" "errors" "fmt" + "runtime" "strings" "sync" + "sync/atomic" "testing" "time" @@ -128,6 +130,32 @@ func composeFromEntries(t *testing.T) { if !nf || nfOther { t.Errorf("entry-not-found told %v for the ghost and %v for the entry held", nf, nfOther) } + // An entry asked again by hand is asked again in the shadow's cache too (lead answer A-319): the shadow holds the ghost as not + // found, the half then holds it, and the node's own composer asked again takes it in on the shadow as well. + var late atomic.Bool + shadowEnv := b.env() + shadowEnv.Entry = EntryCall(func(ctx context.Context, seat, verb string, args any) (json.RawMessage, error) { + if in := args.(meshcatalog.EntryInput); in.Fingerprint == ghost && late.Load() { + return json.Marshal(meshcatalog.EntryAnswer{Entry: meshcatalog.CatalogEntry{Module: "ghost", Fp: ghost}}) + } + return ask(ctx, seat, verb, args) + }) + shadow := NewShadow(shadowEnv, nil) + shadow.c.entries.Need(ctx, ghost) + for deadline := time.Now().Add(10 * time.Second); shadow.c.entries.Rows() == nil || shadow.c.entries.Rows()[0].State != meshcatalogclient.EntryNotFound; { + if time.Now().After(deadline) { + t.Fatal("the shadow did not hold the ghost as not found") + } + runtime.Gosched() + } + late.Store(true) + c.AlsoEntryAgain = shadow.EntryAgain + if err := c.EntryAgain(ctx, ghost); err != nil { + t.Fatal(err) + } + if _, state := shadow.c.entries.Need(ctx, ghost); state != meshcatalogclient.EntryAnswered { + t.Errorf("the shadow holds the ghost's entry as %s after entry-again, want answered", state) + } // Each entry and want it composes from is kept in the composition state before the composition that uses them. state.mu.Lock() _, hasEntry := state.kv[entryKey(fp)] -- 2.54.0 From 650910dc9e699192ea79083da8b03fcacdf2f1a8 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 23:33:31 +0100 Subject: [PATCH 56/59] R3-3: the node's own composer has no path without an entry call; its tests compose from the catalog half MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §628, §421, A-312, A-317 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- cmd/mesh-host/nodeagent_integration_test.go | 8 +- internal/selfcompose/composedstate.go | 49 ++---- internal/selfcompose/entries.go | 30 +--- internal/selfcompose/entries_test.go | 17 -- internal/selfcompose/selfcompose.go | 20 +-- internal/selfcompose/selfcompose_test.go | 164 +++++++++++--------- 6 files changed, 117 insertions(+), 171 deletions(-) diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index 933bbaac..be40a99a 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -490,7 +490,7 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { } return kv } - assignment, modules, manifests := kvOf(buckets.Assignments), kvOf(buckets.Modules), kvOf(buckets.Manifests) + assignment, manifests := kvOf(buckets.Assignments), kvOf(buckets.Manifests) resolved := kvOf(selfcompose.ResolvedBucket) candidates := kvOf(buckets.ProposalCandidates) packages := kvOf(link.PackagesBucket) @@ -502,7 +502,6 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { mraw, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "web", Manifest: man}) mkey, _ := meshcatalogclient.ManifestsKey(digest) fp := fpIt("a") - modv, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40)}) place, _ := meshmoduleassignerclient.EncodeAssignments(meshmoduleassigner.Assignments{Node: itNode, Modules: []meshmoduleassigner.PlacedModule{{Module: "web"}}}) rec, _ := nec.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: itNode, Capabilities: []string{"service-manager", "package-manager"}}}) // it1 in one key per module and node (research 072 row 37), it2 in the form before: a node reads both. @@ -516,7 +515,6 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { mdigest := meshcatalogclient.ManifestDigest([]byte(mailer)) mlraw, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "mailer", Manifest: mailer}) mlkey, _ := meshcatalogclient.ManifestsKey(mdigest) - mlmod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fpIt("b"), Manifest: mdigest, Repository: "novox/mailer", Commit: strings.Repeat("2", 40)}) places2, _ := meshmoduleassignerclient.AssignmentsValues(meshmoduleassigner.Assignments{Node: "it2", Modules: []meshmoduleassigner.PlacedModule{{Module: "mailer"}}}) rec2, _ := nec.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "it2", Capabilities: []string{"service-manager"}}}) pkey2, _ := meshmoduleassignerclient.NodeKey("it2") @@ -525,9 +523,9 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { kv nats.KeyValue key string value []byte - }{{manifests, mkey, mraw}, {modules, "web", modv}, {b.haves, hkey, rec}, + }{{manifests, mkey, mraw}, {b.haves, hkey, rec}, {resolved, itNode + ".web.greeting", []byte(`{"value":"bonjour","source":"module"}`)}, - {manifests, mlkey, mlraw}, {modules, "mailer", mlmod}, {b.haves, hkey2, rec2}, {assignment, pkey2, places2[pkey2]}, {assignment, "it2.mailer", places2["it2.mailer"]}, + {manifests, mlkey, mlraw}, {b.haves, hkey2, rec2}, {assignment, pkey2, places2[pkey2]}, {assignment, "it2.mailer", places2["it2.mailer"]}, {resolved, "it2.mailer.domain", []byte(`{"value":"example.org","source":"module"}`)}} { if _, err := w.kv.Put(w.key, w.value); err != nil { t.Fatal(err) diff --git a/internal/selfcompose/composedstate.go b/internal/selfcompose/composedstate.go index da0eec31..61180a01 100644 --- a/internal/selfcompose/composedstate.go +++ b/internal/selfcompose/composedstate.go @@ -89,47 +89,20 @@ func (c *Composer) publishComposed(ctx context.Context) { // is left out. A module whose newest build the builder holds failed is also returned in failed, by module, with the // indexer's fingerprint and the build's error: it is published as a refusal, not composed. func (c *Composer) newestFPs(ctx context.Context, at *published) (out map[string]string, failed map[string][2]string) { - if c.byEntries() { - // The newest build is the indexer's (and the builder's done key says it is built): neither is read, and the node - // follows no branch (design 47 G §421, §628), so the state holds the wanted builds alone. A placed module's newest - // build that ended is the last trunk end the builder said of it (NoteBuildEnded): a failed one is refused (A-293). - failed = map[string][2]string{} - c.mu.Lock() - if c.endsRead { - for _, module := range at.placed { - if e, ok := c.ends[module]; ok && e.Outcome == builds.BuildFailed { - failed[module] = [2]string{e.Fingerprint, e.Error} - } - } - } - c.mu.Unlock() - return maps.Clone(at.fps), failed - } - out = map[string]string{} + // The newest build is the indexer's (and the builder's done key says it is built): neither is read, and the node + // follows no branch (design 47 G §421, §628), so the state holds the wanted builds alone. A placed module's newest + // build that ended is the last trunk end the builder said of it (NoteBuildEnded): a failed one is refused (A-293). failed = map[string][2]string{} - r := reader{c: c} - for _, module := range at.placed { - c.mu.Lock() - fp2 := c.modules[module].Fp2 - c.mu.Unlock() - if fp2 != "" { - if b, err := r.Build(ctx, module, fp2); err == nil && b != nil { - switch b.State { - case builds.BuildDone: - out[module] = fp2 - continue - case builds.BuildFailed: - failed[module] = [2]string{fp2, b.Error} - } + c.mu.Lock() + if c.endsRead { + for _, module := range at.placed { + if e, ok := c.ends[module]; ok && e.Outcome == builds.BuildFailed { + failed[module] = [2]string{e.Fingerprint, e.Error} } } - if fp, ok := at.fps[module]; ok { - out[module] = fp - } else if fp, ok := at.ran[module]; ok { - out[module] = fp - } } - return out, failed + c.mu.Unlock() + return maps.Clone(at.fps), failed } // writeComposed works out the state's keys from at and writes those that differ, then deletes what is past keeping. @@ -291,7 +264,7 @@ func (c *Composer) composedValues(ctx context.Context, at *published) (map[strin func (c *Composer) endedRefusals(values map[string][]byte) []string { c.mu.Lock() defer c.mu.Unlock() - if !c.byEntries() || !c.endsRead { + if !c.endsRead { return nil } var out []string diff --git a/internal/selfcompose/entries.go b/internal/selfcompose/entries.go index bcc6e610..0af82b00 100644 --- a/internal/selfcompose/entries.go +++ b/internal/selfcompose/entries.go @@ -66,10 +66,6 @@ type WantRow struct { KeptSince time.Time `json:"kept-since"` } -// byEntries says the composer reads the catalog half by fingerprint (the node's own composer), as the shadow, which keeps -// its reads of the indexer until it goes with the checker (§628), does not. -func (c *Composer) byEntries() bool { return c.entries != nil } - // newEntries makes the composer's cache of entries over env's call of `entry`. func (c *Composer) newEntries() { if c.env.Entry == nil { @@ -96,7 +92,7 @@ func wantStateKey(module string) string { return keptWantPrefix + module } // half (§683). A want read stands only until the live read of the wants answers (endKeptWants). A state that cannot be read // is said, and what it would have kept is asked of `entry` and read from the wants as they come. func (c *Composer) loadKept(ctx context.Context) { - if !c.byEntries() || c.env.ComposedState == nil { + if c.env.ComposedState == nil { return } kept, err := c.env.ComposedState.Entries(ctx) @@ -136,9 +132,6 @@ func (c *Composer) loadKept(ctx context.Context) { // says are the ones the node composes from, never a want older than its stream's (§683). A kept want the read did not // announce is gone. func (c *Composer) endKeptWants() { - if !c.byEntries() { - return - } c.mu.Lock() defer c.mu.Unlock() for module := range c.keptWants { @@ -252,9 +245,6 @@ func (c *Composer) wantedBuilds() map[string]bool { // composes from, and lays the picture frombus reads (c.modules, c.manifests) from the entries it holds. It runs at the start of // each pass. func (c *Composer) syncEntries(ctx context.Context) { - if !c.byEntries() { - return - } need := c.wantedBuilds() for fp := range need { c.entryOf(fp) @@ -393,9 +383,6 @@ func entryManifest(e meshcatalog.CatalogEntry) (digest string, raw []byte, err e // entryHolds is, of each module this node's want names, why its entry is not held: asked and not answered, or answered not // found. It is the phase held with a detail opening entry-asked: or entry-missing: and naming the fingerprint (§421, §614). func (c *Composer) entryHolds() map[string]string { - if !c.byEntries() { - return nil - } c.mu.Lock() wants := map[string]meshwants.WantsByWantNodeModuleValue{} for module, w := range c.wants { @@ -476,9 +463,6 @@ func (c *Composer) buildOf(module, fp string) *builds.BuilderBuild { // follows no branch), the entries it keeps and the wants it composes from. func (c *Composer) FirstViews() (views []map[string]any, entries []EntryRow, wants []WantRow) { views, entries, wants = []map[string]any{}, []EntryRow{}, []WantRow{} - if !c.byEntries() { - return - } seen := map[string]bool{} for _, r := range c.entries.Rows() { seen[r.Fingerprint] = true @@ -508,9 +492,6 @@ func (c *Composer) EntryAgain(ctx context.Context, fp string) error { if c.AlsoEntryAgain != nil { c.AlsoEntryAgain(ctx, fp) } - if !c.byEntries() { - return fmt.Errorf("this engine reads no entry by fingerprint") - } c.mu.Lock() _, kept := c.keptEnt[fp] delete(c.keptEnt, fp) @@ -554,9 +535,6 @@ const noEntryWords = "no entry of fingerprint" // to, each read from the entry the composer holds of its fingerprint (design 47 G §296, §297). An entry not held is an error: the // node agent then says it is not known and the move goes on. func (c *Composer) FormChanges(_ context.Context, module, runningFp, appliedFp string) ([]catalogue.FormChange, error) { - if !c.byEntries() { - return nil, fmt.Errorf("this engine reads no entry by fingerprint") - } read := func(fp string) (catalogue.Manifest, error) { e, state := c.entryOf(fp) if state != meshcatalogclient.EntryAnswered { @@ -583,9 +561,6 @@ func (c *Composer) FormChanges(_ context.Context, module, runningFp, appliedFp s // c.manifests from the entries held, and says whether an entry asked is not yet answered: the shadow's picture of the catalog, // where it once watched the indexer's modules (design 47 G §421, A-308). func (c *Composer) syncShadowEntries(ctx context.Context) (pending bool) { - if !c.byEntries() { - return false - } c.syncEntries(ctx) c.mu.Lock() defer c.mu.Unlock() @@ -651,9 +626,6 @@ func (c *Composer) addedModule(ctx context.Context, module string) (fp string, e // and says the first not yet answered: the shadow composes a candidate from entries alone (§421, §628), and an answer wakes it. The // fingerprints are kept beside the candidates' added ones while a proposal stands (addedFps). func (c *Composer) askBuildsEntries(fps map[string]string) (pending string) { - if !c.byEntries() { - return "" - } for _, module := range slices.Sorted(maps.Keys(fps)) { fp := fps[module] if !strings.HasPrefix(fp, meshcatalogclient.SourcePrefix2) { diff --git a/internal/selfcompose/entries_test.go b/internal/selfcompose/entries_test.go index 24d32128..6045d43f 100644 --- a/internal/selfcompose/entries_test.go +++ b/internal/selfcompose/entries_test.go @@ -212,23 +212,6 @@ func composeFromEntries(t *testing.T) { } } -// takeModule lays one module of the catalog half's picture as the composer's own lay of an entry does (layModules): the tests -// that compose from a picture they set read no indexer. -func (c *Composer) takeModule(u agent.Update) { - name, err := meshcatalogclient.ParseModulesKey(u.Key) - if err != nil { - return - } - c.mu.Lock() - defer c.mu.Unlock() - m, err := meshcatalogclient.DecodeModule(u.Value) - if u.Deleted || err != nil { - delete(c.modules, name) - return - } - c.modules[name] = m -} - // catalogHalf is the catalog half's entries a test composes from, by fingerprint, answered through the seat mesh-catalog's verbs // entry and module (design 47 G §421, A-312): the node-engine reads no indexer. type catalogHalf map[string]meshcatalog.CatalogEntry diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index ed533851..d8a65be1 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -134,7 +134,7 @@ type Env struct { SeatListChanged <-chan struct{} // Entry asks the seat mesh-catalog's verb `entry` of a fingerprint (meshcatalogclient.EntryVerb), in the engine's own name. // Set, the composer reads the catalog half by the fingerprints its wants and the runnings it heard name and nothing else of - // the indexer's or the builder's states (design 47 G §421, §628); nil reads the indexer's states, as the shadow does. + // the indexer's or the builder's states (design 47 G §421, §628); a composer is made with it. Entry meshcatalogclient.Call } @@ -187,7 +187,7 @@ type Composer struct { notTaken map[string]string // unmadeSaid is, by module, the stable part of the reason its last journal line said (without where the picture stands). unmadeSaid map[string]string - // entries is the cache of entries asked by fingerprint, nil where the composer reads the indexer's states; keptEnt the entries + // entries is the cache of entries asked by fingerprint; keptEnt the entries // held from the composition state or an answer, keptWantVal the wants it keeps there and keptWants the modules whose want // stands from that state alone until the live read of the wants answers (entries.go). Under mu; keptWrite orders the writes. entries *meshcatalogclient.Entries @@ -303,7 +303,7 @@ type Pass struct { // composed at (frombus.Scope.Holds), for LastHeld's words. holds map[string]compose.Hold // Unmade is each module wanted that the pass composed no composition of at its want's fp, with why: the want not - // taken, the module not assigned here, no manifest in the composer's picture of the indexer's modules (and where that + // taken, the module not assigned here, no entry of its build in the composer's picture of the catalog half (and where that // picture stands), or what the composition left it out for. A module held alone is in Held, not here. Said once per // change as a journal line, so a want that waits for the composer says what for. Unmade map[string]string `json:"unmade,omitempty"` @@ -1557,7 +1557,7 @@ func (r reader) Manifest(ctx context.Context, digest string) ([]byte, error) { if ok { return raw, nil } - if r.c.byEntries() && !r.c.forCandidates { + if !r.c.forCandidates { return nil, fmt.Errorf("the catalog half's entries hold no manifest %s", digest) } key, err := meshcatalogclient.ManifestsKey(digest) @@ -2195,7 +2195,7 @@ func (c *Composer) unmadeOf(fps map[string]string, comps map[string]engcomp.Comp continue } if made[module] == fp { - // Composed, but from the indexer's manifest and not a build: said, since what it names no artifacts of is no module + // Composed, but from the entry's manifest and not a build: said, since what it names no artifacts of is no module // that runs. if w, ok := fromManifest[module]; ok { out[module], keys[module] = w, fp+" "+w @@ -2211,18 +2211,18 @@ func (c *Composer) unmadeOf(fps map[string]string, comps map[string]engcomp.Comp why = fmt.Sprintf("the seat's assignment places no %s on this node (%d modules placed)", module, len(own.Modules)) key = "not placed" case !in: - why = fmt.Sprintf("no manifest in the composer's picture of the indexer's modules (%d modules)", + why = fmt.Sprintf("no entry of its build in the composer's picture of the catalog half (%d modules)", len(c.modules)) key = "no manifest" case slices.Contains(skipped, module): - why = "the composition skipped it: the declaration gives it no fingerprint (indexer's picture: " + m.Fp2 + ")" + why = "the composition skipped it: the declaration gives it no fingerprint (the composer's picture: " + m.Fp2 + ")" key = why default: if at := made[module]; at != "" { - why = fmt.Sprintf("the pass composed it at %s and the want names %s (indexer's picture: %s)", at, fp, m.Fp2) + why = fmt.Sprintf("the pass composed it at %s and the want names %s (the composer's picture: %s)", at, fp, m.Fp2) key = "composed at " + at } else { - why = fmt.Sprintf("the composition made nothing of it at %s (indexer's picture: %s)", fp, m.Fp2) + why = fmt.Sprintf("the composition made nothing of it at %s (the composer's picture: %s)", fp, m.Fp2) key = "made nothing at " + m.Fp2 } } @@ -2444,7 +2444,7 @@ func (c *Composer) EngineModule() string { // stands says the want of a module stands for its placement in the early pass of a start: its want is a kept one the live read has // not answered, and the entry of its build is kept (design 47 G A-117). func (c *Composer) stands(module string, w meshwants.WantsByWantNodeModuleValue) bool { - if !c.byEntries() || !c.keptWants[module] { + if !c.keptWants[module] { return false } k, ok := c.keptEnt[w.Fp] diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index 0620df94..78830f34 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -53,6 +53,7 @@ type fakeBus struct { kv map[string][]byte // bucket/key said []string reads [][2]string // bucket, pattern or key: every watch and get + half catalogHalf // the catalog half the bus's composers ask the entry of } func newFakeBus() *fakeBus { @@ -60,7 +61,7 @@ func newFakeBus() *fakeBus { } func (b *fakeBus) env() Env { - return Env{Node: "n1", + return Env{Node: "n1", Entry: b.half.call(), // the catalog half answers what the test's bus holds of it, nothing until a test sets it Watch: func(_ context.Context, bucket, pattern string) (<-chan agent.Update, error) { b.mu.Lock() defer b.mu.Unlock() @@ -163,10 +164,24 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { man, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "lemurs", Manifest: lemurs}) key, _ := meshcatalogclient.ManifestsKey(digest) b.kv[buckets.Manifests+"/"+key] = man + // The catalog half the node composes from, by fingerprint: the node's own composer asks it and reads nothing of the indexer. + half := catalogHalf{} + half.add(t, "lemurs", fp, lemurs) + oldieFp := meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64) + toolFp := meshcatalogclient.SourcePrefix2 + strings.Repeat("7", 64) + keeperFp := meshcatalogclient.SourcePrefix2 + strings.Repeat("9", 64) + toolBody := `{"module":"tool","version":"1","capabilities":["systemd"],"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},` + + `"resources":[{"id":"service","type":"service","unit":"tool.service","state":"running","boot":"enabled"}]}` + half.add(t, "tool", toolFp, toolBody) + b.half = half + keeperBody := `{"module":"keeper","version":"1","own-secrets":{"password":"/var/lib/mesh/keeper/password"},` + + `"resources":[{"id":"state","type":"directory","path":"/var/lib/mesh/keeper","mode":"0700"}]}` + half.add(t, "keeper", keeperFp, keeperBody) ctx, cancel := context.WithCancel(context.Background()) defer cancel() env := b.env() + env.Entry = half.call() // The vault's own: keeper's password sealed to n1's key, sealed again (other bytes) at every ask. var asks, generation int sealedAt := map[string]int{} // each box handed, by the generation it opens to @@ -195,26 +210,22 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { // oldie is placed and no want names it (a module at a build of the old form, nats on novox): the node runs it inside // its own composition, and composes it there again, even before the agent has applied anything (Running empty). oldie := `{"module":"oldie","version":"1","capabilities":["systemd"],"resources":[{"id":"service","type":"service","unit":"oldie.service","state":"running","boot":"enabled"}]}` - odigest := meshcatalogclient.ManifestDigest([]byte(oldie)) - oman, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "oldie", Manifest: oldie}) - okey, _ := meshcatalogclient.ManifestsKey(odigest) - b.kv[buckets.Manifests+"/"+okey] = oman - omod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64), Manifest: odigest, Repository: "novox/o", Commit: "c"}) + half.add(t, "oldie", oldieFp, oldie) place := assignedValues(t, meshmoduleassigner.Assignments{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"}, Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}}}) have, _ := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}}) - mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"}) want, _ := bucketstest.Want(meshwants.WantsByWantNodeModuleValue{Fp: fp, Self: true}) nodeWant, _ := bucketstest.NodeWant(meshwants.WantsByWantNodeNodeValue{Self: true}) b.putAssigned(t, place) b.put(t, buckets.Have, "n1.node", have) oldHave, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Build: "build-1", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) b.put(t, buckets.Have, "n1.oldie", oldHave) - // The picture of the catalog half is set directly (no indexer is watched): lemurs's entry, which its want names, and oldie, - // which the node runs inside its own composition and no want names. - c.takeModule(agent.Update{Key: "lemurs", Value: mod}) - c.takeModule(agent.Update{Key: "oldie", Value: omod}) - c.holdEntry(t, "lemurs", fp, lemurs) + // The catalog half answers lemurs's entry, which its want names, and oldie's, which the node runs inside its own composition and + // no want names. + c.NoteRunning(nodeengine.Running{Node: "n1", Module: "oldie", Fingerprint: oldieFp}) + c.RunningRead() + c.holdEntry(t, "lemurs", fp, lemurs) // held at the start, as the composition state keeps it, so the first pass composes + c.holdEntry(t, "oldie", oldieFp, oldie) b.put(t, buckets.Wants, "want.n1.lemurs", want) b.put(t, buckets.Wants, "want.n1.node", nodeWant) for _, bucket := range []string{buckets.Wants, buckets.Assignments, buckets.Have, ResolvedBucket} { @@ -289,18 +300,9 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { // Issue 1966 (novox, 2026-11-09): every pass names each wanted module it composed nothing of, or composed from the indexer's // manifest for want of a done build, with why, and a want it did not take; said once; the agent's word that it waits // hands a composition the node believes handed again. - tfp := meshcatalogclient.SourcePrefix2 + strings.Repeat("7", 64) - tool := `{"module":"tool","version":"1","capabilities":["systemd"],"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},` + - `"resources":[{"id":"service","type":"service","unit":"tool.service","state":"running","boot":"enabled"}]}` - tdigest := meshcatalogclient.ManifestDigest([]byte(tool)) - tman, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "tool", Manifest: tool}) - tkey, _ := meshcatalogclient.ManifestsKey(tdigest) - b.mu.Lock() - b.kv[buckets.Manifests+"/"+tkey] = tman - b.mu.Unlock() - tmod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: tfp, Manifest: tdigest, Repository: "novox/t", Commit: "c"}) - c.takeModule(agent.Update{Key: "tool", Value: tmod, Revision: 9}) - for module, w := range map[string]meshwants.WantsByWantNodeModuleValue{"tool": {Fp: tfp, Self: true}, "ghost": {Fp: fp, Self: true}, "nofp": {Self: true}} { + tfp, ghostFp := toolFp, meshcatalogclient.SourcePrefix2+strings.Repeat("8", 64) + c.holdEntry(t, "tool", tfp, toolBody) + for module, w := range map[string]meshwants.WantsByWantNodeModuleValue{"tool": {Fp: tfp, Self: true}, "ghost": {Fp: ghostFp, Self: true}, "nofp": {Self: true}} { raw, _ := bucketstest.Want(w) c.takeWant(agent.Update{Key: "want.n1." + module, Value: raw}) } @@ -320,22 +322,20 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { return n } c.compose(ctx) - why := map[string]string{"tool": c.WhyUnmade("tool"), "ghost": c.WhyUnmade("ghost"), "nofp": c.WhyUnmade("nofp"), "lemurs": c.WhyUnmade("lemurs")} - if !strings.Contains(why["tool"], "the builder holds no done build of it at "+tfp) || !strings.Contains(why["ghost"], "places no ghost") || + saidOfNofp := said("the node composes nothing of nofp") + why := map[string]string{"tool": c.LastHeld()["tool"], "ghost": c.LastHeld()["ghost"], "nofp": c.WhyUnmade("nofp"), "lemurs": c.WhyUnmade("lemurs")} + if !strings.Contains(why["tool"], "tool at "+tfp) || !strings.Contains(why["ghost"], "entry-asked: the entry of the build "+ghostFp) || !strings.Contains(why["nofp"], "want not taken: its want names no fingerprint") || why["lemurs"] != "" { t.Errorf("the pass says it composed nothing of %q", why) } - // An update of another module moves the picture's revision: the same reasons are not said again. - c.takeModule(agent.Update{Key: "lemurs", Value: mod, Revision: 10}) c.compose(ctx) - if n := said("the node composes nothing of ghost"); n != 1 { - t.Errorf("the journal says why ghost is not composed %d times, want once", n) + if n := said("the node composes nothing of nofp"); n != saidOfNofp || n == 0 { + t.Errorf("the journal says why nofp is not composed %d times after the second pass, %d after the first: want the same, not none", n, saidOfNofp) } // The state of this step is taken away again, and the pass that takes back what it handed is made here. for _, module := range []string{"tool", "ghost", "nofp"} { c.takeWant(agent.Update{Key: "want.n1." + module, Deleted: true}) } - c.takeModule(agent.Update{Key: "tool", Deleted: true}) c.mu.Lock() placed.Modules = placed.Modules[:len(placed.Modules)-1] c.assignments["n1"] = placed @@ -365,18 +365,13 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { t.Errorf("a build that cannot be composed was composed today: %s", dig) } chgJSON := strings.Replace(lemurs, `"state":"running"`, `"state":"stopped"`, 1) - chgDigest := meshcatalogclient.ManifestDigest([]byte(chgJSON)) - chgMan, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "lemurs", Manifest: chgJSON}) - chgKey, _ := meshcatalogclient.ManifestsKey(chgDigest) - b.mu.Lock() - b.kv[buckets.Manifests+"/"+chgKey] = chgMan - b.mu.Unlock() - chgMod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: chgDigest, Repository: "novox/x", Commit: "c"}) - c.takeModule(agent.Update{Key: "lemurs", Value: chgMod}) + c.holdEntry(t, "lemurs", fp, chgJSON) + c.layModules() if dig, ok := c.ComposedToday(ctx, "lemurs", fp); !ok || dig == handedDig { t.Errorf("composed today with a changed input: %s %v, want another digest than %s", dig, ok, handedDig) } - c.takeModule(agent.Update{Key: "lemurs", Value: mod}) + c.holdEntry(t, "lemurs", fp, lemurs) + c.layModules() // Issue 1126, the switch made safe: a bus not answering for a moment (a reload of its accounts) is tried again and // refuses nothing; a pass whose run ended refuses nothing either. @@ -405,6 +400,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { for k, v := range c.modules { c2.modules[k] = v } + maps.Copy(c2.keptEnt, c.keptEnt) c.mu.Unlock() return c2, &refusals } @@ -431,7 +427,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { b.mu.Unlock() failed := 0 pctx, pcancel := context.WithCancel(ctx) - c2, refusals := fresh(func(gctx context.Context, bucket, key string) ([]byte, error) { + get := func(gctx context.Context, bucket, key string) ([]byte, error) { if failed < row.failures { failed++ if row.cancel { @@ -447,7 +443,20 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { b.mu.Lock() defer b.mu.Unlock() return b.kv[bucket+"/"+key], nil - }) + } + c2, refusals := fresh(get) + // What fails is the vault's own, asked for keeper's secret through the same read. + c2.holdEntry(t, "keeper", keeperFp, keeperBody) + c2.wants["keeper"] = meshwants.WantsByWantNodeModuleValue{Fp: keeperFp, Self: true} + pk := c2.assignments["n1"] + pk.Modules = append(slices.Clone(pk.Modules), meshmoduleassigner.PlacedModule{Module: "keeper"}) + c2.assignments["n1"] = pk + c2.env.Own = func(octx context.Context, _ map[string]any) (json.RawMessage, error) { + if _, err := get(octx, buckets.Wants, "own"); err != nil { + return nil, fmt.Errorf("the vault's own could not be asked: %w", err) + } + return json.RawMessage(`{"node":"n1","key":"sk","own":[{"module":"keeper","name":"password","origin":"made","generation":1,"sealed":{"a":1}}],"handed":{"keeper":false}}`), nil + } handed := c2.compose(pctx) pcancel() tries := 1 + len(composeAgain) // a bus paused for a moment is tried again at once; a timeout or a refusal is not @@ -503,15 +512,16 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { `{"id":"pkg","type":"package","package":"haproxy"}]}` c3, told := fresh(env.Get) for m, raw := range map[string]string{"de-spiegel": string(spiegel), "proxy": proxy} { - dig := meshcatalogclient.ManifestDigest([]byte(raw)) - enc, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: m, Manifest: raw}) - k, _ := meshcatalogclient.ManifestsKey(dig) - b.mu.Lock() - b.kv[buckets.Manifests+"/"+k] = enc - b.mu.Unlock() mfp := meshcatalogclient.SourcePrefix2 + strings.Repeat(map[string]string{"de-spiegel": "d", "proxy": "e"}[m], 64) - c3.modules[m] = meshcatalog.ModulesByModuleValue{Fp2: mfp, Manifest: dig, Repository: "novox/" + m, Commit: "c"} - c3.wants[m] = meshwants.WantsByWantNodeModuleValue{Fp: mfp, Self: true} + c3.holdEntry(t, m, mfp, raw) + w := meshwants.WantsByWantNodeModuleValue{Fp: mfp, Self: true} + if m == "de-spiegel" { // builds an image: the entry names it, the want the digest its part has + k := c3.keptEnt[mfp] + k.entry.Artifacts = []meshcatalog.Artifact{{Kind: "image", Ref: "de-spiegel:site"}} + c3.keptEnt[mfp] = k + w.Parts = map[string]string{"de-spiegel:site": "sha256:" + strings.Repeat("6", 64)} + } + c3.wants[m] = w } p3 := c3.assignments["n1"] p3.Modules = append(append([]meshmoduleassigner.PlacedModule{}, p3.Modules...), meshmoduleassigner.PlacedModule{Module: "proxy"}, meshmoduleassigner.PlacedModule{Module: "de-spiegel"}) @@ -556,16 +566,16 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { t.Fatal(err) } c4, told4 := fresh(env.Get) - powerDig := meshcatalogclient.ManifestDigest(power) - powerEnc, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "power", Manifest: string(power)}) - powerKey, _ := meshcatalogclient.ManifestsKey(powerDig) - b.mu.Lock() - b.kv[buckets.Manifests+"/"+powerKey] = powerEnc - b.mu.Unlock() powerFP := meshcatalogclient.SourcePrefix2 + strings.Repeat("c", 64) - c4.modules["power"] = meshcatalog.ModulesByModuleValue{Fp2: powerFP, Manifest: powerDig, Repository: "novox/mesh-catalog", Commit: "ca8d6f33", - // The indexer's verdict on the captured manifest (ADR 0578): the engine judges nothing, it takes this. - Verdict: meshcatalogclient.VerdictRefused, Problems: []string{`bar item "shows" is none of them`}} + powerHalf := catalogHalf{} + powerHalf.add(t, "power", powerFP, string(power)) + powerEntry := powerHalf[powerFP] + // The verdict the catalog half's entry carries on the captured manifest (ADR 0578): the engine judges nothing, it takes this. + powerEntry.Repository, powerEntry.Commit, powerEntry.Verdict = "novox/mesh-catalog", "ca8d6f33", meshcatalogclient.VerdictRefused + powerEntry.Inputs = &meshcatalog.EntryInputs{Problems: []string{`bar item "shows" is none of them`}} + c4.mu.Lock() + c4.keptEnt[powerFP] = keptEntry{entry: powerEntry} + c4.mu.Unlock() c4.wants["power"] = meshwants.WantsByWantNodeModuleValue{Fp: powerFP, Self: true} p4 := c4.assignments["n1"] p4.Modules = append(append([]meshmoduleassigner.PlacedModule{}, p4.Modules...), meshmoduleassigner.PlacedModule{Module: "power"}) @@ -735,7 +745,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { b.mu.Lock() n := 0 for _, s := range b.said { - if strings.Contains(s, "lemurs is wanted at "+gone) { + if strings.Contains(s, "lemurs: entry-missing: the catalog half holds no entry of the build "+gone) { n++ } } @@ -750,20 +760,16 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { if u := next(t, out); u.Key != lemursKey { t.Fatalf("lemurs wanted back handed %s", u.Key) } - keeper := `{"module":"keeper","version":"1","own-secrets":{"password":"/var/lib/mesh/keeper/password"},` + - `"resources":[{"id":"state","type":"directory","path":"/var/lib/mesh/keeper","mode":"0700"}]}` + keeper := keeperBody kdigest := meshcatalogclient.ManifestDigest([]byte(keeper)) kman, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: "keeper", Manifest: keeper}) kkey, _ := meshcatalogclient.ManifestsKey(kdigest) b.mu.Lock() b.kv[buckets.Manifests+"/"+kkey] = kman b.mu.Unlock() - kfp9 := meshcatalogclient.SourcePrefix2 + strings.Repeat("9", 64) - kmod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: kfp9, Manifest: kdigest, Repository: "novox/k", Commit: "c"}) + kfp9 := keeperFp both := assignedValues(t, meshmoduleassigner.Assignments{Node: "n1", Decided: meshmoduleassigner.NodeDecided{Hub: true, Address: "10.77.0.1"}, Modules: []meshmoduleassigner.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}, {Module: "keeper"}}}) - c.takeModule(agent.Update{Key: "keeper", Value: kmod}) - c.holdEntry(t, "keeper", kfp9, keeper) // n1 is held in keys per module and node, which the writes below move. bothPlaced, _ := meshmoduleassignerclient.AssignmentsOf(both) bothNode := bothPlaced["n1"] @@ -884,6 +890,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { for k, v := range c.modules { again.modules[k] = v } + maps.Copy(again.keptEnt, c.keptEnt) c.mu.Unlock() var composed string for _, u := range again.compose(ctx) { @@ -1930,7 +1937,19 @@ func TestAnUnreadSeatListHoldsWhatIsNewAndHandsItAtTheRead(t *testing.T) { maps.Copy(c2.assignments, c.assignments) maps.Copy(c2.records, c.records) maps.Copy(c2.modules, c.modules) + maps.Copy(c2.keptEnt, c.keptEnt) c.mu.Unlock() + // What fails is the vault's own, asked for the secret of a module the pass composes (the modules' manifests come from entries). + kfp := meshcatalogclient.SourcePrefix2 + strings.Repeat("9", 64) + c2.holdEntry(t, "keeper", kfp, `{"module":"keeper","version":"1","own-secrets":{"password":"/var/lib/mesh/keeper/password"},`+ + `"resources":[{"id":"state","type":"directory","path":"/var/lib/mesh/keeper","mode":"0700"}]}`) + c2.wants["keeper"] = meshwants.WantsByWantNodeModuleValue{Fp: kfp, Self: true} + pk := c2.assignments["n1"] + pk.Modules = append(slices.Clone(pk.Modules), meshmoduleassigner.PlacedModule{Module: "keeper"}) + c2.assignments["n1"] = pk + c2.env.Own = func(context.Context, map[string]any) (json.RawMessage, error) { + return nil, errors.New("nats says ${dir:mesh-state}, and nats declares no directory") + } c2.compose(ctx) if line := fmt.Sprintf(WaitsForTheSeatListHeld, 2, "mesh-catalog, node-engine") + "; "; !strings.HasPrefix(refusal, line) { t.Errorf("a pass refused while the list is unread says %q, want it to begin %q", refusal, line) @@ -2090,6 +2109,8 @@ func TestTheHolderOfTheCatalogIsStartedFromTheOwnCompositionBeforeTheListIsReadA `"resources":[{"id":"service","type":"service","unit":"other.service","state":"running","boot":"enabled"},` + `{"id":"extra","type":"service","unit":"extra.service","state":"running","boot":"enabled"}]}` b := newFakeBus() + half := catalogHalf{} + b.half = half ctx, cancel := context.WithCancel(context.Background()) defer cancel() env := b.env() @@ -2107,12 +2128,11 @@ func TestTheHolderOfTheCatalogIsStartedFromTheOwnCompositionBeforeTheListIsReadA mods := map[string]string{"catalog": catalog, "other": other} var placed []meshmoduleassigner.PlacedModule for _, m := range []string{"catalog", "other"} { - digest := meshcatalogclient.ManifestDigest([]byte(mods[m])) - man, _ := meshcatalogclient.EncodeManifest(meshcatalog.ManifestsByDigestValue{Module: m, Manifest: mods[m]}) - mk, _ := meshcatalogclient.ManifestsKey(digest) - b.kv[buckets.Manifests+"/"+mk] = man - rec, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64), Manifest: digest, Repository: "novox/x", Commit: "c"}) - c.takeModule(agent.Update{Key: m, Value: rec}) // a module run inside the node's own composition: no want names it, so no entry is asked + mfp := meshcatalogclient.SourcePrefix2 + strings.Repeat(map[string]string{"catalog": "a", "other": "b"}[m], 64) + half.add(t, m, mfp, mods[m]) + // A module run inside the node's own composition: no want names it, the running heard does, so its entry is asked. + c.NoteRunning(nodeengine.Running{Node: "n1", Module: m, Fingerprint: mfp}) + c.holdEntry(t, m, mfp, mods[m]) hk, _ := nodeengineclient.HaveKey("n1", m) hv, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Build: "build-1", Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK}) b.put(t, buckets.Have, hk, hv) -- 2.54.0 From 8656e57eb96e859a75b287e345c2ae576f9c5173 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 23:39:39 +0100 Subject: [PATCH 57/59] Test 36: the recovery wiring is driven through the node agent by a revert want of db MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §296, §297, §554, §560, A-113, A-320 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- cmd/mesh-host/recovery_integration_test.go | 141 +++++++++++++++++++-- 1 file changed, 132 insertions(+), 9 deletions(-) diff --git a/cmd/mesh-host/recovery_integration_test.go b/cmd/mesh-host/recovery_integration_test.go index 2f665064..a45939ce 100644 --- a/cmd/mesh-host/recovery_integration_test.go +++ b/cmd/mesh-host/recovery_integration_test.go @@ -10,12 +10,24 @@ import ( "testing" "time" + "fmt" + "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient" + nec "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" + "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "git.novox.be/novox/mesh-contracts/go/seats/meshrecovery" + nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants" "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-host/internal/buckets" "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/nodegrant" + "github.com/novox/mesh-host/internal/selfcompose" + "strings" + "sync" + "sync/atomic" ) // Design 47 G §560, §630, A-77, A-92, A-108: the node agent's recovery step over a real bus. Its session asks the seat's verbs, hears @@ -50,15 +62,31 @@ func TestNatsTheRecoveryStepAsksHearsBoundsAndCatchesUpOverTheBus(t *testing.T) } } var restoreAsked meshrecovery.RestoreKeptInput + // The kept points the stand-in answers: the one of the first checks, then (below) what the agent's revert finds, and none. + var pointsMu sync.Mutex + points := []meshrecovery.Point{{Module: "db", Kept: "cache", Form: 1, Fingerprint: "f2", Snapshot: "snap-1", + CatchUp: &meshrecovery.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": true}}}} + // The restore the agent's revert asks waits for the stand-in to say it restored: gated until released. + var gated atomic.Bool + release := make(chan struct{}) + var restores atomic.Int32 for verb, answer := range map[string]func(m *nats.Msg) any{ meshrecovery.VerbKeptPoints: func(*nats.Msg) any { - return meshrecovery.KeptPointsAnswer{Points: []meshrecovery.Point{{Module: "db", Kept: "cache", Form: 1, Fingerprint: "f2", Snapshot: "snap-1", - CatchUp: &meshrecovery.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": true}}}}} + pointsMu.Lock() + defer pointsMu.Unlock() + return meshrecovery.KeptPointsAnswer{Points: points} }, meshrecovery.VerbRestoreKept: func(m *nats.Msg) any { + pointsMu.Lock() _ = json.Unmarshal(m.Data, &restoreAsked) - say("r1", meshrecovery.RestoredEventOutcomeDone) - return meshrecovery.RestoreKeptAnswer{Accepted: "r1"} + pointsMu.Unlock() + act := fmt.Sprintf("r%d", restores.Add(1)) + if gated.Load() { + go func() { <-release; say(act, meshrecovery.RestoredEventOutcomeDone) }() + } else { + say(act, meshrecovery.RestoredEventOutcomeDone) + } + return meshrecovery.RestoreKeptAnswer{Accepted: act} }, } { answer := answer @@ -72,8 +100,14 @@ func TestNatsTheRecoveryStepAsksHearsBoundsAndCatchesUpOverTheBus(t *testing.T) t.Cleanup(func() { _ = sub.Unsubscribe() }) } var toolArgs map[string]any + var toolCalls atomic.Int32 toolSub, err := b.conn.Subscribe(link.ToolSubject("db", "rebuild", itNode), func(m *nats.Msg) { - _ = json.Unmarshal(m.Data, &toolArgs) + var args map[string]any + _ = json.Unmarshal(m.Data, &args) + pointsMu.Lock() + toolArgs = args + pointsMu.Unlock() + toolCalls.Add(1) _ = m.Respond([]byte(`{"result":{}}`)) }) if err != nil { @@ -113,14 +147,21 @@ func TestNatsTheRecoveryStepAsksHearsBoundsAndCatchesUpOverTheBus(t *testing.T) t.Fatalf("kept-points: %+v, %v", pts, err) } got, err := r.RestoreKept(ctx, meshrecovery.RestoreKeptInput{Module: "db", Kept: "cache", Form: 1, Fingerprint: "f2", Snapshot: pts.Points[0].Snapshot, Why: "w"}) - if err != nil || got.Accepted != "r1" || restoreAsked.Snapshot != "snap-1" { - t.Fatalf("restore-kept: %+v, %v, asked %+v", got, err, restoreAsked) + pointsMu.Lock() + firstAsked := restoreAsked + pointsMu.Unlock() + if err != nil || got.Accepted != "r1" || firstAsked.Snapshot != "snap-1" { + t.Fatalf("restore-kept: %+v, %v, asked %+v", got, err, firstAsked) } if ev := <-events; ev.Act != "r1" || ev.Outcome != "done" { t.Fatalf("the ending heard is %+v", ev) } - if err := r.CatchUp(ctx, "db", *pts.Points[0].CatchUp); err != nil || toolArgs["all"] != true { - t.Fatalf("catch-up: %v, arguments %v", err, toolArgs) + err = r.CatchUp(ctx, "db", *pts.Points[0].CatchUp) + pointsMu.Lock() + gotArgs := toolArgs + pointsMu.Unlock() + if err != nil || gotArgs["all"] != true { + t.Fatalf("catch-up: %v, arguments %v", err, gotArgs) } expired, end, err := r.Bound(ctx, "recovery", "db.restored.cache", "db", time.Now(), time.Second) if err != nil { @@ -133,6 +174,88 @@ func TestNatsTheRecoveryStepAsksHearsBoundsAndCatchesUpOverTheBus(t *testing.T) t.Fatal("the bound's end, the bus's marker, was not heard") } + // Design 47 G §296, §297, §554, §560, A-113: the node agent whole drives a revert want of db through the same recovery wiring + // (recoveryEnv over the session and the node's composer): db runs at a build whose cache is of form 2, and its want goes back to a + // build of form 1. With no point of form 1 taken before the newer build, its have says the restore point is missing, naming it, + // and the newer build runs again. With the point, the have is held (backing up) until the stand-in says restored, then the + // older build is applied and the point's catch-up tool is called. + dbManifest := func(form int, content string) string { + return `{"module":"db","version":"1","state":[{"name":"cache","form":` + fmt.Sprint(form) + `}],"resources":[{"id":"conf","type":"file","path":"` + + filepath.Join(b.dir, "db.conf") + `","mode":"0644","content":"` + content + `\n"}]}` + } + fpOld, fpNew := fpIt("a"), fpIt("b") + b.registerEntry("db", fpOld, dbManifest(1, "v1")) + b.registerEntry("db", fpNew, dbManifest(2, "v2")) + for _, name := range []string{buckets.Manifests, selfcompose.ResolvedBucket, buckets.ProposalCandidates, link.PackagesBucket} { + if _, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: name, History: 5}); err != nil { + t.Fatal(err) + } + } + assignment, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: buckets.Assignments, History: 5}) + if err != nil { + t.Fatal(err) + } + places, _ := meshmoduleassignerclient.AssignmentsValues(meshmoduleassigner.Assignments{Node: itNode, Modules: []meshmoduleassigner.PlacedModule{{Module: "db"}}}) + hkey, _ := nec.HaveNodeKey(itNode) + rec, _ := nec.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: itNode, Capabilities: []string{"service-manager", "package-manager"}}}) + if _, err := b.haves.Put(hkey, rec); err != nil { + t.Fatal(err) + } + for key, value := range places { + if _, err := assignment.Put(key, value); err != nil { + t.Fatal(err) + } + } + b.want("db", fpNew, "", "") + stopAgent := b.start() + defer stopAgent() + within(t, 20*time.Second, "db does not run its newer build", func() bool { + h := b.have("db") + return h.Fp == fpNew && h.Phase == nec.PhaseRunning && b.file("db") == "v2\n" + }) + // The engine says what runs, as it does after each apply: the composer keeps the entry of the running build while the want moves on. + b.running(itNode, "db", fpNew, meshcatalogclient.ManifestDigest([]byte(dbManifest(2, "v2")))) + // The stand-in holds no point of form 1: the revert is held, the have names the point missing, the newer build runs. + pointsMu.Lock() + points = nil + pointsMu.Unlock() + b.want("db", fpOld, "", "") + within(t, 20*time.Second, "the have does not say the restore point is missing", func() bool { + h := b.have("db") + return h.Phase == nec.PhaseHeld && strings.Contains(h.Detail, "restore point missing") && + strings.Contains(h.Detail, "db's cache has none of form 1 taken before "+fpNew) + }) + if b.have("db").Fp != fpNew || b.file("db") != "v2\n" || restores.Load() != 1 { + t.Fatalf("the missing point left db %+v with %q, %d restore(s) asked", b.have("db"), b.file("db"), restores.Load()) + } + // The stand-in now holds the point: the want answered "again" asks the restore, and the have waits for its ending. + pointsMu.Lock() + points = []meshrecovery.Point{{Module: "db", Kept: "cache", Form: 1, Fingerprint: fpNew, Snapshot: "snap-2", At: time.Now(), + CatchUp: &meshrecovery.CatchUp{Tool: "rebuild", Arguments: map[string]any{"all": "again"}}}} + pointsMu.Unlock() + gated.Store(true) + calls := toolCalls.Load() + b.want("db", fpOld, "", meshwants.WantsByWantNodeModuleValueBackupAgain) + within(t, 20*time.Second, "the restore of the revert is not asked", func() bool { return restores.Load() == 2 }) + within(t, 20*time.Second, "the have does not wait for the restore", func() bool { return b.have("db").Phase == nec.PhaseBackingUp }) + time.Sleep(300 * time.Millisecond) // the older build must not be applied while the stand-in has not said restored + if h := b.have("db"); h.Phase != nec.PhaseBackingUp || b.file("db") != "v2\n" || toolCalls.Load() != calls { + t.Fatalf("before the stand-in says restored the have is %+v, the file %q, %d catch-up call(s) more", h, b.file("db"), toolCalls.Load()-calls) + } + close(release) + within(t, 20*time.Second, "the older build is not applied after the restore", func() bool { + h := b.have("db") + return h.Fp == fpOld && h.Phase == nec.PhaseRunning && b.file("db") == "v1\n" + }) + within(t, 20*time.Second, "the catch-up tool is not called after the restore", func() bool { return toolCalls.Load() == calls+1 }) + pointsMu.Lock() + asked, gotArgs := restoreAsked, toolArgs + pointsMu.Unlock() + if asked.Snapshot != "snap-2" || asked.Fingerprint != fpNew || gotArgs["all"] != "again" { + t.Errorf("the revert asked %+v and called the catch-up with %v", asked, gotArgs) + } + stopAgent() + // Design 47 G A-293: over the same session, the builder's build ends the composer follows: the stream read whole first (the // last end of each module and fingerprint, in the stream's order), then the read's end, then each end said after it. if _, err := b.js.AddStream(&nats.StreamConfig{Name: link.BuildEndsStream, Subjects: []string{link.BuildEndsSubject}, MaxMsgsPerSubject: 1}); err != nil { -- 2.54.0 From a12417fc13c77fa5d7d16e8e27e38289f3e58119 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 12 Nov 2026 23:44:02 +0100 Subject: [PATCH 58/59] Remove the real sleep from the recovery integration test Builds: design 47 G (playbook 07 step 2: no real sleeping in a test) Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- cmd/mesh-host/recovery_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/mesh-host/recovery_integration_test.go b/cmd/mesh-host/recovery_integration_test.go index a45939ce..c9def26a 100644 --- a/cmd/mesh-host/recovery_integration_test.go +++ b/cmd/mesh-host/recovery_integration_test.go @@ -238,7 +238,7 @@ func TestNatsTheRecoveryStepAsksHearsBoundsAndCatchesUpOverTheBus(t *testing.T) b.want("db", fpOld, "", meshwants.WantsByWantNodeModuleValueBackupAgain) within(t, 20*time.Second, "the restore of the revert is not asked", func() bool { return restores.Load() == 2 }) within(t, 20*time.Second, "the have does not wait for the restore", func() bool { return b.have("db").Phase == nec.PhaseBackingUp }) - time.Sleep(300 * time.Millisecond) // the older build must not be applied while the stand-in has not said restored + // The restored event is held until close(release): the older build must not be applied before the stand-in says restored. if h := b.have("db"); h.Phase != nec.PhaseBackingUp || b.file("db") != "v2\n" || toolCalls.Load() != calls { t.Fatalf("before the stand-in says restored the have is %+v, the file %q, %d catch-up call(s) more", h, b.file("db"), toolCalls.Load()-calls) } -- 2.54.0 From 5caa760f5eaa62537d90716ce8217f72510a73d4 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 13 Nov 2026 09:20:46 +0100 Subject: [PATCH 59/59] A-342 engine-node-env: every process runs with its node's name as MESH_NODE The apply gives each process the declaration's For as MESH_NODE beside its own env, so a process that reads MESH_NODE (mesh-sdk, the zfs follower's writer and its node-service-manager calls) names its node. Builds: design 47 G A-342 Claude-Session: https://claude.ai/code/session_01NcbgfGKaxw984XyAPfLm4h --- internal/apply/apply.go | 7 +++++-- internal/apply/filled.go | 2 +- internal/apply/process.go | 19 +++++++++++++++++++ internal/apply/process_test.go | 7 +++++++ 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/internal/apply/apply.go b/internal/apply/apply.go index a1ff719a..f6efe674 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -785,7 +785,7 @@ func applyScoped( // change one apply late, and never misses it. // And one patience with the artifact store for the whole apply (novox/hq issue 291): a fetch it // does not answer during a maintenance window waits for the window instead of failing. - in := inputs{declares: map[string]string{}, known: &known, windows: windows, away: newStoreAway(windows, log), + in := inputs{node: d.For, declares: map[string]string{}, known: &known, windows: windows, away: newStoreAway(windows, log), groups: wanted, handed: handed} // Where nothing is placed or mounted from, judged before anything is applied (novox/hq issue 339), and // which declared directories were already here: one this apply makes is the mesh's, one found may not be. @@ -1625,7 +1625,7 @@ func applyOne(ctx context.Context, sys system.System, r declaration.Resource, ru if err != nil { return begin(r), err } - return applyProcess(ctx, res, run, changed, previous, held, in.away) + return applyProcess(ctx, withNode(res, in.node), run, changed, previous, held, in.away) case *declaration.Action: return applyAction(ctx, res, run) case *declaration.Network: @@ -2816,6 +2816,9 @@ type inputs struct { // fetched is every archive's and process's bytes this apply fetched before anything was replaced (fetchFirst), each // taken by its applier once (artifact). fetched map[string]fetchedArtifact + // node is the node's own name, the declaration's For: every process runs with it as MESH_NODE beside its own env + // (design 47 G A-342). Empty where the declaration names none (the first node's carried bundle). + node string } // fileDigest is what a file the container reads holds, by digest. diff --git a/internal/apply/filled.go b/internal/apply/filled.go index 6bd9b7a5..75cf15e6 100644 --- a/internal/apply/filled.go +++ b/internal/apply/filled.go @@ -95,7 +95,7 @@ func WriteFilledWith( "yet taken (%s), so nothing was written", path, held.Why)} } - in := inputs{declares: map[string]string{}, known: &known, windows: windows, away: newStoreAway(windows, log), + in := inputs{node: d.For, declares: map[string]string{}, known: &known, windows: windows, away: newStoreAway(windows, log), groups: groupsWanted(d.Resources), handed: Handed{}} in.refused = refusedPlaces(d.Resources) in.agentHomes = rootNeverHomes(d) diff --git a/internal/apply/process.go b/internal/apply/process.go index 7eb2a4d9..4d434ab0 100644 --- a/internal/apply/process.go +++ b/internal/apply/process.go @@ -274,6 +274,25 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner, return out, nil } +// NodeEnv is the variable every process the node-engine runs reads its node's name from (design 47 G A-342), the one +// mesh-sdk reads (MESH_NODE). +const NodeEnv = "MESH_NODE" + +// withNode is the process as it runs on node: its own env with NodeEnv beside it (design 47 G A-342). The validator +// refuses a resource that sets it, so the mesh's value is the only one; with no node named the process is as declared. +func withNode(r *declaration.Process, node string) *declaration.Process { + if node == "" { + return r + } + p := *r + p.Env = make(map[string]string, len(r.Env)+1) + for k, v := range r.Env { + p.Env[k] = v + } + p.Env[NodeEnv] = node + return &p +} + // unitFor is the unit the mesh writes for a daemon. // // **Generated whole and never edited in place**, the same rule as every other managed file: an diff --git a/internal/apply/process_test.go b/internal/apply/process_test.go index 146ce1b4..1cab4259 100644 --- a/internal/apply/process_test.go +++ b/internal/apply/process_test.go @@ -319,11 +319,17 @@ func TestAProcessAppliedAgainIsUnchangedAndNotRestarted(t *testing.T) { } d := declare(t, `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"`+serving(t, body)+ `","digest":"`+digest+`","run":["node","main.js"],"env":{"MESH_TOOL_MODULES":"a=/x/index.js"}}`) + d.For = "anchor" first, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, run, nil, nil) if err != nil { t.Fatal(err) } + // Design 47 G A-342: the process runs with its node's name as MESH_NODE beside its own env. + if b, err := os.ReadFile(filepath.Join(units, "node-tools.service")); err != nil || !strings.Contains(string(b), `Environment="MESH_NODE=anchor"`) || + !strings.Contains(string(b), `Environment="MESH_TOOL_MODULES=`) { + t.Errorf("the unit does not carry the node's name beside its env (%v):\n%s", err, b) + } if o := outcomeOf(first, "node-tools.runtime"); o.Action != "created" { t.Fatalf("first apply: %+v, want created", o) } @@ -360,6 +366,7 @@ func TestAProcessAppliedAgainIsUnchangedAndNotRestarted(t *testing.T) { // An input of its unit that differs restarts it, and the line says which (issue 1966). changed := declare(t, `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"`+serving(t, body)+ `","digest":"`+digest+`","run":["node","main.js"],"env":{"MESH_TOOL_MODULES":"a=/x/index.js,b=/y/index.js"}}`) + changed.For = "anchor" fourth, _, err := Apply(context.Background(), archHost(t), changed, state, store.OriginDeclared, run, nil, nil) if err != nil { t.Fatal(err) -- 2.54.0