From 45d034804439345ea36e85464db8023bc0b24695 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:04:13 +0100 Subject: [PATCH 01/21] Revert "Merge pull request 'Revert L3: its node-engine removed every held module on novox, the bus server among them (issue 774)' (#174) from revert/774-l3 into main" This reverts commit 1ec73915b8071bd3cd6ab53842125f299aceb331, reversing changes made to 0a4c35f731a7fe64683a9b06cec3e4f3ddc92d71. --- cmd/mesh-host/authority_test.go | 113 ++- cmd/mesh-host/engineseat.go | 277 +++++++ cmd/mesh-host/found.go | 47 +- cmd/mesh-host/found_test.go | 15 + cmd/mesh-host/libraries_test.go | 40 + cmd/mesh-host/main.go | 27 +- cmd/mesh-host/nodeagent.go | 56 +- cmd/mesh-host/nodeagent_integration_test.go | 3 +- cmd/mesh-host/trust.go | 317 ++++++++ cmd/mesh-host/units_test.go | 59 ++ cmd/mesh-host/whatruns_test.go | 37 + internal/accounts/setuid_cache.go | 11 + internal/link/access_act_test.go | 110 +++ internal/link/engineseat.go | 382 ++++++++++ internal/link/handacts.go | 769 ++++++++++++++++++++ internal/link/handover.go | 20 +- internal/link/handover_test.go | 119 ++- internal/link/have_test.go | 5 +- internal/link/hearing_nats.go | 65 +- internal/link/record.go | 4 +- internal/link/run.go | 21 + internal/link/session.go | 54 ++ internal/link/setuid_search_test.go | 78 ++ internal/selfcompose/selfcompose.go | 77 ++ internal/selfcompose/selfcompose_test.go | 27 + module.json | 6 + packaging/operator_keys_test.go | 9 + 27 files changed, 2689 insertions(+), 59 deletions(-) create mode 100644 cmd/mesh-host/engineseat.go create mode 100644 cmd/mesh-host/trust.go create mode 100644 internal/link/engineseat.go create mode 100644 internal/link/handacts.go diff --git a/cmd/mesh-host/authority_test.go b/cmd/mesh-host/authority_test.go index 48ce708..84d8ea9 100644 --- a/cmd/mesh-host/authority_test.go +++ b/cmd/mesh-host/authority_test.go @@ -1,16 +1,27 @@ package main import ( + "context" + "crypto/ed25519" + "crypto/x509" + "encoding/base64" "encoding/json" + "encoding/pem" "os" "path/filepath" "testing" + "time" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" + + "github.com/novox/mesh-host/internal/identity" ) // Issue 1126 (ADR 0555 §3): the mesh's own certificate a composition carries is kept for the machine's record, and the -// record's keys read it; a composition carrying none changes nothing. +// record's keys read it; a composition carrying none changes nothing. And (L3 s6, s7; ADR 0550 §2) the record says what +// the node trusts a mesh signature from, and the node keeps the certificate the vault signed for it. func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) { state := filepath.Join(t.TempDir(), "state", "state.json") res := func(id, content string) json.RawMessage { @@ -26,7 +37,105 @@ func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) { if err := keepAuthority(state, delivery.Composed{Resources: []json.RawMessage{res("ca-trust.certificate-authority", "CA-1")}}); err != nil { t.Fatal(err) } - if got := recordKeys(state)(); got.Authority != "CA-1" { + if got := recordKeys(state, nil)(); got.Authority != "CA-1" { t.Errorf("the record's keys read the authority %q", got.Authority) } + + // L3 s6: before the vault's signers are read the node trusts the controller's key alone; once read, a message the + // vault signed is accepted and the record names both; once the lease is handed away the controller's key is refused + // and the record names the vault's alone. Each change tells the record's writer. + controllerPub, controller, _ := ed25519.GenerateKey(nil) + vaultPub, vault, _ := ed25519.GenerateKey(nil) + trust := newMeshTrust(controllerPub) + told := 0 + trust.onChange(func() { told++ }) + keys := recordKeys(state, trust.Trusts) + ctrlFP, vaultFP := delivery.SignerFingerprint(controllerPub), delivery.SignerFingerprint(vaultPub) + if got := keys().Trusts; !delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || delivery.TrustsKey(got, delivery.SignerVault, vaultFP) { + t.Errorf("before the signers are read the record trusts %v", got) + } + enrol, _ := delivery.SignContext(delivery.KindEnrolment) + msg := []byte("a reply") + if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(vault, enrol, msg)); err == nil { + t.Error("a vault-signed message is accepted before the vault's key is read") + } + vaultCA := testAuthority(t, vaultPub, vault) + keyValue, _ := json.Marshal(delivery.NewSignerKey(vaultPub, time.Now())) + fp, _ := delivery.AuthorityFingerprint(vaultCA) + authValue, _ := json.Marshal(delivery.SignerAuthority{Certificate: vaultCA, Fingerprint: fp, Made: time.Now()}) + trust.heardSigner(delivery.SignersKeyKey, keyValue, false, func(string) {}) + trust.heardSigner(delivery.SignersAuthorityKey, authValue, false, func(string) {}) + if s, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(vault, enrol, msg)); err != nil || s.Name != delivery.SignerVault { + t.Errorf("a vault-signed message on a node that read signers: %+v, %v", s, err) + } + if got := keys().Trusts; !delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) || + trust.VaultAuthority() != vaultCA || told != 2 { + t.Errorf("after the signers are read the record trusts %v (told %d)", got, told) + } + handed, _ := json.Marshal(buscompose.LeaseHolder{HandedTo: "mesh-vault", HandedBy: "person.operator"}) + trust.heardLease(handed, true, nil) + if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(controller, enrol, msg)); err == nil || trust.Controller() != nil { + t.Error("after the lease is handed away the controller's key is still trusted") + } + if got := keys().Trusts; delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) || told != 3 { + t.Errorf("after the lease is handed away the record trusts %v (told %d)", got, told) + } + trust.heardLease(handed, true, nil) + if told != 3 { + t.Error("a lease heard again unchanged wrote the record again") + } + // A signers' key naming another fingerprint is not trusted. + bad, _ := json.Marshal(delivery.SignerKey{Key: base64.StdEncoding.EncodeToString(controllerPub), Fingerprint: vaultFP}) + trust.heardSigner(delivery.SignersKeyKey, bad, false, func(string) {}) + if !delivery.TrustsKey(keys().Trusts, delivery.SignerVault, vaultFP) { + t.Error("a signers' key that does not check replaced the vault's") + } + + // L3 s7: the certificate the vault signs for this node's internal name and serving key is kept where the record reads + // it, and the record is written again; one signed by another authority is refused and nothing is kept. + serving, err := identity.GenerateServingKey() + if err != nil { + t.Fatal(err) + } + if err := identity.WriteServingKey(identity.ServingKeyPath(state), serving); err != nil { + t.Fatal(err) + } + servingPub, _ := base64.StdEncoding.DecodeString(serving.Public) + signedBy := func(ca string, key ed25519.PrivateKey) string { + parent, _ := delivery.ParseCertificate(ca) + der, err := x509.CreateCertificate(nil, delivery.NodeCertificateTemplate(overlay.InternalName("n1"), servingPub, time.Now().Add(-time.Hour)), + parent, ed25519.PublicKey(servingPub), key) + if err != nil { + t.Fatal(err) + } + return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) + } + var asked []any + answer := signedBy(vaultCA, vault) + ask := func(_ context.Context, verb string, args any) (json.RawMessage, error) { + asked = append(asked, verb, args) + return json.Marshal(delivery.SignAnswer{Kind: delivery.KindNodeCertificate, Node: "n1", Certificate: answer}) + } + written := 0 + if err := certifyThisNode(context.Background(), state, "n1", trust, ask, func() { written++ }); err != nil { + t.Fatal(err) + } + if got := recordKeys(state, nil)(); got.Certificate != answer || written != 1 || len(asked) != 2 || asked[0] != delivery.SignVerb { + t.Errorf("the vault's certificate: kept %v, written %d, asked %v", got.Certificate == answer, written, asked) + } + answer = signedBy(testAuthority(t, controllerPub, controller), controller) + if err := certifyThisNode(context.Background(), state, "n1", trust, ask, func() { written++ }); err == nil || written != 1 { + t.Errorf("a certificate of another authority was kept: %v", err) + } +} + +// testAuthority is a self-signed authority certificate for key, as the vault makes its own. +func testAuthority(t *testing.T, pub ed25519.PublicKey, key ed25519.PrivateKey) string { + t.Helper() + tpl := delivery.AuthorityTemplate(pub, time.Now().Add(-time.Hour)) + der, err := x509.CreateCertificate(nil, tpl, tpl, pub, key) + if err != nil { + t.Fatal(err) + } + return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) } diff --git a/cmd/mesh-host/engineseat.go b/cmd/mesh-host/engineseat.go new file mode 100644 index 0000000..4c11ece --- /dev/null +++ b/cmd/mesh-host/engineseat.go @@ -0,0 +1,277 @@ +package main + +import ( + "bufio" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync/atomic" + "syscall" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + meshasker "git.novox.be/novox/mesh-sdk/go/asker" + "git.novox.be/novox/mesh-sdk/go/asks" + "github.com/novox/mesh-host/internal/accounts" + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/store" +) + +// The node seat node-engine's judgements on this machine (novox/hq ADR 0563): what a hand-over and a library's take +// would set and what the disk holds now, and whether this machine is root-free. + +// diskOf is what is on disk at a directory, as an act compares it when the operator's yes arrives: its owner and +// group, its mode, and which file it is. +func diskOf(info os.FileInfo) string { + owner := "unknown" + if uid, gid, ok := apply.OwnerOf(info); ok { + owner = fmt.Sprintf("%d:%d", uid, gid) + } + which := "" + if st, ok := info.Sys().(*syscall.Stat_t); ok { + which = fmt.Sprintf(" %v:%v", st.Dev, st.Ino) + } + return fmt.Sprintf("%s %o%s", owner, info.Mode().Perm(), which) +} + +// handOverJudgeFrom judges a hand-over through the node seat as the controller's ask is judged (judgeFound): the +// declared owner and mode it will set, and the disk as it is. +func handOverJudgeFrom(statePath string) func(string) (link.Judged, error) { + return func(path string) (link.Judged, error) { + found, info, err := judgeFound(statePath, path) + if err != nil { + return link.Judged{}, err + } + return link.Judged{Subject: path, Path: path, Sets: orUnsaid(found.Declared), Disk: diskOf(info), + Words: "A directory of " + moduleOf(found.ID) + " is used as found: the mesh left its owner and mode."}, nil + } +} + +func moduleOf(id string) string { + if at := strings.LastIndex(id, "."); at > 0 { + return id[:at] + } + return id +} + +// accessJudgeFrom judges the take act on a library through the node seat (ADR 0328 §7, ADR 0563 §4): declared on this +// machine, its home, and its root holding data the mesh has not taken; what the take sets, and the disk as it is. +func accessJudgeFrom(statePath string) func(string, string) (link.Judged, error) { + return func(act, access string) (link.Judged, error) { + d, err := declaredNow(statePath, nil) + if errors.Is(err, store.ErrNothingDeclared) { + return link.Judged{}, errors.New("this node keeps no declaration, so it is declared no library") + } + if err != nil { + return link.Judged{}, fmt.Errorf("what runs on this node does not read: %w", err) + } + for _, l := range d.Libraries { + if apply.LibraryAccess(l.Library) != access { + continue + } + if !l.Home { + return link.Judged{}, fmt.Errorf("this machine is not the home of the library %s, and a library is "+ + "taken on its home", l.Library) + } + ctx, cancel := context.WithTimeout(context.Background(), lookBound) + defer cancel() + if v := apply.JudgeLibrary(ctx, l); v.Kind != apply.VerdictWaitKeep { + return link.Judged{}, fmt.Errorf("the library %s does not wait for the operator's word to be kept "+ + "(it is judged %s), so there is nothing to take", l.Library, v.Kind) + } + info, err := os.Lstat(l.Root) + if err != nil { + return link.Judged{}, fmt.Errorf("the library %s's root cannot be read: %v", l.Library, err) + } + return link.Judged{Subject: access, Path: filepath.Clean(l.Root), + Sets: "the group " + l.Group + ", group write and setgid on everything under its root", + Disk: diskOf(info), Words: "The library " + l.Library + " holds data the mesh has not taken."}, nil + } + return link.Judged{}, fmt.Errorf("the access %s names no library declared on this machine", access) + } +} + +// askTakes asks the take of every library the look found waiting for the operator's word, once per library (ADR 0563 +// §4): asked by the engine itself, never a second time while one is open or after one was asked; access-act asks again +// by hand. +func askTakes(acts *link.HandActs, waiting []string, say func(string)) { + if acts == nil { + return + } + for _, access := range waiting { + if acts.AskedFor(link.ActAccess, access) { + continue + } + if _, err := acts.AskAccess(link.EngineSeat+"-"+acts.Node, link.ActTake, access, + "its root holds data outside its group", true); err != nil && say != nil { + say("the take of " + access + " could not be asked: " + err.Error()) + } + } +} + +// lastAccounts is the newest look at every account (the health loop's), which root-free judges from. +var lastAccounts atomic.Pointer[accounts.Statement] + +// rootFreshFor is how old the verdict root-free judges from may be (ADR 0563 §6, the controller's verdictFreshFor). +const rootFreshFor = 15 * time.Minute + +// judgeRootFree is this machine's root-free (ADR 0259 §8, ADR 0563 §6): free only on the engine's own verdict, at most +// rootFreshFor old, that every account declared never to become root without a person cannot, at least one such +// account declared, and the login shell's execute not served here. Anything else is no pass, and says why. +func judgeRootFree(node string, st *accounts.Statement, now time.Time, served bool, servedWhy string) link.RootVerdict { + v := link.RootVerdict{Machine: node, Judged: now.UTC()} + var not []string + confined := "" + switch { + case st == nil: + not = append(not, "this machine's engine has judged no account yet") + case now.Sub(st.At) > rootFreshFor: + not = append(not, fmt.Sprintf("the engine's newest judgement of its accounts was made at %s, more than %d "+ + "minutes ago, and a verdict that old is not a verdict on now", st.At.UTC().Format(time.RFC3339), + int(rootFreshFor.Minutes()))) + default: + var named []string + for _, a := range st.Accounts { + if !a.Root { + continue + } + named = append(named, a.Name) + if a.State != accounts.Healthy { + not = append(not, "the account "+a.Name+" "+orNoneKnownWord(a.Reason)) + } + } + if len(named) == 0 { + not = append(not, "no account here is declared never to become root, so agents run as the operator's "+ + "account, which may become root") + } else if len(not) == 0 { + confined = "the account " + strings.Join(named, ", ") + " cannot become root without a person (judged " + + st.At.UTC().Format(time.RFC3339) + ")" + } + } + if served { + not = append(not, "the login shell runs any command an agent gives it as the machine's runtime account, which "+ + "can become root ("+servedWhy+")") + } + if len(not) > 0 { + v.Why = strings.Join(not, "; ") + return v + } + v.Free, v.Why = true, confined+"; the login shell's execute is not served here" + return v +} + +func orNoneKnownWord(s string) string { + if s == "" { + return "is not judged" + } + return s +} + +// settingFileSuffix is the resource the login shell's holder renders its setting execute into (ADR 0268: zsh's +// tools-conf, `execute=serve` or `execute=withhold`), read beside the bus's discovery. +const settingFileSuffix = ".tools-conf" + +// executeServedHere is whether the login shell's execute is served on this machine (ADR 0268), failing closed: served +// while the bus's discovery hears it answered here or could not be asked, or while the holder's setting file applied +// here says anything but withhold or cannot be read. why says which. +func executeServedHere(statePath string, heard func() (bool, error)) (bool, string) { + var why []string + if heard == nil { + why = append(why, "the bus could not be asked whether execute is answered here") + } else if h, err := heard(); err != nil { + why = append(why, "the bus could not be asked whether execute is answered here: "+err.Error()) + } else if h { + why = append(why, "the bus hears execute answered here") + } + known, err := store.Load(statePath) + if err != nil { + why = append(why, "what this machine applied could not be read: "+err.Error()) + } + for _, r := range known.Resources { + if r.Type != "file" || !strings.HasSuffix(r.ID, settingFileSuffix) { + continue + } + if value, err := settingIn(r.Target); err != nil { + why = append(why, moduleOf(r.ID)+"'s setting execute cannot be read: "+err.Error()) + } else if value != "withhold" { + why = append(why, moduleOf(r.ID)+"'s setting execute here is "+value) + } + } + return len(why) > 0, strings.Join(why, "; ") +} + +// settingIn is the value of execute in a setting file, "" when it says none. +func settingIn(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + value := "" + lines := bufio.NewScanner(f) + for lines.Scan() { + t := strings.TrimSpace(lines.Text()) + if key, v, ok := strings.Cut(t, "="); ok && !strings.HasPrefix(t, "#") && strings.TrimSpace(key) == "execute" { + value = strings.TrimSpace(v) + } + } + if value == "" { + value = "not said" + } + return value, lines.Err() +} + +// keepWaiting is every library a look found waiting for the operator's word to keep it (ADR 0328 §8). +func keepWaiting(vs []link.ResourceHealth) []string { + var out []string + for _, v := range vs { + for _, w := range v.AccessWaits { + if w.Reason == link.WaitKeep { + out = append(out, w.Access) + } + } + } + return out +} + +// engineActs is the node seat this engine holds (ADR 0563): judged and performed by the same code as the controller's +// signed asks, its asks to the operator over the session's link, its acts kept beside the state. +func engineActs(opts options, node string, queue *link.Session, search *accounts.SetuidCache) *link.HandActs { + state := link.BusState{Session: queue, Node: node} + h := &link.HandActs{Node: node, State: state, + JudgeHandOver: handOverJudgeFrom(opts.state), HandOver: handOverFrom(opts.state), + JudgeAccess: accessJudgeFrom(opts.state), AccessAct: accessActFrom(opts.state, opts.timeout), + JudgeDirectories: judgeDirectoriesFrom(opts.state), SetuidSearch: queue.SetuidSearch, Searching: search.Running, + RootFree: func() link.RootVerdict { + served, why := executeServedHere(opts.state, func() (bool, error) { + return queue.HeardServed("node-login-shell", "execute") + }) + return judgeRootFree(node, lastAccounts.Load(), time.Now(), served, why) + }, + Say: queue.SayEngine} + asker := asks.EngineAsker(node) + h.Asker = &meshasker.Client{Module: asker, State: state, Publish: queue.PublishAsk, + Record: func(id string) (*stdio.StateEntry, error) { return queue.RouterRecord(asker + "." + id) }} + return h +} + +// reconcileEvery is how often the engine catches up its open asks from the router's records (a warrant heard while no +// link was open, an ask past its expiry): besides each link's opening. +const reconcileEvery = 5 * time.Minute + +func reconcileActs(ctx context.Context, h *link.HandActs) { + t := time.NewTicker(reconcileEvery) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + h.Reconcile() + } + } +} diff --git a/cmd/mesh-host/found.go b/cmd/mesh-host/found.go index 48774d3..eeebed0 100644 --- a/cmd/mesh-host/found.go +++ b/cmd/mesh-host/found.go @@ -68,14 +68,23 @@ func foundDirectories(known store.State) []store.Applied { // known (novox/hq ADR 0272): typed on the control-node, from the operator's own account. The directory's // path is the resource's target beside the reason, never in it: the controller says a reason on the // operator's channel, which withholds a path (ADR 0234 §6), and keeps the target as evidence. +// +// From the lease write, heard as the engine hears it (meshTrust), the line has no answerer: the way out is the node +// seat's hand-over with the directory's path, which asks the operator on their channel (novox/hq ADR 0563 §5). func handOverLine() string { node, _ := foundNode.Load().(string) if node == "" { node = "" } + if t := foundTrust.Load(); t != nil && t.HandedAway() { + return fmt.Sprintf("`%s/node-engine.hand-over` with the directory's path, then your yes on your channel", node) + } return fmt.Sprintf("`nox node hand-over %s ` on the control-node", node) } +// foundTrust is what the engine trusts, whose lease decides the way out a directory used as found names. +var foundTrust atomic.Pointer[meshTrust] + // withFoundDirectories adds to a statement every directory used as found, as its module's resource of kind // directory, unhealthy: the reason names its owner and mode as they are now, what was declared, and the line // that hands it over. The path is the target, which the controller keeps as evidence. @@ -129,16 +138,30 @@ func foundNow() []store.Applied { // never follows a path it did not state: a hand-over is root giving a directory to an account at the next // apply, and a path that resolves elsewhere would give that elsewhere. func handOverFound(statePath, path, by string) (string, error) { + found, _, err := judgeFound(statePath, path) + if err != nil { + return "", err + } + if err := store.AddHandedOver(statePath, store.HandOver{ID: found.ID, Path: path, At: time.Now().UTC(), By: by}); err != nil { + return "", err + } + return fmt.Sprintf("%s (%s) is handed to the mesh by %s: the next apply gives it %s, and it is the mesh's from "+ + "then on — removed when empty and no longer declared", path, found.ID, by, orUnsaid(found.Declared)), nil +} + +// judgeFound is the judgement of a hand-over's path, recording nothing: the directory this machine uses as found at +// that path, as the engine states it, and what is on disk there. +func judgeFound(statePath, path string) (store.Applied, os.FileInfo, error) { if !filepath.IsAbs(path) { - return "", fmt.Errorf("hand-over takes the directory's absolute path, and %q is not one; nothing was handed over", path) + return store.Applied{}, nil, fmt.Errorf("hand-over takes the directory's absolute path, and %q is not one; nothing was handed over", path) } if filepath.Clean(path) != path { - return "", fmt.Errorf("%q is not the directory's path as the engine states it (no `..`, no doubled or trailing "+ + return store.Applied{}, nil, fmt.Errorf("%q is not the directory's path as the engine states it (no `..`, no doubled or trailing "+ "separator); nothing was handed over", path) } known, err := store.Load(statePath) if err != nil { - return "", err + return store.Applied{}, nil, err } var found *store.Applied for _, r := range foundDirectories(known) { @@ -149,31 +172,27 @@ func handOverFound(statePath, path, by string) (string, error) { } } if found == nil { - return "", errors.New(path + " is not a directory this machine uses as found (the module's condition names " + + return store.Applied{}, nil, errors.New(path + " is not a directory this machine uses as found (the module's condition names " + "the ones that are); nothing was handed over") } info, err := os.Lstat(path) switch { case err != nil: - return "", fmt.Errorf("%s cannot be read on this machine: %v; nothing was handed over", path, err) + return store.Applied{}, nil, fmt.Errorf("%s cannot be read on this machine: %v; nothing was handed over", path, err) case info.Mode()&os.ModeSymlink != 0: - return "", fmt.Errorf("%s is a link, and the mesh hands over no link (novox/hq ADR 0012); nothing was handed over", path) + return store.Applied{}, nil, fmt.Errorf("%s is a link, and the mesh hands over no link (novox/hq ADR 0012); nothing was handed over", path) case !info.IsDir(): - return "", fmt.Errorf("%s is not a directory on this machine; nothing was handed over", path) + return store.Applied{}, nil, fmt.Errorf("%s is not a directory on this machine; nothing was handed over", path) } resolved, err := filepath.EvalSymlinks(path) if err != nil { - return "", fmt.Errorf("%s cannot be resolved on this machine: %v; nothing was handed over", path, err) + return store.Applied{}, nil, fmt.Errorf("%s cannot be resolved on this machine: %v; nothing was handed over", path, err) } if resolved != path { - return "", fmt.Errorf("%s is reached through a link (it is %s), and the mesh hands over nothing through one "+ + return store.Applied{}, nil, fmt.Errorf("%s is reached through a link (it is %s), and the mesh hands over nothing through one "+ "(novox/hq issue 339); nothing was handed over", path, resolved) } - if err := store.AddHandedOver(statePath, store.HandOver{ID: found.ID, Path: path, At: time.Now().UTC(), By: by}); err != nil { - return "", err - } - return fmt.Sprintf("%s (%s) is handed to the mesh by %s: the next apply gives it %s, and it is the mesh's from "+ - "then on — removed when empty and no longer declared", path, found.ID, by, orUnsaid(found.Declared)), nil + return *found, info, nil } // handOverFrom is the engine's hand-over for the controller's ask: the state beside statePath, the path as asked. diff --git a/cmd/mesh-host/found_test.go b/cmd/mesh-host/found_test.go index 4a13d2c..f756bfc 100644 --- a/cmd/mesh-host/found_test.go +++ b/cmd/mesh-host/found_test.go @@ -40,6 +40,21 @@ func TestAFoundDirectoryIsStatedUnhealthy(t *testing.T) { strings.Contains(r.Reason, "mesh-host") || strings.Contains(r.Reason, dir) { t.Fatalf("the way out is not the operator's line, or carries the path: %q", r.Reason) } + // ADR 0563 §5: from the lease write the remedy is the node seat's hand-over, answered on the operator's channel, and + // no longer the controller's terminal line; the lease is the one the engine hears. + handed := newMeshTrust(nil) + handed.handedAway = true + foundTrust.Store(handed) + if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason, + "`laptop/node-engine.hand-over`") || strings.Contains(r.Reason, "nox node hand-over") || strings.Contains(r.Reason, dir) { + t.Fatalf("after the lease write the way out is %q", r.Reason) + } + foundTrust.Store(newMeshTrust(nil)) + if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason, + "`nox node hand-over laptop `") { + t.Fatalf("while the controller holds its lease the way out is %q", r.Reason) + } + foundTrust.Store((*meshTrust)(nil)) foundNode.Store("") if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason, "`nox node hand-over `") { diff --git a/cmd/mesh-host/libraries_test.go b/cmd/mesh-host/libraries_test.go index 2f3aafb..10910c4 100644 --- a/cmd/mesh-host/libraries_test.go +++ b/cmd/mesh-host/libraries_test.go @@ -8,6 +8,8 @@ import ( "testing" "time" + meshasker "git.novox.be/novox/mesh-sdk/go/asker" + "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/declaration" "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/link" @@ -53,4 +55,42 @@ func TestALibrarySaysItsWaitWithoutWordsToParse(t *testing.T) { if strings.Contains(media.Reason, dir) || len(books.AccessWaits) != 1 || books.AccessWaits[0].Reason != link.WaitDisk { t.Fatalf("a missing root does not wait for its disk, or a reason names a path: %+v %+v", books, media) } + // 774 (ADR 0563 §4): the library's home engine asks its take of the operator itself, once: no second ask while one is + // open, nor after it ended; and a library waiting for its disk is not asked. + t.Run("the home engine asks the take once", func(t *testing.T) { + var asked []asks.Ask + state := &link.MemState{} + h := &link.HandActs{Node: "n1", State: state, + JudgeAccess: func(act, access string) (link.Judged, error) { + return link.Judged{Subject: access, Path: root, Sets: "the group mesh-media", Disk: "0:0 755"}, nil + }, + AccessAct: func(link.AccessAct) (string, error) { return "taken", nil }} + h.Asker = &meshasker.Client{Module: asks.EngineAsker("n1"), State: state, + Publish: func(subject string, body any, id string) error { + if a, ok := body.(asks.Ask); ok { + asked = append(asked, a) + } + return nil + }} + waiting := keepWaiting(vs) + askTakes(h, waiting, nil) + askTakes(h, waiting, nil) + if len(waiting) != 1 || waiting[0] != "library:media" || len(asked) != 1 { + t.Fatalf("waiting %v asked %d times", waiting, len(asked)) + } + h.Heard(asks.Warrant{Ask: asked[0].ID, Asker: asks.EngineAsker("n1"), Outcome: asks.OutcomeExpired, At: time.Now()}) + askTakes(h, waiting, nil) + if len(asked) != 1 { + t.Fatalf("the take was asked again after its ask ended: %d", len(asked)) + } + // Kept in a key of its own (rev-l3 round 1): the acts gone from the record never make it ask again. + keys, _ := state.Keys("act.") + for _, k := range keys { + _ = state.Delete(k) + } + askTakes(h, waiting, nil) + if len(asked) != 1 { + t.Fatalf("the take was asked again with the acts gone: %d", len(asked)) + } + }) } diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index d8b3610..1b6289a 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1105,10 +1105,17 @@ func runLink(ctx context.Context, opts options) error { // the bus, answer what the controller asks of this machine, and say its facts: when a link opens, and when the // controller asks. proof := &proving{statePath: opts.state, version: runningVersion(), say: say} + // What this engine trusts a mesh signature from: the controller's key until its lease is handed away, and the vault's + // once read from its signers (novox/hq ADR 0550 §2). + trust := newMeshTrust(mine.Membership.Signer) + // And the way out a directory used as found names follows the lease it hears (novox/hq ADR 0563 §5). + foundTrust.Store(trust) queue := &link.Session{ Membership: membership, // And the public halves and certificate its record says (novox/hq ADR 0555 §3), read at each facts report. - Keys: recordKeys(opts.state), + Keys: recordKeys(opts.state, trust.Trusts), + // And the controller's key, read per signed ask: none once its lease is handed away (novox/hq ADR 0563 §7). + ControllerKey: trust.Controller, // And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8). HaveWritten: proof.haveWritten, // And the operator's hand-over of a directory used as found, carried by the controller from the @@ -1129,6 +1136,10 @@ func runLink(ctx context.Context, opts options) error { Say: say, Timeout: opts.timeout, } + // And the node seat node-engine this engine holds (novox/hq ADR 0563): its verbs on every link, its own asks to the + // operator under node-engine-, and its acts kept beside its state. + queue.Acts = engineActs(opts, mine.Node, queue, setuidSearch) + go reconcileActs(ctx, queue.Acts) current := &agentNow{} setRunsHere(runsFromAgent(current)) factsOfNow := func(ctx context.Context, ag *agent.Agent) { @@ -1173,7 +1184,10 @@ func runLink(ctx context.Context, opts options) error { } // The node follows its wants: one module at a time from the composition its want names (novox/hq ADR 0539), and the // busy containers are asked for again soon (ADR 0360 §11). - go nodeAgent(aside, opts, mine.Membership.Signer, queue, sched, say, afterApply, current, factsOfNow) + // And what it trusts, followed on the bus, said in its record whenever it changes (ADR 0550 §2). + trust.onChange(func() { factsOfNow(aside, current.get()) }) + go followTrust(aside, queue, trust, say) + go nodeAgent(aside, opts, mine.Membership.Signer, trust, queue, sched, say, afterApply, current, factsOfNow) go busyPasses(aside, func() { if ag := current.get(); ag != nil { ag.Retry(busyModules()...) @@ -1370,6 +1384,7 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, if a := accountJudge; a != nil { as, accountsChanged := a.Look(ctx) accountSt = &as + lastAccounts.Store(&as) owed = owed || accountsChanged if accountsChanged { for _, v := range as.Accounts { @@ -1384,6 +1399,9 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, accessSt, accessesChanged := accesses.look(time.Now()) keeping.poke(ctx) owed = owed || accessesChanged + // And a library whose root holds data outside its group is asked of the operator by this engine, once (novox/hq + // ADR 0563 §4). + askTakes(queue.Acts, keepWaiting(accessSt), say) if accessesChanged { for _, v := range accessSt { if v.State != link.StateHealthy { @@ -2167,9 +2185,12 @@ func withMissing(st liveness.Statement) liveness.Statement { // recordKeys reads the public halves of the node's keys and the certificate the mesh issued for its internal name from // its state directory (novox/hq ADR 0555 §3): what the node's record says of them. A key that cannot be read is left // empty, never invented; the certificate is there only after the first declaration that carried it. -func recordKeys(state string) func() link.RecordKeys { +func recordKeys(state string, trusts func() []string) func() link.RecordKeys { return func() link.RecordKeys { var k link.RecordKeys + if trusts != nil { + k.Trusts = trusts() + } if id, err := identity.Load(identity.Path(state)); err == nil { k.TunnelKey = id.Overlay.Public } diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 430be23..667d18e 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -5,12 +5,14 @@ import ( "crypto/ed25519" "encoding/base64" "encoding/json" + "errors" "fmt" "os" "path/filepath" "sort" "strings" "sync" + "sync/atomic" "time" "git.novox.be/novox/mesh-sdk/go/compose/frombus" @@ -343,13 +345,14 @@ func resourcesOf(cs []delivery.Composed) []declaration.Resource { // controller's key the node was enrolled with (novox/hq ADR 0539 §3, §11); a node adopted later has it at its adoption. const operatorKeysFile = "operator-keys" -// trustedKeys are the keys a composition may be signed by: the controller's, which the node was enrolled with, and the -// operator's, read from operatorKeysFile each time it is asked, so a key that arrives while the agent runs is trusted at -// the next composition. A file that cannot be read trusts the controller's alone, and says so. -func trustedKeys(opts options, controller ed25519.PublicKey) func() []ed25519.PublicKey { +// trustedKeys are the keys a composition may be signed by: the controller's, which the node was enrolled with, while the +// controller holds its lease (ADR 0550 §2: trusted no more once it is handed away), and the operator's, read from +// operatorKeysFile each time it is asked, so a key that arrives while the agent runs is trusted at the next composition. +// A file that cannot be read trusts the controller's alone, and says so. +func trustedKeys(opts options, controllerNow func() ed25519.PublicKey) func() []ed25519.PublicKey { return func() []ed25519.PublicKey { keys := []ed25519.PublicKey{} - if len(controller) == ed25519.PublicKeySize { + if controller := controllerNow(); len(controller) == ed25519.PublicKeySize { keys = append(keys, controller) } raw, err := os.ReadFile(filepath.Join(filepath.Dir(opts.state), operatorKeysFile)) @@ -373,13 +376,16 @@ func trustedKeys(opts options, controller ed25519.PublicKey) func() []ed25519.Pu } // keptOnce reads the kept declaration once and splits it by module: its resources (the node's own are those that say -// none, ADR 0539 §9) and its filled paths. +// none, ADR 0539 §9) and its filled paths. Once the read is retired (keptRetired) nothing is read. func keptOnce(opts options, signer ed25519.PublicKey) func() (map[string][]json.RawMessage, map[string][]string) { var once sync.Once var by map[string][]json.RawMessage var filled map[string][]string return func() (map[string][]json.RawMessage, map[string][]string) { once.Do(func() { + if keptRetired.Load() { + return + } raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer) if err != nil { return @@ -429,7 +435,7 @@ func splitKept(raw []byte) (map[string][]json.RawMessage, map[string][]string) { // nodeAgent runs the node agent under the session's links for as long as ctx lasts: it waits for a link, follows the // node's wants and compositions on it, and begins again with the next link. A watch it cannot make (a bucket this node's // account may not read) is said, and tried again with a growing wait, as the link is. -func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, session *link.Session, sched *apply.Scheduler, +func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trust *meshTrust, session *link.Session, sched *apply.Scheduler, say link.Announce, afterApply func(), current *agentNow, sayFacts func(ctx context.Context, ag *agent.Agent)) { wait := 2 * time.Second for ctx.Err() == nil { @@ -446,6 +452,11 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess return } var ag *agent.Agent + // The kept release-5 declaration is no longer read once every module the seat's placement puts here has a have of its + // own (ADR 0550 §2: its signature is the controller's, trusted no more after the lease write). + if retired, why := placedAllHave(linked, session, node); retired && !keptRetired.Swap(true) { + say("the kept declaration is no longer read: " + why) + } kept := keptOnce(opts, signer) // The mesh's own certificate the declaration kept from before carries, kept for the record before this machine // composes its own (ADR 0555 §3): without it a module that asks for a certificate would be stood in for. @@ -458,7 +469,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess } // The bus's user list, written alone from the newest composition of the bus module (ADR 0547 §8): on every // node, and a node whose compositions carry no list writes nothing. - users := userListFollower(opts, session, composedBucket, trustedKeys(opts, signer), say, func() { + users := userListFollower(opts, session, composedBucket, trustedKeys(opts, trust.Controller), say, func() { // This engine wrote the list, so the bus reloads its accounts and answers no JetStream request for a few // seconds: what failed in them is tried again once that is over, not at its next wait. time.AfterFunc(reloadSettles, func() { @@ -495,7 +506,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess } return out, nil }, - Trusted: trustedKeys(opts, signer), + Trusted: trustedKeys(opts, trust.Controller), Backup: backupNode(session), // What the filling refused before this apply may be named now: the provisioner's client is told to fill // again (ace, 2026-10-25 21:17). @@ -572,11 +583,17 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess now.NotComposedEach(run, why, held) // a module held alone marks that module, never the others (issue 1126) } }, + // And the vault's authority beside the controller's, and this node's certificate asked of the vault when a + // module here asks for one (ADR 0550 §2). + vaultAuthority: trust.VaultAuthority, + certify: func(ctx context.Context) error { + return certifyThisNode(ctx, opts.state, node, trust, session.AskVault, func() { sayFacts(ctx, ag) }) + }, } // And its answer to every placement candidate, placed or not, applying nothing (novox/hq ADR 0559 §4): written into // its own key, `.node`, under candidates. shadowHooks := hooks - shadowHooks.refused = nil + shadowHooks.refused, shadowHooks.certify = nil, nil shadowEnv := composerEnv(session, node, say, shadowHooks) // And, after each answer, what runs of an own secret the answer says the vault holds another value of, handed to // the vault while the node applies the controller's compositions (issue 1126, cause C4; ownresync). @@ -725,8 +742,19 @@ func busyModules() []string { return out } -// keptDeclaration is the declaration the node kept before the switch, verified with the mesh's key when it is given. +// keptRetired says the kept declaration is read no more: every module the seat's placement puts on this node has a have of +// its own (placedAllHave), so what runs is the agent's alone (ADR 0550 §2, design 47's L3 s9). +var keptRetired atomic.Bool + +// errKeptRetired is keptDeclaration's answer once the read is retired. +var errKeptRetired = errors.New("the kept declaration is read no more: every module placed on this node has a have of its own") + +// keptDeclaration is the declaration the node kept before the switch, verified with the mesh's key when it is given; +// once the read is retired, nothing is read. func keptDeclaration(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) { + if keptRetired.Load() { + return nil, errKeptRetired + } var raw []byte if len(signer) > 0 { var err error @@ -903,6 +931,9 @@ type composerHooks struct { unseal func(string) ([]byte, error) refused func(why string, held map[string]string) held func(module string) (delivery.Seals, []string, error) + // vaultAuthority and certify are selfcompose.Env's VaultAuthority and Certify. + vaultAuthority func() string + certify func(ctx context.Context) error } func selfComposer(ctx context.Context, session *link.Session, node string, say func(string), hooks composerHooks) (*selfcompose.Composer, string) { @@ -966,7 +997,8 @@ func composerEnv(session *link.Session, node string, say func(string), hooks com Hear: session.Hear, // And a moved module's templates archive, in the artifact store, as every archive is fetched (issue 1126). Fetch: apply.Fetch, - Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held} + Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held, + VaultAuthority: hooks.vaultAuthority, Certify: hooks.certify} } // endWhenUnplaced calls stop once this node's placement key is deleted from the seat's placement (after the watch diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index b214352..852447b 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -159,7 +159,8 @@ func (b *itBed) start(trustOperator bool) (stop func()) { done := make(chan struct{}) go func() { defer close(done) - nodeAgent(ctx, b.opts, b.controll.Public().(ed25519.PublicKey), b.session, nil, b.say, func() {}, + controller := b.controll.Public().(ed25519.PublicKey) + nodeAgent(ctx, b.opts, controller, newMeshTrust(controller), b.session, nil, b.say, func() {}, &agentNow{}, func(context.Context, *agent.Agent) {}) }() return func() { cancel(); <-done } diff --git a/cmd/mesh-host/trust.go b/cmd/mesh-host/trust.go new file mode 100644 index 0000000..2eadbc6 --- /dev/null +++ b/cmd/mesh-host/trust.go @@ -0,0 +1,317 @@ +package main + +import ( + "context" + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" + + "github.com/novox/mesh-host/internal/identity" + "github.com/novox/mesh-host/internal/link" +) + +// **What this engine trusts a mesh signature from** (novox/hq ADR 0550 §2; design 47's L3 item d). Two signers: the +// controller's key, which the node was enrolled with (its membership's signer), and the vault's key, read from the vault's +// state `signers`, which only the vault may write: the bus grants are its anchor, as the wants' and the placement's (the +// lead's design limit, 2026-10-28). The controller's key is trusted until its lease is handed away, heard on the lease key +// (buscompose.LeaseHandedAway) and never on a clock; from then the vault's alone. What it trusts now the node states in its +// record (NodeRecord.Trusts), so the gate before the lease write reads every node's from the bus. +type meshTrust struct { + mu sync.Mutex + controller ed25519.PublicKey + handedAway bool + vault ed25519.PublicKey + authority string + // changed is told when what it trusts changed, so the record is written again. + changed func() +} + +// newMeshTrust trusts the controller's key alone until the vault's is read. +func newMeshTrust(controller ed25519.PublicKey) *meshTrust { + t := &meshTrust{} + if len(controller) == ed25519.PublicKeySize { + t.controller = controller + } + return t +} + +// onChange names who is told when what it trusts changed. +func (t *meshTrust) onChange(f func()) { + t.mu.Lock() + t.changed = f + t.mu.Unlock() +} + +// Controller is the controller's key while the controller holds its lease, and nil once the lease is handed away. +func (t *meshTrust) Controller() ed25519.PublicKey { + if t == nil { + return nil + } + t.mu.Lock() + defer t.mu.Unlock() + if t.handedAway { + return nil + } + return t.controller +} + +// HandedAway says the controller's lease was handed away, as heard on the lease key. +func (t *meshTrust) HandedAway() bool { + if t == nil { + return false + } + t.mu.Lock() + defer t.mu.Unlock() + return t.handedAway +} + +// Signers are the keys a mesh signature is verified against now: the controller's while it holds the lease, and the +// vault's once read. +func (t *meshTrust) Signers() []delivery.Signer { + t.mu.Lock() + defer t.mu.Unlock() + var out []delivery.Signer + if t.controller != nil && !t.handedAway { + out = append(out, delivery.Signer{Name: delivery.SignerController, Key: t.controller}) + } + if t.vault != nil { + out = append(out, delivery.Signer{Name: delivery.SignerVault, Key: t.vault}) + } + return out +} + +// Verify is the signer that signed msg as kind, or why none it trusts did. +func (t *meshTrust) Verify(kind string, msg, sig []byte) (delivery.Signer, error) { + context, err := delivery.SignContext(kind) + if err != nil { + return delivery.Signer{}, err + } + return delivery.VerifyBy(t.Signers(), context, msg, sig) +} + +// VaultAuthority is the vault's authority certificate as read from `signers`, or "". +func (t *meshTrust) VaultAuthority() string { + if t == nil { + return "" + } + t.mu.Lock() + defer t.mu.Unlock() + return t.authority +} + +// Trusts is what the record says it trusts: each signer's key and the vault's authority, by fingerprint. +func (t *meshTrust) Trusts() []string { + if t == nil { + return nil + } + authorities := map[string]string{} + if a := t.VaultAuthority(); a != "" { + authorities[delivery.SignerVault] = a + } + return delivery.Trusts(t.Signers(), authorities) +} + +// change applies f under the lock and tells who follows when Trusts differs after it. +func (t *meshTrust) change(f func()) { + before := strings.Join(t.Trusts(), "\n") + t.mu.Lock() + f() + changed := t.changed + t.mu.Unlock() + if strings.Join(t.Trusts(), "\n") != before && changed != nil { + changed() + } +} + +// heardSigner takes one key of the vault's `signers`: its key or its authority, refused (and said) when it does not +// check; a deleted key is no longer trusted. +func (t *meshTrust) heardSigner(key string, value []byte, deleted bool, say link.Announce) { + switch key { + case delivery.SignersKeyKey: + var pub ed25519.PublicKey + if !deleted && len(value) > 0 { + k, err := delivery.DecodeSignerKey(value) + if err == nil { + pub, err = k.PublicKey() + } + if err != nil { + say("the vault's signing key in " + delivery.SignersRead + " is not trusted: " + err.Error()) + return + } + } + t.change(func() { t.vault = pub }) + case delivery.SignersAuthorityKey: + pem := "" + if !deleted && len(value) > 0 { + a, err := delivery.DecodeSignerAuthority(value) + if err != nil { + say("the vault's authority in " + delivery.SignersRead + " is not trusted: " + err.Error()) + return + } + pem = a.Certificate + } + t.change(func() { t.authority = pem }) + } +} + +// heardLease takes the controller's lease key as read: once handed away to a holder that is no controller, the +// controller's key is trusted no more. A lease that cannot be read changes nothing. +func (t *meshTrust) heardLease(raw []byte, found bool, readErr error) { + if readErr != nil { + return + } + handed, _ := buscompose.LeaseHandedAway(raw, found, nil) + t.change(func() { t.handedAway = handed }) +} + +// trustFollowAgain bounds the wait before a watch that could not be made is tried again on the same link: the vault may +// not serve `sign` yet, and until it does this node is not granted the reads. +var trustFollowAgain = 2 * time.Minute + +// followTrust follows the vault's `signers` and the controller's lease on every link, as watches (never a poll), for as +// long as ctx lasts. A watch that cannot be made is said once per reason and tried again. +func followTrust(ctx context.Context, session *link.Session, trust *meshTrust, say link.Announce) { + said := "" + sayOnce := func(line string) { + if line != said { + said = line + if line != "" { + say(line) + } + } + } + for ctx.Err() == nil { + linked, cancel, err := session.WhenLinked(ctx) + if err != nil { + return + } + watching, stop := context.WithCancel(linked) + signers, serr := session.KVWatch(watching, delivery.SignersBucket, ">") + lease, lerr := session.KVWatch(watching, buscompose.LeaseBucket, buscompose.ControllerLeaseKey) + if serr != nil || lerr != nil { + stop() + sayOnce(fmt.Sprintf("what this node trusts a mesh signature from is not followed yet (the vault's signers: %v; "+ + "the controller's lease: %v): it trusts what it last read, and looks again in %s", serr, lerr, trustFollowAgain)) + signers, lease = nil, nil + } else { + sayOnce("") + } + for signers != nil || lease != nil { + select { + case u, ok := <-signers: + if !ok { + signers = nil + continue + } + if !u.Caught { + trust.heardSigner(u.Key, u.Value, u.Deleted, say) + } + case u, ok := <-lease: + if !ok { + lease = nil + continue + } + if !u.Caught { + trust.heardLease(u.Value, !u.Deleted && len(u.Value) > 0, nil) + } + } + } + stop() + select { + case <-linked.Done(): + case <-time.After(trustFollowAgain): + } + cancel() + } +} + +// placedAllHave says whether every module the seat mesh-placement places on node has a have of its own, read from the +// bus: then the kept declaration is read no more (keptRetired). A node the seat does not place, or whose placement or haves +// cannot be read, keeps reading it. +func placedAllHave(ctx context.Context, session *link.Session, node string) (bool, string) { + key, err := delivery.PlacementKey(node) + if err != nil { + return false, err.Error() + } + h, err := session.KVHistory(ctx, delivery.PlacementBucket, key) + if err != nil || len(h) == 0 || h[len(h)-1].Deleted { + return false, "its placement is not read" + } + p, err := delivery.DecodePlacement(h[len(h)-1].Value) + if err != nil { + return false, err.Error() + } + return allHave(p, func(module string) (bool, error) { + _, ok, err := session.ReadHaveErr(ctx, module) + return ok, err + }) +} + +// allHave says whether has finds a have for every module p places, and why not. +func allHave(p delivery.Placement, has func(module string) (bool, error)) (bool, string) { + if len(p.Modules) == 0 { + return false, "the seat mesh-placement places no module here" + } + for _, m := range p.Modules { + ok, err := has(m.Module) + if err != nil { + return false, "the have of " + m.Module + " is not read: " + err.Error() + } + if !ok { + return false, m.Module + " has no have yet" + } + } + return true, fmt.Sprintf("each of the %d modules placed here has a have of its own", len(p.Modules)) +} + +// certifyThisNode asks the vault's `sign` for this node's own certificate (kind node-certificate, ADR 0550 §2), checks +// it is for this node's internal name and serving key and signed by the authority the answer names, which must be the +// vault's this node trusts, keeps it where the record reads it, and writes the record again (written). +func certifyThisNode(ctx context.Context, state, node string, trust *meshTrust, + ask func(ctx context.Context, verb string, args any) (json.RawMessage, error), written func()) error { + serving, err := identity.LoadServingKey(identity.ServingKeyPath(state)) + if err != nil { + return err + } + key, err := base64.StdEncoding.DecodeString(serving.Public) + if err != nil || len(key) != ed25519.PublicKeySize { + return fmt.Errorf("this node's serving key is not an ed25519 key") + } + raw, err := ask(ctx, delivery.SignVerb, map[string]any{"kind": delivery.KindNodeCertificate, "node": node}) + if err != nil { + return err + } + var a delivery.SignAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return fmt.Errorf("the vault's answer is not a signing: %w", err) + } + trusted := trust.VaultAuthority() + if trusted == "" { + return fmt.Errorf("this node has read no authority of the vault's to check the certificate against") + } + if err := delivery.CheckNodeCertificate(a.Certificate, []string{trusted}, overlay.InternalName(node), + ed25519.PublicKey(key), time.Now()); err != nil { + return err + } + path := identity.CertificatePath(state) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + if err := os.WriteFile(path, []byte(a.Certificate), 0o644); err != nil { + return err + } + if written != nil { + written() + } + return nil +} diff --git a/cmd/mesh-host/units_test.go b/cmd/mesh-host/units_test.go index 22ce465..bbbcf32 100644 --- a/cmd/mesh-host/units_test.go +++ b/cmd/mesh-host/units_test.go @@ -1,6 +1,9 @@ package main import ( + "errors" + "os" + "path/filepath" "strings" "testing" "time" @@ -9,6 +12,7 @@ import ( "github.com/novox/mesh-host/internal/accounts" "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/liveness" + "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/units" ) @@ -103,6 +107,61 @@ func TestTheStatementSaysAnAccountJudgedForRoot(t *testing.T) { if got["claude-code.agent"] != "never" || got["openrazer.account"] != "" { t.Fatalf("root said: %v", got) } + // 774 (ADR 0563 §6): the node seat's root-free passes only on the engine's own positive verdict at most 15 minutes old + // with execute not served here; a stale, missing or negative verdict, or execute served, is no pass and says why. + t.Run("this machine's root-free", func(t *testing.T) { + agent := accounts.Verdict{Account: accounts.Account{Module: "claude-code", ID: "claude-code.agent", Name: "agent", Root: true}, + State: accounts.Healthy, Since: at} + free := &accounts.Statement{At: at, Accounts: []accounts.Verdict{agent}} + for _, row := range []struct { + name string + st *accounts.Statement + after time.Duration + served bool + free bool + why string + }{ + {name: "a positive verdict", st: free, after: 14 * time.Minute, free: true, why: "the account agent cannot become root"}, + {name: "a stale verdict", st: free, after: 16 * time.Minute, why: "more than 15 minutes ago"}, + {name: "no verdict", st: nil, why: "judged no account yet"}, + {name: "a way to root", st: as, why: "in the group docker, which grants root"}, + {name: "no agent account", st: &accounts.Statement{At: at}, why: "agents run as the operator's account"}, + {name: "execute served", st: free, served: true, why: "the login shell runs any command"}, + } { + v := judgeRootFree("ace", row.st, at.Add(row.after), row.served, "the bus hears execute answered here") + if v.Free != row.free || !strings.Contains(v.Why, row.why) || v.Machine != "ace" { + t.Errorf("%s: %+v", row.name, v) + } + } + // execute is served while the holder's setting file says anything but withhold, or the bus hears it. + dir := t.TempDir() + statePath, conf := filepath.Join(dir, "state.json"), filepath.Join(dir, "tools.conf") + if err := store.Save(statePath, store.State{Resources: []store.Applied{{ID: "zsh.tools-conf", Type: "file", Target: conf}}}); err != nil { + t.Fatal(err) + } + quiet := func() (bool, error) { return false, nil } + for _, row := range []struct { + name, setting string + heard func() (bool, error) + served bool + }{ + {name: "withheld and not heard", setting: "# the mesh's\nexecute=withhold\n", heard: quiet}, + {name: "served", setting: "execute=serve\n", heard: quiet, served: true}, + {name: "withheld and heard", setting: "execute=withhold\n", heard: func() (bool, error) { return true, nil }, served: true}, + {name: "the bus not asked", setting: "execute=withhold\n", heard: func() (bool, error) { return false, errors.New("no link") }, served: true}, + {name: "no setting file", heard: quiet, served: true}, + } { + _ = os.Remove(conf) + if row.setting != "" { + if err := os.WriteFile(conf, []byte(row.setting), 0o644); err != nil { + t.Fatal(err) + } + } + if served, why := executeServedHere(statePath, row.heard); served != row.served { + t.Errorf("%s: served %v (%s)", row.name, served, why) + } + } + }) } // A verdict waiting for the setuid search says how far it got, in the reason the root-free verb repeats and as diff --git a/cmd/mesh-host/whatruns_test.go b/cmd/mesh-host/whatruns_test.go index 2ffd7c6..1d4601e 100644 --- a/cmd/mesh-host/whatruns_test.go +++ b/cmd/mesh-host/whatruns_test.go @@ -4,6 +4,7 @@ import ( "crypto/ed25519" "encoding/json" "errors" + "path/filepath" "strings" "testing" @@ -11,6 +12,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-host/internal/agent" + "github.com/novox/mesh-host/internal/store" ) func aFileOf(module, id string) json.RawMessage { @@ -59,6 +61,41 @@ func TestDeclaredNowReadsWhatRunsWhenTheAgentIsRunning(t *testing.T) { if err != nil || d.Provisioner != "from-agent" { t.Fatalf("declaredNow read %+v (%v), not what runs", d, err) } + + // L3 s9 (ADR 0550 §2): a node whose placed modules all have a have reads the kept declaration no more, though the + // file is there and readable and the controller's key is dropped; until then it reads it. + state := filepath.Join(t.TempDir(), "state", "state.json") + if err := store.SaveDeclared(store.DeclaredPath(state), store.Declared{Declaration: []byte(`{"declaration":1,` + + `"resources":[{"id":"zsh.rc","type":"file","module":"zsh","path":"/etc/zshrc","content":"x"}]}`)}); err != nil { + t.Fatal(err) + } + if d, err := keptDeclaration(state, nil); err != nil || len(d.Resources) != 1 { + t.Fatalf("before the read is retired: %+v, %v", d, err) + } + placed := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "zsh"}, {Module: "lemurs"}}} + for _, row := range []struct { + name string + has map[string]bool + want bool + }{ + {"every placed module has a have", map[string]bool{"zsh": true, "lemurs": true}, true}, + {"one has none yet", map[string]bool{"zsh": true}, false}, + } { + if got, why := allHave(placed, func(m string) (bool, error) { return row.has[m], nil }); got != row.want { + t.Errorf("%s: %v (%s)", row.name, got, why) + } + } + if got, _ := allHave(delivery.Placement{Node: "n1"}, nil); got { + t.Error("a node with nothing placed retires the read") + } + keptRetired.Store(true) + defer keptRetired.Store(false) + if _, err := keptDeclaration(state, nil); !errors.Is(err, errKeptRetired) { + t.Errorf("after the read is retired: %v", err) + } + if by, filled := keptOnce(options{state: state}, nil)(); by != nil || filled != nil { + t.Errorf("after the read is retired the kept declaration was split: %v %v", by, filled) + } } // 1945/AC17 (unit): a resource the node's own composition shares with a running module's composition is declared once, diff --git a/internal/accounts/setuid_cache.go b/internal/accounts/setuid_cache.go index 777bbf5..eadd8a0 100644 --- a/internal/accounts/setuid_cache.go +++ b/internal/accounts/setuid_cache.go @@ -240,6 +240,17 @@ func (c *SetuidCache) SearchAgain() { c.forgetProgress() } +// Running says whether a search walks now (novox/hq ADR 0563 §2: the node seat starts no second while one does). +// Nil c runs none. +func (c *SetuidCache) Running() bool { + if c == nil { + return false + } + c.mu.Lock() + defer c.mu.Unlock() + return c.running +} + // SearchNow throws away the last complete search, kept or in memory, and any walk under way or kept, and has // the next look start a full search at once, past AgainAfter and any back-off (novox/hq issue 361): the // operator changed by hand what the last one found, which no apply says. Until the new one completes the diff --git a/internal/link/access_act_test.go b/internal/link/access_act_test.go index 0bd38f6..82bf596 100644 --- a/internal/link/access_act_test.go +++ b/internal/link/access_act_test.go @@ -6,6 +6,10 @@ import ( "strings" "testing" "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asker" + "git.novox.be/novox/mesh-sdk/go/asks" ) // The take act's ask (novox/hq ADR 0325 §8): a hand-over's fields and the act's own in one signed object, under a @@ -68,6 +72,112 @@ func TestTheTakeActRunsOnlyOnAWarrant(t *testing.T) { if a.Said != "started" || len(given) != 1 || given[0].Warrant != "w-1" || given[0].Path != "/srv/media/movies" { t.Fatalf("a warranted ask did not reach the act whole: %+v %+v", a, given) } + // 774 (ADR 0563 §3): a hand-over asked through the node seat is the engine's own ask, performed only on the + // operator's warrant to it: never on the call, not on a warrant for another digest, once on a warrant heard twice, + // and not when the disk changed since it was asked. + t.Run("the node seat's hand-over acts only on the operator's warrant", func(t *testing.T) { + for _, row := range []struct { + name string + warrant func(asked asks.Ask, w asks.Warrant) asks.Warrant + diskAfter string + twice bool + done int + state string + }{ + {name: "no warrant", done: 0, state: HandAsked}, + {name: "the operator's yes", warrant: yes, done: 1, state: HandDone}, + {name: "the yes heard twice", warrant: yes, twice: true, done: 1, state: HandDone}, + {name: "a warrant for another digest", warrant: func(a asks.Ask, w asks.Warrant) asks.Warrant { + w = yes(a, w) + w.AskDigest = "sha256:another" + return w + }, done: 0, state: HandAsked}, + {name: "a no", warrant: func(a asks.Ask, w asks.Warrant) asks.Warrant { + w = yes(a, w) + w.Option, w.Label, w.Level = "no", "No, leave it", asks.Acknowledge + return w + }, done: 0, state: HandRefused}, + {name: "the disk changed", warrant: yes, diskAfter: "0:0 700 9:2", done: 0, state: HandRefused}, + {name: "expired", warrant: func(a asks.Ask, w asks.Warrant) asks.Warrant { + return asks.Warrant{Ask: a.ID, Asker: w.Asker, Outcome: asks.OutcomeExpired, At: w.At} + }, done: 0, state: HandRefused}, + } { + t.Run(row.name, func(t *testing.T) { + h, asked, done := anEngine(t) + disk := "1000:1000 755 9:1" + h.JudgeHandOver = func(path string) (Judged, error) { + return Judged{Subject: path, Path: path, Sets: "owner jo, group jo, mode 0750", Disk: disk, + Words: "A directory used as found."}, nil + } + if _, err := h.Call(EngineCall{Verb: ActHandOver, Caller: "shanks.node-tools", + Args: map[string]any{"path": "/srv/x", "why": "a test"}}); err != nil { + t.Fatal(err) + } + if len(*asked) != 1 || *done != 0 { + t.Fatalf("the call asked %d and performed %d", len(*asked), *done) + } + // The act is one key, and the open ask one key made by a create (ADR 0563 §3). + if acts, _ := h.State.Keys("act."); len(acts) != 1 || acts[0] != "act."+(*asked)[0].ID { + t.Fatalf("the act is kept as %v", acts) + } + if open, _ := h.State.Keys("open."); len(open) != 1 || !strings.HasPrefix(open[0], "open.hand-over.") { + t.Fatalf("the open ask is kept as %v", open) + } + if _, err := h.Call(EngineCall{Verb: ActHandOver, Caller: "x", Args: map[string]any{"path": "/srv/x"}}); err == nil { + t.Fatal("a second ask of the same path was asked while one is open") + } + if row.diskAfter != "" { + disk = row.diskAfter + } + if row.warrant != nil { + w := row.warrant((*asked)[0], asks.Warrant{Asker: asks.EngineAsker("n1"), At: time.Now()}) + h.Heard(w) + if row.twice { + h.Heard(w) + } + } + got, _ := h.Acts(0) + acts := got.(map[string]any)["acts"].([]HandAct) + if *done != row.done || len(acts) != 1 || acts[0].State != row.state { + t.Fatalf("performed %d (want %d), acts %+v (want %s)", *done, row.done, acts, row.state) + } + if open, _ := h.State.Keys("open."); (len(open) == 1) != (row.state == HandAsked) { + t.Fatalf("an ask %s leaves its open key as %v", row.state, open) + } + if row.state == HandDone && (acts[0].By != "operator" || acts[0].Via == "" || acts[0].Ask == "") { + t.Fatalf("the act does not record the warrant: %+v", acts[0]) + } + }) + } + }) +} + +// yes is the operator's yes to an ask, as the router writes it. +func yes(a asks.Ask, w asks.Warrant) asks.Warrant { + w.Ask, w.Outcome, w.Option, w.Label, w.Level = a.ID, asks.OutcomeChosen, "yes", "Yes", asks.Approve + w.By = &asks.Person{Who: asks.Operator, Kind: "telegram", Verified: "user id verified"} + w.AskDigest, w.Proofs = a.Digest(), []string{"P1"} + return w +} + +// anEngine is a node seat with its own state in a temporary folder, a router that keeps what it was asked, and a +// hand-over that counts what it performed. +func anEngine(t *testing.T) (*HandActs, *[]asks.Ask, *int) { + t.Helper() + var asked []asks.Ask + done := 0 + state := &MemState{} + h := &HandActs{Node: "n1", State: state, + HandOver: func(path, by string) (string, error) { done++; return path + " handed over by " + by, nil }} + h.Asker = &asker.Client{Module: asks.EngineAsker("n1"), State: state, + Publish: func(subject string, body any, id string) error { + if a, ok := body.(asks.Ask); ok && subject == asks.AskSubject(asks.EngineAsker("n1")) { + asked = append(asked, a) + } + return nil + }, + Record: func(string) (*stdio.StateEntry, error) { return nil, nil }} + return h, &asked, &done } // The controller names an access by its declared resource id exactly, which may hold a `/`: it reaches the act, diff --git a/internal/link/engineseat.go b/internal/link/engineseat.go new file mode 100644 index 0000000..258e3f5 --- /dev/null +++ b/internal/link/engineseat.go @@ -0,0 +1,382 @@ +package link + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/micro" +) + +// The node seat node-engine, served on the engine's own link (novox/hq ADR 0563 §1). +// +// Each verb is answered on this node's subject alone, `mesh.seat.node-engine.tool..`, and on the same +// address naming its caller, `mesh.seat.node-engine.call...`, whose last token the bus lets each user +// publish with its own name alone (novox/hq issue 365): the caller is that token, and a call on the subject naming none is +// from an unproven caller, recorded so. The body is the call's arguments; the answer is {result, error, node}, as every +// tool's (mesh-tools node-tools). One service, `node-engine`, answers discovery with each verb's seat, tool and node. + +// EngineSeat is the seat. +const EngineSeat = catalogue.EngineSeat + +// EngineToolSubject is where a verb of this node's seat is asked naming no caller. +func EngineToolSubject(verb, node string) string { + return "mesh.seat." + EngineSeat + ".tool." + verb + "." + node +} + +// EngineCallPattern is where a verb of this node's seat is asked naming its caller, any caller. +func EngineCallPattern(verb, node string) string { + return "mesh.seat." + EngineSeat + ".call." + verb + "." + node + ".*" +} + +// EngineEventSubject is where the seat says one of its events about this node. +func EngineEventSubject(event, node string) string { + return "mesh.seat." + EngineSeat + ".event." + event + "." + node +} + +// EngineCall is one call of the seat: its verb, its caller as the bus proved it, and its arguments. +type EngineCall struct { + Verb string + Caller string + Args map[string]any +} + +// EngineFunc answers one call: its result, or why it is refused. +type EngineFunc func(EngineCall) (any, error) + +// CallerOfSubject is the caller a call's subject names for this node's verb: the last token of a caller-named subject, +// each `~` a dot, or stdio.Unproven on the subject that names none or on any other. +func CallerOfSubject(subject, verb, node string) string { + token, ok := strings.CutPrefix(subject, "mesh.seat."+EngineSeat+".call."+verb+"."+node+".") + if !ok || token == "" || strings.ContainsAny(token, ".*> ") { + return stdio.Unproven + } + return strings.ReplaceAll(token, "~", ".") +} + +// MaxEngineCall bounds a call's arguments: a path, an access and a few words. +const MaxEngineCall = 16 << 10 + +// AnswerEngineCall is the envelope one call is answered with: the verb's result, or its refusal in words. +func AnswerEngineCall(handle EngineFunc, verb, node, subject string, body []byte) []byte { + envelope := map[string]any{"node": node} + say := func() []byte { + raw, err := json.Marshal(envelope) + if err != nil { + raw, _ = json.Marshal(map[string]any{"node": node, "error": "the answer could not be written: " + err.Error()}) + } + return raw + } + args := map[string]any{} + switch { + case handle == nil: + envelope["error"] = "this engine serves no verb of " + EngineSeat + return say() + case len(body) > MaxEngineCall: + envelope["error"] = fmt.Sprintf("the call is %d bytes, more than a call of %s takes", len(body), EngineSeat) + return say() + case len(strings.TrimSpace(string(body))) > 0: + if err := json.Unmarshal(body, &args); err != nil { + envelope["error"] = "the call's arguments are not an object: " + err.Error() + return say() + } + } + result, err := handle(EngineCall{Verb: verb, Caller: CallerOfSubject(subject, verb, node), Args: args}) + if err != nil { + envelope["error"] = err.Error() + } else { + envelope["result"] = result + } + return say() +} + +// EngineSeats is a link that serves this node's seat. Optional, as Asked is. +type EngineSeats interface { + ServeEngineSeat(EngineFunc) +} + +// ServeEngineSeat answers every verb of the seat on this node's subjects, as the service node-engine, which says each +// verb's seat, tool and node to discovery (novox/hq ADR 0197). A bus whose user list is older than the grant refuses the +// subscriptions, said by the client; a caller then hears nobody answer. +func (l *natsLink) ServeEngineSeat(handle EngineFunc) { + if handle == nil || l.conn == nil { + return + } + svc, err := micro.AddService(l.conn, micro.Config{Name: EngineSeat, Version: "1.0.0", + Description: "the node seat node-engine of " + l.node + " (novox/hq ADR 0563)", QueueGroupDisabled: true}) + if err != nil { + return + } + for _, v := range catalogue.EngineVerbs() { + verb := v.Name + answer := micro.HandlerFunc(func(r micro.Request) { + _ = r.Respond(AnswerEngineCall(handle, verb, l.node, r.Subject(), r.Data())) + }) + meta := map[string]string{"seat": EngineSeat, "tool": verb, "node": l.node} + _ = svc.AddEndpoint(verb, answer, micro.WithEndpointSubject(EngineToolSubject(verb, l.node)), + micro.WithEndpointMetadata(meta)) + _ = svc.AddEndpoint(verb+"-called", answer, micro.WithEndpointSubject(EngineCallPattern(verb, l.node)), + micro.WithEndpointMetadata(meta)) + } + l.engine = svc +} + +// SayEngine says one of the seat's events about this node, on the events stream. +func (l *natsLink) SayEngine(event string, body any) error { + raw, err := json.Marshal(body) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err = l.js.Publish(EngineEventSubject(event, l.node), raw, nats.Context(ctx)) + return err +} + +// The engine asks the operator under its own asker name (asks.EngineAsker) on the router's seat (ADR 0259 §3, as ADR +// 0563 §3 changes it): its ask and cancel on the seat's work stream, its warrants heard on its own decided subject, and +// its asks' state read from the router's records under its own name. + +// RouterRecords is the router's bucket of asks, which each asker reads under its own name: the state `asks` of the +// operator-channel's holder, the messenger (mesh-catalog modules/messenger). +const RouterRecords = "messenger_asks" + +// OperatorAsks is a link that asks the operator as this node's engine. Optional. +type OperatorAsks interface { + PublishAsk(subject string, body any, id string) error + RouterRecord(key string) (*stdio.StateEntry, error) + HearWarrants(asker string, heard func(asks.Warrant)) error +} + +// PublishAsk publishes an ask or a cancel on the router's seat, once per id within the stream's duplicate window. +func (l *natsLink) PublishAsk(subject string, body any, id string) error { + raw, err := json.Marshal(body) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err = l.js.Publish(subject, raw, nats.MsgId(id), nats.Context(ctx)) + return err +} + +// RouterRecord reads one key of the router's records, `.`; nil when it holds none. +func (l *natsLink) RouterRecord(key string) (*stdio.StateEntry, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + msg, err := l.conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+RouterRecords+".$KV."+RouterRecords+"."+key, nil) + if err != nil { + return nil, err + } + if status := msg.Header.Get("Status"); status == "404" { + return nil, nil + } else if status != "" { + return nil, fmt.Errorf("the router's record %s could not be read: %s %s", key, status, msg.Header.Get("Description")) + } + return &stdio.StateEntry{Key: key, Value: msg.Data}, nil +} + +// HearWarrants hears the warrants for asker on its decided subject, each handed to heard as it arrives. +func (l *natsLink) HearWarrants(asker string, heard func(asks.Warrant)) error { + sub, err := l.conn.Subscribe(asks.DecidedSubject(asker), func(msg *nats.Msg) { + var w asks.Warrant + if json.Unmarshal(msg.Data, &w) == nil { + heard(w) + } + }) + if err != nil { + return err + } + l.warrants = sub + return nil +} + +// errNoAsking is a link that cannot ask the operator. +var errNoAsking = errors.New("the link open now cannot ask the operator") + +// discoveryQuiet is how long the engine waits after the last answer to the bus's discovery, and discoveryMost the +// longest it waits in all. +const ( + discoveryQuiet = 300 * time.Millisecond + discoveryMost = 2 * time.Second +) + +// HeardServed asks the bus's discovery (novox/hq ADR 0197) whether a seat's verb is answered on this machine: an +// endpoint naming the seat, the verb as its tool and this node. +func (l *natsLink) HeardServed(seat, verb string) (bool, error) { + inbox := l.conn.NewRespInbox() + sub, err := l.conn.SubscribeSync(inbox) + if err != nil { + return false, err + } + defer func() { _ = sub.Unsubscribe() }() + if err := l.conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil { + return false, fmt.Errorf("asking the bus who serves what: %w", err) + } + heard := false + deadline := time.Now().Add(discoveryMost) + for time.Now().Before(deadline) { + msg, err := sub.NextMsg(discoveryQuiet) + if err != nil { + break + } + var info micro.Info + if json.Unmarshal(msg.Data, &info) != nil { + continue + } + for _, e := range info.Endpoints { + node := e.Metadata["node"] + if node == "" { + node = info.ID + } + if e.Metadata["seat"] == seat && e.Metadata["tool"] == verb && node == l.node { + heard = true + } + } + } + return heard, nil +} + +// HeardServed is the link open now asking the bus's discovery whether a seat's verb is answered on this machine. +func (q *Session) HeardServed(seat, verb string) (bool, error) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + asker, ok := bus.(interface { + HeardServed(string, string) (bool, error) + }) + if bus == nil || !ok { + return false, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked) + } + return asker.HeardServed(seat, verb) +} + +// ---- the engine's acts in the bus's own storage (ADR 0563 §2, §3) ------------------------------------------------- + +// actsStore is this node's keys of ActsBucket over the link open now: bound, never created (the grants part creates +// it, as the node-engine's declared state). +func (l *natsLink) actsStore() (nats.KeyValue, error) { + kv, err := l.js.KeyValue(ActsBucket) + if err != nil { + return nil, fmt.Errorf("the node-engine's state %s cannot be bound (it is created from the node-engine's "+ + "manifest; asked again once it is): %w", ActsBucket, err) + } + return kv, nil +} + +// BusState is the engine's ActStore: its own node's keys of ActsBucket, `.`, over whichever link the session +// has open; with none open every read and write is refused, and nothing is kept anywhere else. +type BusState struct { + Session *Session + Node string +} + +func (b BusState) kv() (nats.KeyValue, error) { + b.Session.init() + b.Session.mu.Lock() + bus := b.Session.bus + b.Session.mu.Unlock() + l, ok := bus.(interface{ actsStore() (nats.KeyValue, error) }) + if bus == nil || !ok { + return nil, fmt.Errorf("%w: this machine has no link to the bus open now, where its acts are kept", ErrNotAsked) + } + return l.actsStore() +} + +func (b BusState) key(k string) string { return b.Node + "." + k } + +// changed is a write that lost its compare-and-set, as stdio.ErrChanged. +func changed(err error) error { + var api *nats.APIError + if errors.Is(err, nats.ErrKeyExists) || (errors.As(err, &api) && api.ErrorCode == nats.JSErrCodeStreamWrongLastSequence) { + return stdio.ErrChanged + } + return err +} + +func (b BusState) Get(key string) (*stdio.StateEntry, error) { + kv, err := b.kv() + if err != nil { + return nil, err + } + e, err := kv.Get(b.key(key)) + if errors.Is(err, nats.ErrKeyNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return &stdio.StateEntry{Key: key, Value: e.Value(), Revision: e.Revision()}, nil +} + +func (b BusState) Create(key string, value any) (uint64, error) { + kv, err := b.kv() + if err != nil { + return 0, err + } + raw, err := json.Marshal(value) + if err != nil { + return 0, err + } + rev, err := kv.Create(b.key(key), raw) + return rev, changed(err) +} + +func (b BusState) Update(key string, value any, revision uint64) (uint64, error) { + kv, err := b.kv() + if err != nil { + return 0, err + } + raw, err := json.Marshal(value) + if err != nil { + return 0, err + } + rev, err := kv.Update(b.key(key), raw, revision) + return rev, changed(err) +} + +func (b BusState) Delete(key string) error { + kv, err := b.kv() + if err != nil { + return err + } + return kv.Delete(b.key(key)) +} + +// Keys are this node's keys under prefix, without the node: a watch of `.>` alone, the one filter the node's +// grant names, its values left out. +func (b BusState) Keys(prefix string) ([]string, error) { + kv, err := b.kv() + if err != nil { + return nil, err + } + w, err := kv.Watch(b.Node+".>", nats.MetaOnly(), nats.IgnoreDeletes()) + if err != nil { + return nil, err + } + defer func() { _ = w.Stop() }() + var out []string + for { + select { + case e := <-w.Updates(): + if e == nil { + sort.Strings(out) + return out, nil + } + if k := strings.TrimPrefix(e.Key(), b.Node+"."); strings.HasPrefix(k, prefix) { + out = append(out, k) + } + case <-time.After(10 * time.Second): + return nil, errors.New("the node-engine's acts could not be listed within 10s") + } + } +} diff --git a/internal/link/handacts.go b/internal/link/handacts.go new file mode 100644 index 0000000..a411516 --- /dev/null +++ b/internal/link/handacts.go @@ -0,0 +1,769 @@ +package link + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "strings" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asker" + "git.novox.be/novox/mesh-sdk/go/asks" +) + +// The node seat node-engine's acts (novox/hq ADR 0563). +// +// A hand-over and an access act are the engine's own asks, performed only on the warrant (§3): a call performs nothing. +// The engine judges every value as it does for the controller's signed ask, then refuses in words or asks the operator +// under its own asker name (asks.EngineAsker) at the level approve, binding the digest of the verb, the node, every +// argument and the owner, group and mode it will set. A warrant is claimed once by compare-and-set in the engine's own +// state (asker.Client.Take), held to the ask it answers, and to the act bound and the disk as judged, judged again now; +// only then is the act performed and recorded with the warrant. A decline, an expiry or a changed disk says refused and +// changes nothing. One ask is open per path or access. +// +// The read-only verbs need no warrant (§2): judge-directories changes nothing; setuid-search is bounded here, one at a +// time and at most one started in an hour. Every act, with who asked as the runtime proved it, is kept in the engine's own +// declared state acts (ActsBucket, created by the grants part), under its own node's keys alone, read by the verb acts. + +// The acts of the seat, as its verbs name them. +const ( + ActHandOver = "hand-over" + ActAccess = "access-act" + ActJudgeDirectories = "judge-directories" + ActSetuidSearch = "setuid-search" +) + +// The states of an act. +const ( + HandAsked = "asked" + HandDone = "done" + HandRefused = "refused" + HandAnswered = "answered" +) + +// SearchEvery is how often a setuid search may be started on a node (ADR 0563 §2): one in any hour. +const SearchEvery = time.Hour + +// askMargin is how long past an ask's expiry the router is given to say it ended, before the engine closes it itself. +const askMargin = 5 * time.Minute + +// ActsBucket is the node-engine's declared state acts (its manifest's state, ADR 0563 §2, §3), the bucket the grants +// part creates (mesh-sdk buscompose EngineActsBucket): keys `.act.`, `.open..` and +// `.ask.`, each written by this node's engine alone. +const ActsBucket = "mesh-host_acts" + +// Judged is what the engine judged of one act before it asks: what it is about, what it will set (the owner, group and +// mode, in words), and what the disk holds now (Disk), compared again when the warrant arrives. +type Judged struct { + // Subject is the path of a hand-over, or the access of an access act. + Subject string + // Path is the directory the act changes, as the engine states it. + Path string + // Sets is the owner, group and mode the act will set, in words; bound in the ask's digest. + Sets string + // Disk is what the engine read on disk when it judged: owner, group, mode and the file it is. + Disk string + // Words is what the operator is shown of it, without a path (ADR 0234 §6): the path is in the whole words alone. + Words string +} + +// HandAct is one act asked through the seat, as acts reads it. +type HandAct struct { + ID string `json:"id"` + Act string `json:"act"` + Subject string `json:"subject,omitempty"` + Caller string `json:"caller"` + Why string `json:"why,omitempty"` + At time.Time `json:"at"` + State string `json:"state"` + Said string `json:"said,omitempty"` + Ended time.Time `json:"ended,omitempty"` + // Ask is the operator's ask of a hand-over or an access act; Sets and Disk what was judged when it was asked. + Ask string `json:"ask,omitempty"` + Sets string `json:"sets,omitempty"` + Disk string `json:"disk,omitempty"` + Path string `json:"path,omitempty"` + // Expires is when the ask ends unanswered. + Expires time.Time `json:"expires,omitempty"` + // By, Via, Proofs are the operator's warrant, recorded with the act (ADR 0563 §3). + By string `json:"by,omitempty"` + Via string `json:"via,omitempty"` + Proofs []string `json:"proofs,omitempty"` + // Auto says the engine asked it itself, on finding a library's root holding data outside its group (§4). + Auto bool `json:"auto,omitempty"` +} + +// RootVerdict is the engine's answer to root-free (ADR 0563 §6), in the controller's words. +type RootVerdict struct { + Machine string `json:"machine"` + Free bool `json:"free"` + Why string `json:"why"` + Judged time.Time `json:"judged"` +} + +// HandActs is the engine's side of the seat. Every func is the engine's own judgement or act; nil refuses. +type HandActs struct { + Node string + // State is where the acts and the asks are kept: this node's keys of ActsBucket (BusState), each key one act. + State ActStore + // Asker asks the operator as asks.EngineAsker(Node), with State as its store. + Asker *asker.Client + // JudgeHandOver judges a hand-over's path, refusing in words; HandOver records it on the warrant. + JudgeHandOver func(path string) (Judged, error) + HandOver HandOverFunc + // JudgeAccess judges an access act on an access, refusing in words; AccessAct runs it on the warrant. + JudgeAccess func(act, access string) (Judged, error) + AccessAct AccessActFunc + // JudgeDirectories, SetuidSearch and Searching are the read-only verbs' and whether a search runs now. + JudgeDirectories JudgeDirectoriesFunc + SetuidSearch SetuidSearchFunc + Searching func() bool + // RootFree is this machine's root-free verdict now. + RootFree func() RootVerdict + // Say says one of the seat's events, asked, done or refused; nil says nothing. + Say func(event string, body any) + Now func() time.Time + // Nonce keeps two asks of one second apart; nil is random. + Nonce func() string + + mu sync.Mutex +} + +func (h *HandActs) now() time.Time { + if h.Now != nil { + return h.Now() + } + return time.Now() +} + +func (h *HandActs) nonce() string { + if h.Nonce != nil { + return h.Nonce() + } + var b [2]byte + _, _ = rand.Read(b[:]) + return hex.EncodeToString(b[:]) +} + +func (h *HandActs) say(event string, a HandAct) { + if h.Say != nil { + h.Say(event, map[string]any{"act": a.Act, "subject": a.Subject, "caller": a.Caller, "ask": a.Ask, + "state": a.State, "said": a.Said, "node": h.Node}) + } +} + +// Call is the seat's one entry: a verb, the caller as the runtime proved it, and its arguments. +func (h *HandActs) Call(c EngineCall) (any, error) { + arg := func(k string) string { s, _ := c.Args[k].(string); return strings.TrimSpace(s) } + switch c.Verb { + case ActHandOver: + return h.AskHandOver(c.Caller, arg("path"), arg("why"), false) + case ActAccess: + return h.AskAccess(c.Caller, arg("act"), arg("access"), arg("why"), false) + case ActJudgeDirectories: + return h.Judge(c.Caller) + case ActSetuidSearch: + return h.Search(c.Caller, arg("why")) + case "root-free": + if h.RootFree == nil { + return nil, errors.New("this engine judges no machine root-free") + } + return map[string]any{"machines": []RootVerdict{h.RootFree()}}, nil + case "acts": + if id := arg("id"); id != "" { + return h.Act(id) + } + limit, _ := strconv.Atoi(arg("limit")) + if f, ok := c.Args["limit"].(float64); ok { + limit = int(f) + } + return h.Acts(limit) + } + return nil, fmt.Errorf("the node seat %s serves no verb %q", EngineSeat, c.Verb) +} + +// ---- the record -------------------------------------------------------------------------------------------------- + +// ActStore is the engine's own state, this node's keys alone (BusState prefixes them with the node): each act one key, +// written by compare-and-set on its revision, an open ask one key made by a create. +type ActStore interface { + Get(key string) (*stdio.StateEntry, error) + Create(key string, value any) (uint64, error) + Update(key string, value any, revision uint64) (uint64, error) + Delete(key string) error + // Keys are the keys under prefix. + Keys(prefix string) ([]string, error) +} + +func actKey(id string) string { return "act." + id } + +// openKey is the one key an open ask of act on subject holds: made by a create, so a second ask of it is refused. +func openKey(act, subject string) string { + sum := sha256.Sum256([]byte(subject)) + return "open." + act + "." + hex.EncodeToString(sum[:12]) +} + +// read is every act kept, oldest first. +func (h *HandActs) read() ([]HandAct, error) { + keys, err := h.State.Keys("act.") + if err != nil { + return nil, fmt.Errorf("the engine's acts cannot be listed: %w", err) + } + var out []HandAct + for _, k := range keys { + a, _, err := h.readOne(k) + if err != nil { + return nil, err + } + if a != nil { + out = append(out, *a) + } + } + sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) }) + return out, nil +} + +func (h *HandActs) readOne(key string) (*HandAct, uint64, error) { + e, err := h.State.Get(key) + if err != nil || e == nil { + return nil, 0, err + } + var a HandAct + if err := json.Unmarshal(e.Value, &a); err != nil { + return nil, 0, fmt.Errorf("the engine's act %s cannot be read: %w", key, err) + } + return &a, e.Revision, nil +} + +// judgedKey keeps the node's judgements and refused searches, the last keptJudged of them: any caller may ask them, so +// they roll in one key and never add one (ADR 0563 §2: a wrong caller gains nothing, and fills no bucket). +const ( + judgedKey = "judged" + keptJudged = 50 +) + +// roll adds a to the rolling key by compare-and-set, the oldest falling out. +func (h *HandActs) roll(a HandAct) error { + for range 5 { + e, err := h.State.Get(judgedKey) + if err != nil { + return err + } + var kept []HandAct + if e != nil { + if err := json.Unmarshal(e.Value, &kept); err != nil { + kept = nil // an unreadable roll is started again, never a reason to refuse + } + } + kept = append(kept, a) + if len(kept) > keptJudged { + kept = kept[len(kept)-keptJudged:] + } + if e == nil { + _, err = h.State.Create(judgedKey, kept) + } else { + _, err = h.State.Update(judgedKey, kept, e.Revision) + } + if !errors.Is(err, stdio.ErrChanged) { + return err + } + } + return errors.New("the engine's judgements kept changing under its write") +} + +func (h *HandActs) rolled() ([]HandAct, error) { + e, err := h.State.Get(judgedKey) + if err != nil || e == nil { + return nil, err + } + var kept []HandAct + if err := json.Unmarshal(e.Value, &kept); err != nil { + return nil, fmt.Errorf("the engine's judgements cannot be read: %w", err) + } + return kept, nil +} + +func (h *HandActs) record(a HandAct) error { + _, err := h.State.Create(actKey(a.ID), a) + return err +} + +// end marks the open act of an ask ended, once, by compare-and-set on its revision, and frees its open key: false when +// it was not open. +func (h *HandActs) end(ask string, set func(*HandAct)) (HandAct, bool, error) { + for range 5 { + a, rev, err := h.readOne(actKey(ask)) + if err != nil || a == nil || a.State != HandAsked { + return HandAct{}, false, err + } + set(a) + a.Ended = h.now().UTC() + if _, err := h.State.Update(actKey(ask), *a, rev); errors.Is(err, stdio.ErrChanged) { + continue + } else if err != nil { + return HandAct{}, false, err + } + _ = h.State.Delete(openKey(a.Act, a.Subject)) + return *a, true, nil + } + return HandAct{}, false, errors.New("the engine's act " + ask + " kept changing under its write") +} + +// Act is the verb acts with an id: that one act. +func (h *HandActs) Act(id string) (any, error) { + a, _, err := h.readOne(actKey(id)) + if err != nil { + return nil, err + } + if a == nil { + return nil, fmt.Errorf("this engine keeps no act %q", id) + } + return map[string]any{"node": h.Node, "acts": []HandAct{*a}}, nil +} + +// Acts is the verb acts: every act kept, newest first, at most limit when given. +func (h *HandActs) Acts(limit int) (any, error) { + acts, err := h.read() + if err != nil { + return nil, err + } + judged, err := h.rolled() + if err != nil { + return nil, err + } + acts = append(acts, judged...) + sort.SliceStable(acts, func(i, j int) bool { return acts[i].At.Before(acts[j].At) }) + out := make([]HandAct, 0, len(acts)) + for i := len(acts) - 1; i >= 0; i-- { + out = append(out, acts[i]) + } + if limit > 0 && len(out) > limit { + out = out[:limit] + } + return map[string]any{"node": h.Node, "acts": out}, nil +} + +// ---- the read-only verbs (§2) ------------------------------------------------------------------------------------- + +// Judge is judge-directories: the engine's judgement, read-only, recorded with its caller. +func (h *HandActs) Judge(caller string) (any, error) { + if h.JudgeDirectories == nil { + return nil, errors.New("this engine judges no directory") + } + said, err := h.JudgeDirectories(caller) + a := HandAct{ID: h.id(ActJudgeDirectories), Act: ActJudgeDirectories, Caller: caller, At: h.now().UTC(), State: HandAnswered} + if err != nil { + a.State, a.Said = HandRefused, err.Error() + } + if rerr := h.roll(a); rerr != nil && err == nil { + return nil, rerr + } + if err != nil { + return nil, err + } + return map[string]any{"node": h.Node, "said": said}, nil +} + +// Search is setuid-search: refused while one runs and when one was started within the hour, naming when the next may +// start; otherwise the last search is thrown away and a full one starts, recorded with its caller. +func (h *HandActs) Search(caller, why string) (any, error) { + if h.SetuidSearch == nil { + return nil, errors.New("this engine keeps no search for setuid programs") + } + h.mu.Lock() + defer h.mu.Unlock() + now := h.now().UTC() + if h.Searching != nil && h.Searching() { + return nil, errors.New("a search for setuid programs runs on this machine now; no second is started while it does") + } + // The last start is a key of its own, claimed by compare-and-set before the search starts: no number of other acts, + // nor the record of acts, moves it (ADR 0563 §2). + if err := h.claimSearch(now, caller); err != nil { + return nil, err + } + said, err := h.SetuidSearch(caller) + a := HandAct{ID: h.id(ActSetuidSearch), Act: ActSetuidSearch, Caller: caller, Why: why, At: now, State: HandDone, Said: said} + if err != nil { + a.State, a.Said = HandRefused, err.Error() + } + // A search started is bounded to one an hour and kept as an act of its own; a refused one, which any caller may + // cause, rolls with the judgements. + record := h.record + if err != nil { + record = h.roll + } + if rerr := record(a); rerr != nil { + return nil, rerr + } + if err != nil { + return nil, err + } + return map[string]any{"node": h.Node, "said": said}, nil +} + +// searchKey keeps the last search's start on this node, and libKey each library's own take ask: keys of their own, which +// no other act's key moves, kept for the bucket's age (ninety days, the node-engine's manifest). +const searchKey = "search.last" + +func libKey(access string) string { + sum := sha256.Sum256([]byte(access)) + return "lib." + hex.EncodeToString(sum[:12]) + ".asked" +} + +// started is the last search's start as kept, and who asked it. +type started struct { + At time.Time `json:"at"` + By string `json:"by"` +} + +// claimSearch refuses a search within SearchEvery of the last start, naming when the next may start, and otherwise +// claims this one by compare-and-set: a second claim at the same moment loses. +func (h *HandActs) claimSearch(now time.Time, caller string) error { + e, err := h.State.Get(searchKey) + if err != nil { + return err + } + var rev uint64 + if e != nil { + var last started + if err := json.Unmarshal(e.Value, &last); err != nil { + return fmt.Errorf("the last search's start cannot be read, so none is started: %w", err) + } + if now.Sub(last.At) < SearchEvery { + return fmt.Errorf("a search was started at %s by %s, and one is started in any hour: the next may start "+ + "at %s", last.At.Format(time.RFC3339), last.By, last.At.Add(SearchEvery).Format(time.RFC3339)) + } + rev = e.Revision + } + claim := started{At: now, By: caller} + if rev == 0 { + _, err = h.State.Create(searchKey, claim) + } else { + _, err = h.State.Update(searchKey, claim, rev) + } + if errors.Is(err, stdio.ErrChanged) { + return errors.New("another search was started at this moment; no second is started") + } + return err +} + +// ---- the asks (§3, §4) -------------------------------------------------------------------------------------------- + +func (h *HandActs) id(act string) string { + return act + "-" + h.now().UTC().Format("20060102t150405") + "-" + h.nonce() +} + +// bound is the act an ask's yes binds: the verb, the node, every argument and what it will set (ADR 0563 §3). +func (h *HandActs) bound(a HandAct) asks.Act { + act := asks.Act{"verb": a.Act, "node": h.Node, "arg.subject": a.Subject, "arg.why": a.Why, "sets": a.Sets} + if a.Path != "" { + act["path"] = a.Path + } + return act +} + +// AskHandOver is hand-over: judged, refused in words or asked of the operator. auto is the engine asking by itself. +func (h *HandActs) AskHandOver(caller, path, why string, auto bool) (any, error) { + if h.JudgeHandOver == nil || h.HandOver == nil { + return nil, errors.New("this engine takes no hand-over") + } + j, err := h.JudgeHandOver(path) + if err != nil { + return nil, err + } + return h.ask(ActHandOver, caller, why, auto, j) +} + +// AskAccess is access-act: the act take on an access, judged on the library's home, refused or asked of the operator. +func (h *HandActs) AskAccess(caller, act, access, why string, auto bool) (any, error) { + if h.JudgeAccess == nil || h.AccessAct == nil { + return nil, errors.New("this engine runs no act on an accessed folder") + } + if act != ActTake { + return nil, fmt.Errorf("the act asked is %q, and the one act on an access is %q", act, ActTake) + } + j, err := h.JudgeAccess(act, access) + if err != nil { + return nil, err + } + return h.ask(ActAccess, caller, why, auto, j) +} + +// AskedFor says whether an act on subject was ever asked by the engine itself: a library's take is asked by itself once +// (ADR 0563 §4), and again only by hand. +func (h *HandActs) AskedFor(act, subject string) bool { + if act == ActAccess { + e, err := h.State.Get(libKey(subject)) + return err != nil || e != nil // unread is never a reason to ask again + } + acts, err := h.read() + if err != nil { + return true + } + for _, a := range acts { + if a.Act == act && a.Subject == subject && (a.Auto || a.State == HandAsked) { + return true + } + } + return false +} + +func (h *HandActs) ask(act, caller, why string, auto bool, j Judged) (any, error) { + if h.Asker == nil { + return nil, errors.New("this engine reaches no operator to ask: nothing was asked and nothing changed") + } + h.mu.Lock() + defer h.mu.Unlock() + now := h.now().UTC() + a := HandAct{ID: h.id(act), Act: act, Subject: j.Subject, Path: j.Path, Caller: caller, Why: why, At: now, + State: HandAsked, Sets: j.Sets, Disk: j.Disk, Expires: now.Add(asks.ApproveLasts - time.Minute), Auto: auto} + a.Ask = a.ID + // One ask open per path or access: its open key made by a create, which a second ask cannot make. + if _, err := h.State.Create(openKey(act, j.Subject), map[string]string{"ask": a.Ask}); errors.Is(err, stdio.ErrChanged) { + held := "another" + if e, _ := h.State.Get(openKey(act, j.Subject)); e != nil { + var o struct { + Ask string `json:"ask"` + } + if json.Unmarshal(e.Value, &o) == nil && o.Ask != "" { + held = o.Ask + } + } + return nil, fmt.Errorf("%s of %s is asked already: the ask %s waits for the operator's answer", act, j.Subject, held) + } else if err != nil { + return nil, err + } + if err := h.record(a); err != nil { + _ = h.State.Delete(openKey(act, j.Subject)) + return nil, err + } + bind, err := asks.ActDigest(h.bound(a)) + if err != nil { + return nil, err + } + headline, does, stays := "Hand a directory on "+h.Node+" to the mesh?", "The mesh gives it "+j.Sets+".", + "The directory stays as it was found." + if act == ActAccess { + headline, does, stays = "Keep the library "+strings.TrimPrefix(j.Subject, "library:")+" on "+h.Node+"?", + "The library's group, group write and setgid are set on everything under its root.", + "The library waits and nothing under its root changes." + } + if len([]rune(headline)) > asks.HeadlineLength { + headline = string([]rune(headline)[:asks.HeadlineLength]) + } + asked := asks.Ask{ID: a.Ask, Headline: headline, Who: asks.Operator, + Explanation: j.Words + " Your yes sets " + j.Sets + "; nothing changes until you answer.", + Whole: j.Words + " (" + j.Path + ") Your yes sets " + j.Sets + "; nothing changes until you answer.", + Details: "Asked by " + caller + " on " + h.Node + orWhy(why), + Options: []asks.Option{{ID: "yes", Label: "Yes", Does: does, Level: asks.Approve, Binds: bind}, + {ID: "no", Label: "No, leave it", Does: stays, Level: asks.Acknowledge}}, + Expires: a.Expires, OnExpiry: stays, About: EngineSeat + "-" + h.Node + "-" + act} + if err := h.Asker.Ask(asked); err != nil { + _, _, _ = h.end(a.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, "the operator could not be asked: "+err.Error() }) + return nil, fmt.Errorf("the operator could not be asked, and nothing changed: %w", err) + } + // Once asked, a library's own take ask is kept's own take ask: once, in a key of its own (ADR 0563 §4). + if auto && act == ActAccess { + _, _ = h.State.Create(libKey(j.Subject), map[string]any{"ask": a.Ask, "at": now}) + } + h.say(HandAsked, a) + return map[string]any{"asked": a.Ask, "node": h.Node, "says": "the operator is asked on their channel; only " + + "their yes changes anything. acts shows where it stands."}, nil +} + +func orWhy(why string) string { + if why == "" { + return "" + } + return ": " + why +} + +// Warranted is the act on the operator's answer, handed once by asker.Client.Take after it held the warrant to the +// ask: a no is recorded refused; a yes is held to the act the option bound and the disk as judged, judged again now, +// and only then performed and recorded done with the warrant. +func (h *HandActs) Warranted(o asks.Option, w asks.Warrant) error { + a, _, err := h.readOne(actKey(w.Ask)) + if err != nil { + return err + } + if a == nil || a.State != HandAsked { + return fmt.Errorf("the warrant answers the ask %s, which is not open here: nothing is done", w.Ask) + } + refuse := func(why string) error { + ended, ok, err := h.end(w.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, why; warrantOn(x, w) }) + if err == nil && ok { + h.say(HandRefused, ended) + } + return err + } + if o.ID != "yes" { + return refuse("the operator said no") + } + var j Judged + switch a.Act { + case ActHandOver: + j, err = h.JudgeHandOver(a.Subject) + case ActAccess: + j, err = h.JudgeAccess(ActTake, a.Subject) + default: + err = fmt.Errorf("the act %s is not performed on a warrant", a.Act) + } + if err != nil { + return refuse("judged again on the operator's yes, it is refused: " + err.Error()) + } + now := *a + now.Sets, now.Path = j.Sets, j.Path + if err := o.Performs(h.bound(now)); err != nil { + return refuse("what the yes bound is not what would be done now: " + err.Error()) + } + if j.Disk != a.Disk { + return refuse("the disk changed since the operator was asked (" + a.Disk + ", now " + j.Disk + "): ask again") + } + // Recorded with the way it came (ADR 0563 §5): the operator's yes to this engine's ask, not a terminal's line. + by := operatorOf(w) + ", to " + EngineSeat + "'s ask " + w.Ask + var said string + switch a.Act { + case ActHandOver: + said, err = h.HandOver(a.Subject, by) + case ActAccess: + said, err = h.AccessAct(AccessAct{Act: ActTake, Access: a.Subject, Path: j.Path, Warrant: w.Ask, By: by}) + } + if err != nil { + return refuse(err.Error()) + } + ended, ok, err := h.end(w.Ask, func(x *HandAct) { x.State, x.Said = HandDone, said; warrantOn(x, w) }) + if err == nil && ok { + h.say(HandDone, ended) + } + return err +} + +func warrantOn(a *HandAct, w asks.Warrant) { + a.Proofs = w.Proofs + if w.By != nil { + a.By, a.Via = w.By.Who, w.By.Kind + if w.By.Verified != "" { + a.Via += " (" + w.By.Verified + ")" + } + } +} + +func operatorOf(w asks.Warrant) string { + if w.By == nil { + return "the operator" + } + return "the " + w.By.Who + " via " + w.By.Kind +} + +// Heard is a warrant heard on the engine's decided subject: taken once (asker.Client.Take), and an ask that ended with +// no choice recorded refused. +func (h *HandActs) Heard(w asks.Warrant) { + if h.Asker == nil { + return + } + // A warrant the ask kept here does not hold to (another digest, another option, too late) ends nothing: it is no + // answer to the ask, which stays open for the operator's. + if handled, _ := h.Asker.Take(w, h.Warranted); handled == asker.Recorded { + why := "the ask ended " + string(w.Outcome) + if w.Words != "" { + why += ": " + w.Words + } + if ended, ok, _ := h.end(w.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, why; warrantOn(x, w) }); ok { + h.say(HandRefused, ended) + } + } +} + +// Reconcile catches up what the engine missed while it was away: a warrant the router holds for an open ask is taken, +// and an ask past its expiry and a margin is closed refused, expired. +func (h *HandActs) Reconcile() { + if h.Asker == nil { + return + } + acts, err := h.read() + if err != nil { + return + } + for _, a := range acts { + if a.State != HandAsked || a.Ask == "" { + continue + } + if _, w, err := h.Asker.Where(a.Ask); err == nil && w != nil { + h.Heard(*w) + continue + } + if !a.Expires.IsZero() && h.now().After(a.Expires.Add(askMargin)) { + if ended, ok, _ := h.end(a.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, "the ask expired unanswered" }); ok { + h.say(HandRefused, ended) + } + } + } +} + +// MemState is an ActStore in memory: an engine's acts while no bus is reached in a test. +type MemState struct { + mu sync.Mutex + keys map[string]stdio.StateEntry + top uint64 +} + +func (m *MemState) Get(key string) (*stdio.StateEntry, error) { + m.mu.Lock() + defer m.mu.Unlock() + if e, ok := m.keys[key]; ok { + return &e, nil + } + return nil, nil +} + +func (m *MemState) Create(key string, value any) (uint64, error) { return m.put(key, value, 0, true) } + +func (m *MemState) Update(key string, value any, revision uint64) (uint64, error) { + return m.put(key, value, revision, false) +} + +func (m *MemState) put(key string, value any, revision uint64, create bool) (uint64, error) { + raw, err := json.Marshal(value) + if err != nil { + return 0, err + } + m.mu.Lock() + defer m.mu.Unlock() + was, kept := m.keys[key] + if (create && kept) || (!create && (!kept || was.Revision != revision)) { + return 0, stdio.ErrChanged + } + if m.keys == nil { + m.keys = map[string]stdio.StateEntry{} + } + m.top++ + m.keys[key] = stdio.StateEntry{Key: key, Value: raw, Revision: m.top} + return m.top, nil +} + +func (m *MemState) Delete(key string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.keys, key) + return nil +} + +func (m *MemState) Keys(prefix string) ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []string + for k := range m.keys { + if strings.HasPrefix(k, prefix) { + out = append(out, k) + } + } + sort.Strings(out) + return out, nil +} diff --git a/internal/link/handover.go b/internal/link/handover.go index 1c5188b..e6103f0 100644 --- a/internal/link/handover.go +++ b/internal/link/handover.go @@ -76,9 +76,13 @@ const MaxHandOverBy = 200 // HandOverCheck is what an ask is held to: the mesh's key, this node's name, the clock, and the nonces taken. type HandOverCheck struct { Signer ed25519.PublicKey - Node string - Now func() time.Time - Nonces *Nonces + // SignerNow, when set, is the controller's key as the engine trusts it at this ask, read per ask in place of Signer: + // nil once the controller's lease is handed away, which refuses every ask signed by its key (novox/hq ADR 0563 §7, + // ADR 0550 §2). + SignerNow func() ed25519.PublicKey + Node string + Now func() time.Time + Nonces *Nonces } // Nonces are the nonces of asks taken, each kept until its ask expires: an ask is taken once. Kept in memory — @@ -157,8 +161,14 @@ func believedBytesWithin(check HandOverCheck, context string, withPath bool, bod if err := json.Unmarshal(body, &signed); err != nil { return HandOverAsk{}, nil, "the ask is not a signed ask: " + err.Error() } - if len(check.Signer) != ed25519.PublicKeySize || - !ed25519.Verify(check.Signer, append([]byte(context), signed.Ask...), signed.Signature) { + signer := check.Signer + if check.SignerNow != nil { + if signer = check.SignerNow(); signer == nil { + return HandOverAsk{}, nil, "the controller's lease is handed away, and no ask signed by its key is taken" + } + } + if len(signer) != ed25519.PublicKeySize || + !ed25519.Verify(signer, append([]byte(context), signed.Ask...), signed.Signature) { return HandOverAsk{}, nil, "the ask is not signed by the mesh" } var ask HandOverAsk diff --git a/internal/link/handover_test.go b/internal/link/handover_test.go index 3442a33..a19d009 100644 --- a/internal/link/handover_test.go +++ b/internal/link/handover_test.go @@ -9,6 +9,9 @@ import ( "strings" "testing" "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "github.com/nats-io/nats.go" ) // The hand-over's ask and answer hold these field names; the controller's side holds the same list @@ -145,6 +148,21 @@ func TestAHandOverAskIsBelievedOnlySignedForThisNodeUnexpiredAndOnce(t *testing. if strings.Join(given, "; ") != "/srv/notes by jo through mesh-cli on anchor; /refused by jo through mesh-cli on anchor" { t.Fatalf("the handler was given %v", given) } + // ADR 0563 §7, ADR 0550 §2: after the lease write the engine answers no ask signed by the controller's key. The key + // is read per ask from what the engine trusts now; nil (the lease handed away) refuses, and nothing is handed over. + trusted := m.Signer + following := HandOverCheck{Signer: m.Signer, SignerNow: func() ed25519.PublicKey { return trusted }, Node: m.Node, + Now: func() time.Time { return now }, Nonces: &Nonces{}} + before := len(given) + trusted = nil + late := signedAsk(t, key, with(func(a *HandOverAsk) { a.Expires = now.Add(time.Minute); a.Nonce = strings.Repeat("cd", 16) })) + if a := AnswerHandOver(following, handle, late); !strings.Contains(a.Refused, "lease is handed away") || len(given) != before { + t.Fatalf("after the lease write an ask signed by the controller was answered %+v", a) + } + trusted = m.Signer + if a := AnswerHandOver(following, handle, late); a.Refused != "" || len(given) != before+1 { + t.Fatalf("while the controller holds the lease its ask was answered %+v", a) + } // A nonce is forgotten once its ask has expired, so the set does not grow for ever. now = now.Add(HandOverGood + time.Second) check.Nonces.take("ff", now.Add(time.Second), now) @@ -174,17 +192,23 @@ func TestAHandOverIsAnsweredOnlyWhenItCarriesAReply(t *testing.T) { // handingLink is a quiet link that hears hand-overs, keeping the handler it was given. type handingLink struct { *quietLink - given chan HandOverFunc + given chan HandOverFunc + signer chan func() ed25519.PublicKey } func (l handingLink) HearHandOvers(h HandOverFunc) { l.given <- h } +// FollowSigner keeps what the link is told to verify the controller's asks with. +func (l handingLink) FollowSigner(f func() ed25519.PublicKey) { l.signer <- f } + // **The link is given the queue's hand-over** when it is served: what the controller asks reaches the engine's // judgement, and nothing else. func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) { m, _ := aMember(t) - l := handingLink{quietLink: newQuietLink(), given: make(chan HandOverFunc, 1)} + l := handingLink{quietLink: newQuietLink(), given: make(chan HandOverFunc, 1), signer: make(chan func() ed25519.PublicKey, 1)} q := &Session{Membership: m, Timeout: time.Second} + // The link verifies the controller's asks with the key the engine trusts now (ADR 0563 §7). + q.ControllerKey = func() ed25519.PublicKey { return nil } q.HandOver = func(path, by string) (string, error) { return "the queue's: " + path + " by " + by, nil } ctx, stop := context.WithCancel(context.Background()) served := make(chan error, 1) @@ -197,8 +221,40 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) { case <-time.After(5 * time.Second): t.Fatal("the link was never given the queue's hand-over") } + select { + case f := <-l.signer: + if f() != nil { + t.Fatal("the link was given another key than the queue's") + } + case <-time.After(5 * time.Second): + t.Fatal("the link was never given the key the engine trusts now") + } stop() <-served + // 774 (ADR 0563 §1): a call of the node seat reaches its verb with its caller as the subject proves it, unproven on + // the subject that names none; a verb the seat does not serve is refused, and the answer is a tool's envelope. + t.Run("a call of the node seat reaches its verb with its proven caller", func(t *testing.T) { + h := &HandActs{Node: "n1", State: &MemState{}, + RootFree: func() RootVerdict { return RootVerdict{Machine: "n1", Free: true, Why: "judged"} }, + JudgeDirectories: func(by string) (string, error) { return "judged for " + by, nil }} + for _, row := range []struct{ verb, subject, want string }{ + {"judge-directories", "mesh.seat.node-engine.call.judge-directories.n1.shanks~node-tools", `"said":"judged for shanks.node-tools"`}, + {"judge-directories", EngineToolSubject("judge-directories", "n1"), `"said":"judged for unproven"`}, + {"judge-directories", "mesh.seat.node-engine.call.judge-directories.n2.shanks~node-tools", `"said":"judged for unproven"`}, + {"root-free", "mesh.seat.node-engine.call.root-free.n1.ace~messenger", `"free":true`}, + {"unserved", "mesh.seat.node-engine.call.unserved.n1.x", `"error":"the node seat node-engine serves no verb \"unserved\""`}, + } { + if got := string(AnswerEngineCall(h.Call, row.verb, "n1", row.subject, []byte(`{}`))); !strings.Contains(got, row.want) || + !strings.Contains(got, `"node":"n1"`) { + t.Errorf("%s on %s: %s", row.verb, row.subject, got) + } + } + acts, _ := h.Acts(0) + if got := acts.(map[string]any)["acts"].([]HandAct); len(got) != 3 || got[0].Caller != "unproven" || + got[2].Caller != "shanks.node-tools" || got[2].Act != ActJudgeDirectories { + t.Errorf("the judgements are recorded as %+v", got) + } + }) } // **Against a real server**: a signed request on the node's own subject is answered; one on another node's is @@ -237,4 +293,63 @@ func TestNatsAHandOverIsAskedOfTheNode(t *testing.T) { if len(handled) != 1 { t.Fatalf("handled %d asks; want only the one with a reply", len(handled)) } + // 774 (ADR 0563 §2, §3): the engine's acts are its node's keys of its declared state on the bus, each written by + // compare-and-set: a second create and a stale revision lose, and another node's keys are not this node's. + t.Run("the acts are kept in the bus's own storage under this node's keys", func(t *testing.T) { + _ = js.DeleteKeyValue(ActsBucket) + if _, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: ActsBucket}); err != nil { + t.Fatal(err) + } + q := &Session{Membership: m, Timeout: time.Second} + q.bus = l + mine, theirs := BusState{Session: q, Node: m.Node}, BusState{Session: q, Node: "another"} + rev, err := mine.Create("act.a-1", map[string]string{"state": "asked"}) + if err != nil { + t.Fatal(err) + } + if _, err := mine.Create("act.a-1", map[string]string{}); !errors.Is(err, stdio.ErrChanged) { + t.Errorf("a second create: %v", err) + } + if _, err := mine.Update("act.a-1", map[string]string{"state": "done"}, rev+7); !errors.Is(err, stdio.ErrChanged) { + t.Errorf("an update at a stale revision: %v", err) + } + if _, err := mine.Update("act.a-1", map[string]string{"state": "done"}, rev); err != nil { + t.Errorf("an update at its revision: %v", err) + } + _, _ = theirs.Create("act.b-1", map[string]string{}) + if keys, err := mine.Keys("act."); err != nil || len(keys) != 1 || keys[0] != "act.a-1" { + t.Errorf("this node's acts are %v, %v", keys, err) + } + if e, _ := js.KeyValue(ActsBucket); e != nil { + if got, err := e.Get(m.Node + ".act.a-1"); err != nil || !strings.Contains(string(got.Value()), "done") { + t.Errorf("the bucket holds %v, %v", got, err) + } + } + if err := mine.Delete("act.a-1"); err != nil { + t.Fatal(err) + } + if e, err := mine.Get("act.a-1"); err != nil || e != nil { + t.Errorf("a deleted act reads %v, %v", e, err) + } + }) + // 774 (ADR 0563 §1): the node seat is served on this node's subjects alone, naming its caller, and said to discovery. + t.Run("the node seat is served on this node and discovered", func(t *testing.T) { + l.ServeEngineSeat(func(c EngineCall) (any, error) { return c.Verb + " for " + c.Caller, nil }) + reply, err := conn.Request("mesh.seat.node-engine.call.acts."+m.Node+".shanks~node-tools", []byte(`{}`), 5*time.Second) + if err != nil || !strings.Contains(string(reply.Data), `"result":"acts for shanks.node-tools"`) { + t.Fatalf("the call answered %v, %v", reply, err) + } + if _, err := conn.Request(EngineToolSubject("acts", "another"), []byte(`{}`), 300*time.Millisecond); err == nil { + t.Fatal("another node's verb was answered here") + } + for seat, want := range map[string]bool{EngineSeat: true, "node-login-shell": false} { + verb := "acts" + if !want { + verb = "execute" + } + if heard, err := l.HeardServed(seat, verb); err != nil || heard != want { + t.Errorf("discovery hears %s.%s here: %v, %v", seat, verb, heard, err) + } + } + }) } diff --git a/internal/link/have_test.go b/internal/link/have_test.go index 4280811..86f5a95 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -274,7 +274,8 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) { // 1126 (ADR 0555 §3): with its keys, the node key carries the node's record: the profile's present capabilities and // networks, the outward links, the health's contract and the public keys; a report that changes none of it writes // nothing, a changed fact writes the key once, and a report without a profile or health leaves them standing. - keys := RecordKeys{TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert"} + keys := RecordKeys{TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert", + Trusts: []string{"controller key sha256:aa", "vault key sha256:bb"}} // and what it trusts (L3 s6, ADR 0550 §2) q.Keys = func() RecordKeys { return keys } rep.Profile = map[string]any{"capabilities": []any{map[string]any{"name": "systemd", "present": true}, map[string]any{"name": "docker", "present": false}, map[string]any{"name": "apt", "present": true}}, @@ -286,7 +287,7 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) { t.Fatalf("the node key holds no record: %+v, %v", got, err) } wantRecord := delivery.NodeRecord{V: delivery.Version, Node: "n1", TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", - Certificate: "cert", Capabilities: []string{"apt", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true, + Certificate: "cert", Trusts: keys.Trusts, Capabilities: []string{"apt", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true, Networks: []catalogue.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, Contract: NodeKeyContract} if !delivery.SameNodeRecord(*got.Record, wantRecord) { t.Errorf("record = %+v, want %+v", *got.Record, wantRecord) diff --git a/internal/link/hearing_nats.go b/internal/link/hearing_nats.go index da68fa2..9bd0cde 100644 --- a/internal/link/hearing_nats.go +++ b/internal/link/hearing_nats.go @@ -6,9 +6,11 @@ import ( "errors" "fmt" "strings" + "sync" "time" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/micro" ) // The host's link on the bus: one connection, and what is asked of this node over it. The node hears no declaration @@ -35,11 +37,14 @@ func natsURL(address string) string { // natsLink is this node's connection as a JetStream subscription. type natsLink struct { - conn *nats.Conn - js nats.JetStreamContext - asking *nats.Subscription - asked chan struct{} - handing *nats.Subscription + // mu guards signerNow (FollowSigner). + mu sync.Mutex + signerNow func() ed25519.PublicKey + conn *nats.Conn + js nats.JetStreamContext + asking *nats.Subscription + asked chan struct{} + handing *nats.Subscription // searching hears the controller's terminal asking for a fresh setuid search (novox/hq issue 361). searching *nats.Subscription // judging hears the controller asking for a read-only judgement of this node's directories (issue 358). @@ -48,7 +53,10 @@ type natsLink struct { acting *nats.Subscription // moving hears the controller asking to move a value sealed to this machine into the vault (issue 676). moving *nats.Subscription - node string + // engine serves the node seat node-engine, and warrants hears the operator's answers to its asks (ADR 0563). + engine micro.Service + warrants *nats.Subscription + node string // signer is the mesh's key, which a hand-over ask is verified with as a declaration is (issue 356). signer ed25519.PublicKey lost chan error @@ -141,6 +149,21 @@ func (l *natsLink) hearAsks() { } } +// FollowSigner has every signed ask heard on this link verified with the key the engine trusts at that ask, the +// controller's while it holds its lease and none from its hand-over (novox/hq ADR 0563 §7, ADR 0550 §2). +func (l *natsLink) FollowSigner(f func() ed25519.PublicKey) { + l.mu.Lock() + l.signerNow = f + l.mu.Unlock() +} + +// check is how a signed ask is believed on this link: the static membership signer until FollowSigner names the trust. +func (l *natsLink) check() HandOverCheck { + l.mu.Lock() + defer l.mu.Unlock() + return HandOverCheck{Signer: l.signer, SignerNow: l.signerNow, Node: l.node} +} + // HearHandOvers listens for the controller asking this node to hand a directory used as found to the mesh // (novox/hq issue 356): a request on this machine's own subject, answered once on the reply it carried. The // ask is verified with the mesh's key before anything is read out of it (AnswerHandOver), because a responder @@ -151,7 +174,7 @@ func (l *natsLink) HearHandOvers(handle HandOverFunc) { if handle == nil { return } - check := HandOverCheck{Signer: l.signer, Node: l.node} + check := l.check() handing, err := l.conn.Subscribe(AskHandOverSubject(l.node), func(msg *nats.Msg) { if body, answer := HandOverReply(check, handle, msg.Reply, msg.Data); answer { _ = msg.Respond(body) @@ -170,7 +193,7 @@ func (l *natsLink) HearSetuidSearches(handle SetuidSearchFunc) { if handle == nil { return } - check := HandOverCheck{Signer: l.signer, Node: l.node} + check := l.check() searching, err := l.conn.Subscribe(AskSetuidSearchSubject(l.node), func(msg *nats.Msg) { if body, answer := SetuidSearchReply(check, handle, msg.Reply, msg.Data); answer { _ = msg.Respond(body) @@ -189,7 +212,7 @@ func (l *natsLink) HearDirectoryJudgements(handle JudgeDirectoriesFunc) { if handle == nil { return } - check := HandOverCheck{Signer: l.signer, Node: l.node} + check := l.check() judging, err := l.conn.Subscribe(AskJudgeDirectoriesSubject(l.node), func(msg *nats.Msg) { if body, answer := JudgeDirectoriesReply(check, handle, msg.Reply, msg.Data); answer { _ = msg.Respond(body) @@ -207,7 +230,7 @@ func (l *natsLink) HearAccessActs(handle AccessActFunc) { if handle == nil { return } - check := HandOverCheck{Signer: l.signer, Node: l.node} + check := l.check() acting, err := l.conn.Subscribe(AskAccessActSubject(l.node), func(msg *nats.Msg) { if body, answer := AccessActReply(check, handle, msg.Reply, msg.Data); answer { _ = msg.Respond(body) @@ -225,7 +248,7 @@ func (l *natsLink) HearMoveSecrets(handle MoveSecretFunc) { if handle == nil { return } - check := HandOverCheck{Signer: l.signer, Node: l.node} + check := l.check() moving, err := l.conn.Subscribe(AskMoveSecretSubject(l.node), func(msg *nats.Msg) { if body, answer := MoveSecretReply(check, handle, msg.Reply, msg.Data); answer { _ = msg.Respond(body) @@ -258,6 +281,12 @@ func (l *natsLink) Close() { if l.judging != nil { _ = l.judging.Unsubscribe() } + if l.engine != nil { + _ = l.engine.Stop() + } + if l.warrants != nil { + _ = l.warrants.Unsubscribe() + } if l.conn != nil { l.conn.Close() } @@ -301,10 +330,12 @@ func (l *natsLink) Ask(ctx context.Context, subject string, body []byte) ([]byte // What the queue asks of a live link beside Bus, each by a type assertion that fails quietly: said here // so a link that drops one does not build. var ( - _ Bus = (*natsLink)(nil) - _ HealthBus = (*natsLink)(nil) - _ HaveBus = (*natsLink)(nil) - _ ToolBus = (*natsLink)(nil) - _ Asker = (*natsLink)(nil) - _ Asked = (*natsLink)(nil) + _ Bus = (*natsLink)(nil) + _ HealthBus = (*natsLink)(nil) + _ HaveBus = (*natsLink)(nil) + _ ToolBus = (*natsLink)(nil) + _ Asker = (*natsLink)(nil) + _ Asked = (*natsLink)(nil) + _ EngineSeats = (*natsLink)(nil) + _ OperatorAsks = (*natsLink)(nil) ) diff --git a/internal/link/record.go b/internal/link/record.go index 1dc28a6..f176bab 100644 --- a/internal/link/record.go +++ b/internal/link/record.go @@ -23,6 +23,8 @@ type RecordKeys struct { Certificate string // Authority is the mesh's own certificate as this machine last applied it (public). Authority string + // Trusts is what this machine trusts a mesh signature from now (delivery.Trusts, ADR 0550 §2). + Trusts []string } // recordOf is the node's record from one facts report and its keys; nil for a report that carries no profile (one that @@ -45,7 +47,7 @@ func recordOf(node string, r Report, keys RecordKeys) *delivery.NodeRecord { return nil } rec := &delivery.NodeRecord{V: delivery.Version, Node: node, TunnelKey: keys.TunnelKey, SealingKey: keys.SealingKey, - ServingKey: keys.ServingKey, Certificate: keys.Certificate, Authority: keys.Authority} + ServingKey: keys.ServingKey, Certificate: keys.Certificate, Authority: keys.Authority, Trusts: keys.Trusts} for _, c := range p.Capabilities { if c.Present && c.Name != "" { rec.Capabilities = append(rec.Capabilities, c.Name) diff --git a/internal/link/run.go b/internal/link/run.go index 3d17258..eac39ef 100644 --- a/internal/link/run.go +++ b/internal/link/run.go @@ -7,6 +7,8 @@ import ( "errors" "fmt" "time" + + meshasks "git.novox.be/novox/mesh-sdk/go/asks" ) // Membership is what a node needs to reach its mesh again, held by the caller. @@ -185,6 +187,12 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann } // And the controller asking this node to hand a directory used as found to the mesh (novox/hq issue // 356), answered on the link the question came over. + // Every signed ask is verified with the key trusted at that ask, none after the lease write (ADR 0563 §7). + if f, ok := link.(interface { + FollowSigner(func() ed25519.PublicKey) + }); ok && queue.ControllerKey != nil { + f.FollowSigner(queue.ControllerKey) + } if h, ok := link.(HandOvers); ok { h.HearHandOvers(queue.HandOver) } @@ -200,6 +208,19 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann if s, ok := link.(AccessActs); ok { s.HearAccessActs(queue.AccessAct) } + // And its node seat, node-engine, with the operator's answers to its own asks (novox/hq ADR 0563): served on this + // link, and what was answered while no link was open caught up from the router's records. + if queue.Acts != nil { + if s, ok := link.(EngineSeats); ok { + s.ServeEngineSeat(queue.Acts.Call) + } + if a, ok := link.(OperatorAsks); ok { + if err := a.HearWarrants(meshasks.EngineAsker(m.Node), queue.Acts.Heard); err != nil { + say("cannot hear the operator's answers to this engine's asks: " + err.Error()) + } + go queue.Acts.Reconcile() + } + } // And asking to move a value sealed to this machine into the vault (design 57's V3, issue 676). if s, ok := link.(MoveSecrets); ok { s.HearMoveSecrets(queue.MoveSecret) diff --git a/internal/link/session.go b/internal/link/session.go index 31f8a5a..e33314d 100644 --- a/internal/link/session.go +++ b/internal/link/session.go @@ -2,12 +2,14 @@ package link import ( "context" + "crypto/ed25519" "encoding/json" "errors" "fmt" "sync" "time" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/health" ) @@ -47,6 +49,12 @@ type Session struct { // MoveSecret moves one value sealed to this machine into the vault, asked by the controller during the move of // the secrets (design 57's V3, issue 676). Nil answers every such ask with a refusal. MoveSecret MoveSecretFunc + // ControllerKey is the controller's key as the engine trusts it now: nil once its lease is handed away (novox/hq + // ADR 0563 §7). Each link verifies every signed ask with it, read per ask; nil ControllerKey keeps the membership's. + ControllerKey func() ed25519.PublicKey + // Acts is the node seat node-engine this engine holds (novox/hq ADR 0563): served on every link, and its asks to the + // operator asked and heard over it. Nil serves no seat. + Acts *HandActs // Keys reads the node's public keys and its certificate for its record (ADR 0555 §3), at each facts report; nil writes // no record. Keys func() RecordKeys @@ -337,3 +345,49 @@ func (q *Session) AskSeat(ctx context.Context, seat, verb string, args any, with } return env.Result, nil } + +// PublishAsk publishes an ask or a cancel to the operator's router over the link open now (novox/hq ADR 0563 §3). +func (q *Session) PublishAsk(subject string, body any, id string) error { + asking, err := q.operatorAsks() + if err != nil { + return err + } + return asking.PublishAsk(subject, body, id) +} + +// RouterRecord reads one key of the router's records over the link open now. +func (q *Session) RouterRecord(key string) (*stdio.StateEntry, error) { + asking, err := q.operatorAsks() + if err != nil { + return nil, err + } + return asking.RouterRecord(key) +} + +// SayEngine says one of the node seat's events over the link open now; with none open it is not said. +func (q *Session) SayEngine(event string, body any) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + if sayer, ok := bus.(interface{ SayEngine(string, any) error }); ok { + if err := sayer.SayEngine(event, body); err != nil { + q.Say("could not say the node seat's " + event + ": " + err.Error()) + } + } +} + +func (q *Session) operatorAsks() (OperatorAsks, error) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + if bus == nil { + return nil, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked) + } + asking, ok := bus.(OperatorAsks) + if !ok { + return nil, errNoAsking + } + return asking, nil +} diff --git a/internal/link/setuid_search_test.go b/internal/link/setuid_search_test.go index 5e90229..8b333e7 100644 --- a/internal/link/setuid_search_test.go +++ b/internal/link/setuid_search_test.go @@ -57,4 +57,82 @@ func TestASetuidSearchAskIsBelievedOnlySignedForThisNodeAndOnce(t *testing.T) { if a := AnswerSetuidSearch(check, nil, body); a.Refused == "" { t.Fatalf("no handler: %+v", a) } + // 774 (ADR 0563 §2): the node seat's setuid-search starts none while one runs, at most one in any hour, naming when the + // next may start, and records who asked. + t.Run("the node seat's search is bounded", func(t *testing.T) { + at, running, started := now, false, 0 + h := &HandActs{Node: "n1", State: &MemState{}, Now: func() time.Time { return at }, + Searching: func() bool { return running }, + SetuidSearch: func(by string) (string, error) { started++; return "a new search starts", nil }} + for _, row := range []struct { + name string + after time.Duration + running bool + refused string + started int + }{ + {name: "one runs", running: true, refused: "runs on this machine now", started: 0}, + {name: "the first", started: 1}, + {name: "within the hour", after: 59 * time.Minute, refused: "the next may start at 2026-10-10T03:00:00Z", started: 1}, + {name: "after the hour", after: time.Hour + time.Second, started: 2}, + } { + at, running = now.Add(row.after), row.running + _, err := h.Call(EngineCall{Verb: ActSetuidSearch, Caller: "ace.node-tools"}) + if (row.refused == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), row.refused)) || started != row.started { + t.Errorf("%s: %v, %d started", row.name, err, started) + } + } + acts, _ := h.Acts(1) + if got := acts.(map[string]any)["acts"].([]HandAct); len(got) != 1 || got[0].Caller != "ace.node-tools" || got[0].State != HandDone { + t.Errorf("the last search is recorded as %+v", got) + } + // The bound is kept in a key of its own (rev-l3 round 1): two hundred judgements before a search, or the acts + // gone from the record, never let a second search start within the hour. + h.JudgeDirectories = func(string) (string, error) { return "judged", nil } + for range 200 { + if _, err := h.Call(EngineCall{Verb: ActJudgeDirectories, Caller: "ace.node-tools"}); err != nil { + t.Fatal(err) + } + } + at = now.Add(time.Hour + 2*time.Minute) + if _, err := h.Call(EngineCall{Verb: ActSetuidSearch, Caller: "ace.node-tools"}); err == nil || !strings.Contains(err.Error(), "the next may start") { + t.Errorf("a search after two hundred judgements: %v", err) + } + keys, _ := h.State.Keys("act.") + for _, k := range keys { + _ = h.State.Delete(k) + } + if _, err := h.Call(EngineCall{Verb: ActSetuidSearch, Caller: "ace.node-tools"}); err == nil || !strings.Contains(err.Error(), "the next may start") || started != 2 { + t.Errorf("a search with the acts gone: %v, %d started", err, started) + } + // rev-l3 round 2 (ADR 0563 §2, a wrong caller gains nothing): judgements and refused searches, which any caller + // may ask, keep one rolling key of the node's, so ten thousand of them write a bounded number of bytes and add no + // key to the bucket every node shares. + keysNow := func() (int, int) { + all, _ := h.State.Keys("") + size := 0 + for _, k := range all { + if e, _ := h.State.Get(k); e != nil { + size += len(e.Value) + } + } + return len(all), size + } + _, _ = h.Call(EngineCall{Verb: ActJudgeDirectories, Caller: "unproven"}) + keysBefore, _ := keysNow() + for i := range 10000 { + verb := ActJudgeDirectories + if i%10 == 0 { + verb = ActSetuidSearch + } + _, _ = h.Call(EngineCall{Verb: verb, Caller: "unproven"}) + } + if n, size := keysNow(); n != keysBefore || size > 64<<10 { + t.Errorf("ten thousand judgements and refused searches left %d keys (was %d) of %d bytes", n, keysBefore, size) + } + listed, _ := h.Acts(0) + if got := listed.(map[string]any)["acts"].([]HandAct); len(got) == 0 || got[0].Caller != "unproven" { + t.Errorf("acts does not show the latest judgement first: %d", len(got)) + } + }) } diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index fb14265..8a596e8 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -12,7 +12,9 @@ package selfcompose import ( "context" + "crypto/ed25519" "crypto/sha256" + "encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -28,6 +30,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/compose" "git.novox.be/novox/mesh-sdk/go/compose/frombus" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" "github.com/nats-io/nats.go" @@ -75,6 +78,13 @@ type Env struct { // Fetch reads an archive at a URL in the artifact store: a moved module's templates archive, whose texts its built // manifest carries (issue 1126). Read once per digest. Nil reads none, and a moved module does not compose. Fetch func(ctx context.Context, url string) ([]byte, error) + // VaultAuthority is the vault's authority as this node read it from the vault's `signers` (ADR 0550 §2), composed + // beside the controller's as the mesh's authority (frombus.SignersReader); "" or nil composes the controller's alone. + VaultAuthority func() string + // Certify asks the vault's `sign` for this node's own certificate, keeps it and writes the node's record again (ADR + // 0550 §2): asked only when a module placed here asks for a certificate and the record holds none from an authority + // the node trusts. Nil asks nothing, and such a module stands in, as before. + Certify func(ctx context.Context) error } // VaultSecretChanged is the vault's event that a secret changed: a name and its generation, never a value. @@ -99,6 +109,8 @@ type Composer struct { // sealedOwn is each own secret's sealed value as first handed, by module/name and generation: a value sealed again // is other bytes, so the one first had is kept while the generation stands, and the composition stays the same. sealedOwn map[string]ownSealed + // certified is the serving key a certificate was last asked for successfully, so it is asked once per key. + certified string said string refusal string // held is what the composer's last pass held: each module refused alone, and the node's own ("node") when it could @@ -501,6 +513,8 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { if c.env.Running != nil { running = c.env.Running() } + // Its own certificate, asked of the vault when a module here asks for one and the record holds none it trusts. + c.certifyIfWanted(ctx, r, own) scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running} missed := c.missed.Load() compositions, _, held, err := frombus.Compose(ctx, r, c.env.Mesh, scope, fps) @@ -746,6 +760,69 @@ func (c *Composer) Policy(ctx context.Context, module string) string { // reader is the composer's view of the bus as frombus reads it. type reader struct{ c *Composer } +// VaultAuthority is the vault's authority the node follows (frombus.SignersReader). +func (r reader) VaultAuthority(context.Context) string { + if r.c.env.VaultAuthority == nil { + return "" + } + return r.c.env.VaultAuthority() +} + +var _ frombus.SignersReader = reader{} + +// certificateWanted says whether a module the placement p puts on this node asks for a certificate for the machine's +// internal name (its manifest's `certificate`), read from the indexer's manifests; a manifest that cannot be read is +// taken as asking none, and the composition says what it misses. +func (c *Composer) certificateWanted(ctx context.Context, r reader, p delivery.Placement) bool { + for _, m := range p.Modules { + c.mu.Lock() + digest := c.modules[m.Module].Manifest + c.mu.Unlock() + if digest == "" { + continue + } + raw, err := r.Manifest(ctx, digest) + if err != nil { + continue + } + if parsed, err := catalogue.ParseManifestForInputs(raw); err == nil && parsed.Certificate != nil { + return true + } + } + return false +} + +// certifyIfWanted asks Certify once per serving key when a module placed here asks for a certificate and this node's +// record holds none for its internal name and serving key from an authority it trusts (the one it holds, or the vault's). +func (c *Composer) certifyIfWanted(ctx context.Context, r reader, p delivery.Placement) { + if c.env.Certify == nil || !c.certificateWanted(ctx, r, p) { + return + } + c.mu.Lock() + rec, ok := c.records[c.env.Node] + asked := c.certified + c.mu.Unlock() + if !ok || rec.ServingKey == "" || asked == rec.ServingKey { + return + } + key, err := base64.StdEncoding.DecodeString(rec.ServingKey) + if err != nil || len(key) != ed25519.PublicKeySize { + return + } + trusted := []string{rec.Authority, r.VaultAuthority(ctx)} + if rec.Certificate != "" && delivery.CheckNodeCertificate(rec.Certificate, trusted, overlay.InternalName(c.env.Node), + ed25519.PublicKey(key), time.Now()) == nil { + return + } + if err := c.env.Certify(ctx); err != nil { + c.sayOnce(fmt.Sprintf("a module here asks for a certificate, and the vault's sign gave none: %v", err)) + return + } + c.mu.Lock() + c.certified = rec.ServingKey + c.mu.Unlock() +} + func (r reader) Placements(context.Context) (map[string]delivery.Placement, error) { r.c.mu.Lock() defer r.c.mu.Unlock() diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index afe1490..e6c36b4 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -3,6 +3,7 @@ package selfcompose import ( "context" "crypto/sha256" + "encoding/base64" "encoding/hex" "encoding/json" "fmt" @@ -1221,6 +1222,32 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { } } b.mu.Unlock() + + // L3 s7 (ADR 0550 §2): the node asks the vault for its own certificate only when a module placed here asks for one + // and its record holds none from an authority it trusts; once per serving key. + certBus := newFakeBus() + certAsks := 0 + certEnv := certBus.env() + certEnv.Certify = func(context.Context) error { certAsks++; return nil } + certEnv.VaultAuthority = func() string { return "" } + cc := New(certEnv) + withCert := `{"module":"web","version":"1","certificate":{"into":"/etc/web/tls.crt"}}` + cdigest := delivery.ManifestDigest([]byte(withCert)) + cc.manifests[digest] = []byte(lemurs) + cc.manifests[cdigest] = []byte(withCert) + cc.modules = map[string]delivery.Module{"lemurs": {Manifest: digest}, "web": {Manifest: cdigest}} + cc.records["n1"] = delivery.NodeRecord{Node: "n1", ServingKey: base64.StdEncoding.EncodeToString(make([]byte, 32))} + onlyLemurs := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "lemurs"}}} + cc.certifyIfWanted(context.Background(), reader{c: cc}, onlyLemurs) + if certAsks != 0 { + t.Errorf("no module asks for a certificate, and the vault was asked %d times", certAsks) + } + withWeb := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "web"}}} + cc.certifyIfWanted(context.Background(), reader{c: cc}, withWeb) + cc.certifyIfWanted(context.Background(), reader{c: cc}, withWeb) + if certAsks != 1 { + t.Errorf("a module asks for a certificate: the vault was asked %d times, want once", certAsks) + } } // patternBus is a bus whose every watch gets the updates of its bucket matching its pattern, as the bus's own watches diff --git a/module.json b/module.json index 8fd4fa6..d10f063 100644 --- a/module.json +++ b/module.json @@ -17,6 +17,12 @@ "consumes": [ "restic.backed-up" ], + "state": [ + { + "name": "acts", + "ttl-seconds": 7776000 + } + ], "config": { "operator-signers": { "type": "string", diff --git a/packaging/operator_keys_test.go b/packaging/operator_keys_test.go index 5935047..6794b9e 100644 --- a/packaging/operator_keys_test.go +++ b/packaging/operator_keys_test.go @@ -68,4 +68,13 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) { if len(m.Consumes) != 1 || m.Consumes[0] != "restic.backed-up" { t.Fatalf("consumes %v", m.Consumes) } + // 774 (ADR 0563 §2, §3): the acts of the node seat and the engine's own asks are its declared state acts, the bucket + // mesh-host_acts the grants part creates and each node's engine writes under its own keys. + parsed, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatal(err) + } + if len(parsed.State) != 1 || parsed.State[0].Name != "acts" || parsed.State[0].TTLSeconds != 90*24*60*60 { + t.Fatalf("the manifest declares the state %+v", parsed.State) + } } -- 2.54.0 From d8de0a8853a214526b16a0f08f1de1e4ab30ef0f Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 01:26:46 +0100 Subject: [PATCH 02/21] Judge a module the agent has not taken yet by the composition its have names, and knock the filling at the start and whenever a module is taken as running (issue 924) A held module on a node composing its own never gets a composition, so the agent never took it and the filling judged its files by the kept declaration, older than power's three new drop-ins: refused from every engine start on ace, g14 and shanks until some module settled (2026-10-29 00:09 UTC). The second meeting of issue 924. Quick fix: knock the filling once the haves are read at the start. Thorough fix (chosen, the kind recurs for every file added since the switch): what runs of such a module is the composition its have names, from the bucket or its revisions, ahead of the kept declaration; the filling is knocked on every change of what runs, not only on a settle. One unit test replaced by a table of four cases; the count stays. --- cmd/mesh-host/filling.go | 11 ++-- cmd/mesh-host/nodeagent.go | 7 +- internal/agent/agent.go | 123 +++++++++++++++++++++++++++++++++-- internal/agent/agent_test.go | 91 ++++++++++++++++++++++---- 4 files changed, 208 insertions(+), 24 deletions(-) diff --git a/cmd/mesh-host/filling.go b/cmd/mesh-host/filling.go index d77d2e9..5257e3e 100644 --- a/cmd/mesh-host/filling.go +++ b/cmd/mesh-host/filling.go @@ -30,9 +30,10 @@ func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, s var fillingNow atomic.Pointer[filling.Engine] // knockFilling tells the provisioner's client to fill again what the filling refused or failed and the declaration -// applied now may let through (filling.Engine.Knock), once a module's apply or removal settled (agent.Env.Settled). -// What it told, or why it could not, is said. -func knockFilling(module string, say func(string)) { +// applied now may let through (filling.Engine.Knock), after what runs changed: a module's apply or removal settled +// (agent.Env.Settled, " settled"), the haves were read at the start, or a module was taken as running +// (agent.Env.RunsChanged, issue 924). What it told, or why it could not, is said. +func knockFilling(after string, say func(string)) { e := fillingNow.Load() if e == nil { return @@ -40,9 +41,9 @@ func knockFilling(module string, say func(string)) { told, err := e.Knock(filling.Socket) switch { case err != nil: - say(fmt.Sprintf("after %s settled: %v", module, err)) + say(fmt.Sprintf("after %s: %v", after, err)) case told != "": - say(fmt.Sprintf("after %s settled, the provisioner's client is told to fill again: %s", module, told)) + say(fmt.Sprintf("after %s, the provisioner's client is told to fill again: %s", after, told)) } } diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 667d18e..781045d 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -510,8 +510,11 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus Backup: backupNode(session), // What the filling refused before this apply may be named now: the provisioner's client is told to fill // again (ace, 2026-10-25 21:17). - Settled: func(module string) { knockFilling(module, say) }, - Say: say, + Settled: func(module string) { knockFilling(module+" settled", say) }, + // And once the haves are read at the start, and whenever a module is taken as running without an apply, so + // a refusal from the engine's start does not wait for some module to settle (issue 924). + RunsChanged: func(why string) { knockFilling(why, say) }, + Say: say, // A failure of the bus not answering for a moment is tried again: 2 s, 5 s, 15 s, 60 s, then every 5 min. Backoff: link.Backoff, Apply: func(ctx context.Context, c delivery.Composed, others []delivery.Composed) agent.Result { diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 5bf7222..62df761 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -109,7 +109,9 @@ type Env struct { Kept func() map[string][]json.RawMessage // KeptFilled is the paths the kept declaration names as filled by the provisioner, by the module whose file is at // each (the node's own under NodeModule). A module that still runs from the kept declaration keeps its filled files - // filled: without them the filling refused each one as "not named as filled" (issue 1945). Nil names none. + // filled: without them the filling refused each one as "not named as filled" (issue 1945). Nil names none. A module + // whose have names a composition the bucket keeps runs from that composition, never from the kept declaration + // (issue 924): the kept declaration is older than every composition applied since the switch. KeptFilled func() map[string][]string // Trusted is the public keys a composition may be signed by: the controller's and the operator's (ADR 0539 §3). A // composition unsigned, signed by another key, or changed after it was signed is refused and nothing of it is @@ -131,6 +133,12 @@ type Env struct { // them. The node-engine's filling tells the provisioner's client then to fill again what it refused (mesh-host // internal/filling Knock; ace, 2026-10-25 21:17). Nil tells nothing. Settled func(module string) + // RunsChanged is told, outside the agent's lock, that what runs (Everything) changed other than by an apply or a + // removal: once at the start, when the haves and the compositions they name are read, and each time a module is + // taken as running from its have or the kept declaration. Like Settled, the filling then tells the provisioner's + // client to fill again what it refused (issue 924, the second meeting: ace, g14 and shanks on 2026-10-29 at 00:09 + // UTC refused power's three new drop-ins from an engine start on, as no module settled after it). Nil tells nothing. + RunsChanged func(why string) // Backoff is how long to wait before the try-th retry (1, 2, …) of an apply that failed transiently. Nil retries a // transient failure only on RetryTransient or a change of either key. Backoff func(try int) time.Duration @@ -160,6 +168,10 @@ type Agent struct { // applied is the composition each module was last applied from: a revision a want named from the key's history is // not the key's latest value, and what it holds is still held out of the others' applies. applied map[string]compNow + // had is, for each module whose have names a composition the bucket keeps (its latest value or a revision), that + // composition, read once at the start (loadHad): what runs of a module the agent has not taken yet, ahead of the kept + // declaration (issue 924). + had map[string]compNow // attempt is the last apply of a module that failed or is held, with the revisions it was made from. attempt map[string]attempt checked map[string]bool @@ -218,7 +230,7 @@ func New(env Env) *Agent { if env.Say == nil { env.Say = func(string) {} } - return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{}, + return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{}, had: map[string]compNow{}, attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, notComposed: map[string]notComposedMark{}, retry: make(chan []string, 8), reconcile: make(chan []string, 64), tries: map[string]int{}, again: make(chan againAsk, 64), transient: make(chan struct{}, 1), stopped: make(chan struct{})} @@ -236,7 +248,7 @@ func (a *Agent) Run(ctx context.Context) error { if err != nil { return fmt.Errorf("the agent cannot watch its compositions: %w", err) } - var wantsCaught, compsCaught bool + var wantsCaught, compsCaught, hadRead bool dirty := map[string]bool{} for { select { @@ -303,7 +315,17 @@ func (a *Agent) Run(ctx context.Context) error { } // Nothing is applied until what was there at the start is all in: a module's composition may arrive after its // want, and the node's own composition comes before any module's. - if !(wantsCaught && compsCaught) || len(dirty) == 0 { + if !(wantsCaught && compsCaught) { + continue + } + if !hadRead { + // Before the first settle: what runs of each module with a have, from the composition it names, so the + // filling judges by it at once and not by the kept declaration until some module settles (issue 924). + hadRead = true + a.loadHad(ctx) + a.runsChanged("the haves were read at the start") + } + if len(dirty) == 0 { continue } a.settle(ctx, dirty) @@ -726,7 +748,12 @@ func (a *Agent) revisionNamed(ctx context.Context, module, fp, digest string) (c if err != nil { continue } - dig, err := delivery.VerifyComposed(c, trusted) + var dig string + if a.env.SelfComposed { + dig, err = delivery.ComposedDigestOf(c) // the node's own: unsigned (ADR 0547 §2) + } else { + dig, err = delivery.VerifyComposed(c, trusted) + } if err != nil || dig != digest { continue } @@ -755,13 +782,27 @@ func (a *Agent) others(except string) []delivery.Composed { out = append(out, c.C) } else if c, ok := a.applied[m]; ok && c.Dig == h.Composed { out = append(out, c.C) + } else if c, ok := a.had[m]; ok && c.Dig == h.Composed { + out = append(out, c.C) // a revision its have named, replaced since under the same key } } + hadNames := make([]string, 0, len(a.had)) + for m := range a.had { + if _, held := a.running[m]; !held && m != except { + hadNames = append(hadNames, m) + } + } + sort.Strings(hadNames) + for _, m := range hadNames { + out = append(out, a.had[m].C) + } if a.env.Kept != nil { kept := a.env.Kept() names := make([]string, 0, len(kept)) for m := range kept { - if _, held := a.running[m]; !held && m != except { + _, held := a.running[m] + _, had := a.had[m] + if !held && !had && m != except { names = append(names, m) } } @@ -780,6 +821,73 @@ func (a *Agent) others(except string) []delivery.Composed { return out } +// loadHad reads, once at the start, the have of every module with a want or in the kept declaration and, where it names +// a composition, that composition: the bucket's value at its key when its digest is the have's, else the revision the +// bucket keeps with that digest. A have naming none, or one the bucket no longer keeps, leaves the module to the kept +// declaration, said once (issue 924). +func (a *Agent) loadHad(ctx context.Context) { + a.mu.Lock() + seen := map[string]bool{} + for m, w := range a.wants { + if m != NodeModule && w.Present && !w.Want.Removed { + seen[m] = true + } + } + if a.env.Kept != nil { + for m := range a.env.Kept() { + if m != NodeModule { + seen[m] = true + } + } + } + a.mu.Unlock() + names := make([]string, 0, len(seen)) + for m := range seen { + names = append(names, m) + } + sort.Strings(names) + for _, m := range names { + if ctx.Err() != nil { + return + } + h, ok := a.env.Haves.Read(ctx, m) + if !ok || h.Composed == "" || h.Phase == string(delivery.PhaseRemoved) { + continue + } + a.mu.Lock() + c, found := a.compositionFor(m, h.FP) + a.mu.Unlock() + if !found || c.Err != nil || c.Dig != h.Composed { + c, found = a.revisionNamed(ctx, m, h.FP, h.Composed) + } + a.mu.Lock() + if found { + a.had[m] = c + } else if _, kept := a.keptHas(m); kept { + a.sayOnce("had:"+m, fmt.Sprintf("%s's have names the composition %s, which the bucket no longer keeps: it runs "+ + "from the kept declaration until it is applied or taken", m, short(h.Composed))) + } + a.mu.Unlock() + } +} + +// keptHas says whether the kept declaration has resources for the module. Called holding a.mu or not: Kept takes no lock +// of the agent's. +func (a *Agent) keptHas(module string) ([]json.RawMessage, bool) { + if a.env.Kept == nil { + return nil, false + } + rs, ok := a.env.Kept()[module] + return rs, ok +} + +// runsChanged tells RunsChanged, outside the agent's lock. +func (a *Agent) runsChanged(why string) { + if a.env.RunsChanged != nil { + a.env.RunsChanged(why) + } +} + // unreadableModules are the keys of the module compositions in the node's bucket that cannot be read or are refused, of a // module the assigner has not removed: for each module only the composition its want names, else its newest, since a // stale one nobody applies (written before a rule the SDK now holds) must not hold the node's own back. Called holding @@ -1048,6 +1156,7 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning} } a.mu.Unlock() + a.runsChanged(module + " was taken as held") return true } if h.Composed != "" { @@ -1068,6 +1177,7 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com a.mu.Lock() a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning} a.mu.Unlock() + a.runsChanged(module + " was taken from its have") // Said again, so the session holds the module and its key carries the health this engine judges, // not the one an older engine left there (xorg's unknown on g14 and shanks, 2026-10-25). a.writeHave(ctx, Have{Module: module, FP: h.FP, Composed: h.Composed, @@ -1096,6 +1206,7 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com a.mu.Lock() a.running[module] = delivery.Have{FP: w.Want.FP, Composed: w.Want.Composed, Phase: delivery.PhaseRunning} a.mu.Unlock() + a.runsChanged(module + " was taken from the kept declaration") a.writeHave(ctx, Have{Module: module, FP: w.Want.FP, Composed: w.Want.Composed, Phase: string(delivery.PhaseRunning)}) return true } diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 7c256bb..f3c0b7e 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -1026,18 +1026,87 @@ func TestTheRevisionAWantNamesIsAppliedFromHistory(t *testing.T) { } } -// 1945/AC18 (unit): a module that still runs from the kept declaration carries the kept declaration's filled paths of -// its files, so the filling writes them ("not named as filled" for claude-code and screen-lock, 2026-10-25). -func TestAModuleRunningFromTheKeptDeclarationKeepsItsFilledPaths(t *testing.T) { - b := newBed(t) - b.kept = map[string][]json.RawMessage{"claude-code": {file("claude-code", "settings", "")}} - b.env.KeptFilled = func() map[string][]string { - return map[string][]string{"claude-code": {"/tmp/r6/claude-code/settings"}} +// 1945/AC18 and issue 924 (unit): what runs of a module the agent has not taken yet carries the filled paths the +// filling judges by. From the kept declaration while the module has no have naming a composition ("not named as filled" +// for claude-code and screen-lock, 2026-10-25); from the composition its have names once it has one, the kept +// declaration being older, read at the start before any module settles and told to the filling then, so a file the +// newer composition fills is accepted on the first fill (power's three drop-ins on ace, g14 and shanks, 2026-10-29). +func TestWhatRunsBeforeTheAgentTakesAModuleNamesItsFilledPaths(t *testing.T) { + const old, added = "/tmp/r6/power/settings", "/tmp/r6/power/bound" + filledFile := func(id string) json.RawMessage { + return json.RawMessage(`{"id":"power.` + id + `","type":"file","module":"power","path":"/tmp/r6/power/` + id + + `","mode":"0644","owner-at-runtime":"owned"}`) } - a := New(b.env) - got := a.Everything() - if len(got) != 1 || got[0].Module != "claude-code" || strings.Join(got[0].Filled, ",") != "/tmp/r6/claude-code/settings" { - t.Fatalf("what runs from the kept declaration: %+v", got) + fp := fpOf("a") + ck, _ := delivery.ComposedKey(node, "power", fp) + applied := delivery.Composed{Node: node, Module: "power", FP: fp, Filled: []string{old, added}, + Resources: []json.RawMessage{filledFile("settings"), filledFile("bound")}} + _, appliedRaw, appliedDig := signedBy(t, applied, ck, controllerKey) + newer := applied + newer.Resources = append([]json.RawMessage{}, applied.Resources...) + newer.Resources = append(newer.Resources, file("power", "later", "1")) + _, newerRaw, _ := signedBy(t, newer, ck, controllerKey) + for _, tc := range []struct { + name string + // have is the module's have; bucket the value its composition key holds now, history the revisions kept. + have *Have + bucket []byte + history []Update + filled, told string + }{ + {name: "no have: the kept declaration's", filled: old}, + {name: "a have naming the bucket's value", have: &Have{Module: "power", FP: fp, Composed: appliedDig}, + bucket: appliedRaw, filled: old + "," + added, told: "the haves were read at the start"}, + {name: "a have naming a revision replaced since", have: &Have{Module: "power", FP: fp, Composed: appliedDig}, + bucket: newerRaw, history: []Update{{Key: ck, Value: appliedRaw, Revision: 3}, {Key: ck, Value: newerRaw, Revision: 7}}, + filled: old + "," + added, told: "the haves were read at the start"}, + {name: "a have naming a composition the bucket no longer keeps", have: &Have{Module: "power", FP: fp, Composed: appliedDig}, + bucket: newerRaw, filled: old, told: "the haves were read at the start"}, + } { + t.Run(tc.name, func(t *testing.T) { + b := newBed(t) + b.kept = map[string][]json.RawMessage{"power": {filledFile("settings")}} + b.env.KeptFilled = func() map[string][]string { return map[string][]string{"power": {old}} } + if tc.have != nil { + b.haves.stored = map[string]Have{"power": *tc.have} + } + if tc.bucket != nil { + b.comps.before(ck, tc.bucket) + } + b.env.Revisions = func(_ context.Context, key string) ([]Update, error) { return tc.history, nil } + var told []string + var mu sync.Mutex + b.env.RunsChanged = func(why string) { mu.Lock(); told = append(told, why); mu.Unlock() } + filledNow := func() string { + for _, c := range b.ag.Everything() { + if c.Module == "power" { + return strings.Join(c.Filled, ",") + } + } + return "" + } + if tc.told == "" { // no want, no have: nothing to start, and the kept declaration is what runs + a := New(b.env) + b.ag = a + if got := filledNow(); got != tc.filled { + t.Fatalf("filled %q, want %q", got, tc.filled) + } + return + } + stop := b.run() + defer stop() + eventually(t, "the filling was not told what runs at the start", func() bool { + mu.Lock() + defer mu.Unlock() + return len(told) > 0 && told[0] == tc.told + }) + if got := filledNow(); got != tc.filled { + t.Fatalf("filled %q, want %q", got, tc.filled) + } + if got := b.appliedList(); len(got) != 0 { + t.Fatalf("applied %v: nothing has a want", got) + } + }) } } -- 2.54.0 From f38ac87ac9285f5a5330ea7de60cdfc3864426d2 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 01:37:40 +0100 Subject: [PATCH 03/21] What runs from the kept declaration says Data as the module's manifest does, and only a have that runs counts at the start (issue 924) The kept-declaration fallback built a composition without Data, so after a restart a data module (mongodb, mssql on novox) was judged to move its data against its first composition and the operator was asked a false yes. Now every such composition takes Data from the manifest (agent.Env.HoldsData, the composer's DataItems), and a module whose manifest cannot be read is said to be unjudgeable rather than taken to hold none. The composition a have names comes before the kept declaration in the judgement too. loadHad takes a have only in the phases baseline takes as running (running, held, a composer's mark): one backing up, applying, checking, failed or waiting names a composition not applied (review of 031e106). Two rows added to the same table test; the count stays. --- cmd/mesh-host/nodeagent.go | 1 + internal/agent/agent.go | 86 +++++++++++++++++++++++++---- internal/agent/agent_test.go | 21 ++++++- internal/selfcompose/selfcompose.go | 21 +++++++ 4 files changed, 116 insertions(+), 13 deletions(-) diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 781045d..88d9b6b 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -618,6 +618,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus if composer != nil { composes.set(delivery.ComposesOwn) env.Composed, env.SelfComposed, env.Trusted, env.Policy = composer, true, nil, composer.Policy + env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does session.NoteComposes(linked, delivery.ComposesOwn, why) // And once the catalogue no longer places it, the agent begins again on the controller's compositions: the // placement key deleted ends this run, the way back (issue 1126, the seed's revert). diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 62df761..acb91db 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -151,6 +151,11 @@ type Env struct { // move of a module of policy record that recreates a container or moves its data waits for a person's yes in its // want (ADR 0547 §3). Nil judges no move. Policy func(ctx context.Context, module string) string + // HoldsData says whether a module's manifest declares data (catalogue.Manifest.DataItems), and whether its manifest + // could be read: what a composition's Data is, for what runs of a module from the kept declaration, which carries no + // Data of its own. A composition says Data, so without it a data module from the kept declaration is judged to move + // its data against its first composition after a restart (mongodb and mssql on novox, issue 924). Nil reads none. + HoldsData func(ctx context.Context, module string) (data, known bool) } // transientFor is how many transient retries say the phase applying before the have says failed: past them the bus has @@ -172,6 +177,9 @@ type Agent struct { // composition, read once at the start (loadHad): what runs of a module the agent has not taken yet, ahead of the kept // declaration (issue 924). had map[string]compNow + // keptData is, for each module of the kept declaration whose manifest could be read, whether it declares data + // (Env.HoldsData): the Data of what runs of it from the kept declaration. + keptData map[string]bool // attempt is the last apply of a module that failed or is held, with the revisions it was made from. attempt map[string]attempt checked map[string]bool @@ -230,7 +238,7 @@ func New(env Env) *Agent { if env.Say == nil { env.Say = func(string) {} } - return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{}, had: map[string]compNow{}, + return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{}, had: map[string]compNow{}, keptData: map[string]bool{}, attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, notComposed: map[string]notComposedMark{}, retry: make(chan []string, 8), reconcile: make(chan []string, 64), tries: map[string]int{}, again: make(chan againAsk, 64), transient: make(chan struct{}, 1), stopped: make(chan struct{})} @@ -808,10 +816,7 @@ func (a *Agent) others(except string) []delivery.Composed { } sort.Strings(names) for _, m := range names { - c := delivery.Composed{Node: a.env.Node, Module: m, Resources: kept[m]} - if a.env.KeptFilled != nil { - c.Filled = a.env.KeptFilled()[m] - } + c, _ := a.keptComposed(m, kept[m]) if m == NodeModule { c.Module = "" } @@ -846,12 +851,13 @@ func (a *Agent) loadHad(ctx context.Context) { names = append(names, m) } sort.Strings(names) + a.readKeptData(ctx, names...) for _, m := range names { if ctx.Err() != nil { return } h, ok := a.env.Haves.Read(ctx, m) - if !ok || h.Composed == "" || h.Phase == string(delivery.PhaseRemoved) { + if !ok || h.Composed == "" || !haveRuns(h) { continue } a.mu.Lock() @@ -871,6 +877,17 @@ func (a *Agent) loadHad(ctx context.Context) { } } +// haveRuns says a have names what runs, in the phases baseline takes as running: running (or empty), held for a failed +// backup, or a composer's mark over what still runs. A have backing up, applying, checking, failed or waiting names a +// composition not applied, or not yet (issue 924, review of 031e106). +func haveRuns(h Have) bool { + switch delivery.Phase(h.Phase) { + case "", delivery.PhaseRunning, delivery.PhaseHeld: + return true + } + return composerMark(h) +} + // keptHas says whether the kept declaration has resources for the module. Called holding a.mu or not: Kept takes no lock // of the agent's. func (a *Agent) keptHas(module string) ([]json.RawMessage, bool) { @@ -1009,7 +1026,8 @@ func (a *Agent) heldOut(except string) []delivery.Composed { } } if len(rs) > 0 { - out = append(out, delivery.Composed{Node: a.env.Node, Module: m, Resources: rs}) + // Held out, never judged: its Data from its manifest where known, as every other composition says it. + out = append(out, delivery.Composed{Node: a.env.Node, Module: m, Resources: rs, Data: a.keptData[m]}) } } return out @@ -1397,6 +1415,7 @@ func (a *Agent) waitsForYes(ctx context.Context, module string, w wantNow, comp if module == NodeModule || a.env.Policy == nil || a.env.Policy(ctx, module) != catalogue.PolicyRecord || w.Want.Yes == comp.Dig { return false } + a.readKeptData(ctx, module) // the manifest may read now though it did not at the start a.mu.Lock() r, ran := a.running[module] if ran && r.Composed == comp.Dig && r.Phase != delivery.PhaseRemoved { @@ -1442,14 +1461,61 @@ func (a *Agent) waitsForYes(ctx context.Context, module string, w wantNow, comp // runningComposition is what the module runs now: the composition last applied, else its resources in the declaration // the node kept from before the switch. Called with a.mu held. +// +// The composition its have named at the start comes before the kept declaration (issue 924). From the kept declaration, +// what runs carries Data as its manifest declares it; a module whose manifest could not be read cannot be judged against +// it (false), and that is said, rather than taking it to hold no data and judging a false move of its data. func (a *Agent) runningComposition(module string) (delivery.Composed, bool) { if c, ok := a.applied[module]; ok && c.Err == nil { return c.C, true } - if a.env.Kept != nil { - if rs, ok := a.env.Kept()[module]; ok { - return delivery.Composed{Node: a.env.Node, Module: module, Resources: rs}, true + if c, ok := a.had[module]; ok { + return c.C, true + } + if rs, ok := a.keptHas(module); ok { + c, known := a.keptComposed(module, rs) + if !known { + a.sayOnce("kept-data:"+module, fmt.Sprintf("%s runs from the kept declaration, and whether it holds data cannot be "+ + "read from its manifest: a move of it cannot be judged", module)) } + return c, known } return delivery.Composed{}, false } + +// keptComposed is what runs of a module from the kept declaration: its resources there, the paths it names as filled, +// and Data as the module's manifest declares it; false when the manifest could not be read, so Data is not known. Called +// with a.mu held. +func (a *Agent) keptComposed(module string, rs []json.RawMessage) (delivery.Composed, bool) { + c := delivery.Composed{Node: a.env.Node, Module: module, Resources: rs} + if a.env.KeptFilled != nil { + c.Filled = a.env.KeptFilled()[module] + } + data, known := a.keptData[module] + c.Data = data + return c, known || module == NodeModule +} + +// readKeptData reads, outside the agent's lock, whether each module of the kept declaration not known yet declares data +// (Env.HoldsData), and keeps the answers it could read. +func (a *Agent) readKeptData(ctx context.Context, modules ...string) { + if a.env.HoldsData == nil { + return + } + for _, m := range modules { + if m == NodeModule { + continue + } + a.mu.Lock() + _, done := a.keptData[m] + a.mu.Unlock() + if _, kept := a.keptHas(m); done || !kept { + continue + } + if data, known := a.env.HoldsData(ctx, m); known { + a.mu.Lock() + a.keptData[m] = data + a.mu.Unlock() + } + } +} diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index f3c0b7e..a73faa2 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -1031,6 +1031,9 @@ func TestTheRevisionAWantNamesIsAppliedFromHistory(t *testing.T) { // for claude-code and screen-lock, 2026-10-25); from the composition its have names once it has one, the kept // declaration being older, read at the start before any module settles and told to the filling then, so a file the // newer composition fills is accepted on the first fill (power's three drop-ins on ace, g14 and shanks, 2026-10-29). +// Only a have in a phase baseline takes as running names what runs; and what runs carries Data, from the composition +// or, from the kept declaration, from the module's manifest, so a data module is not judged to move its data against +// its first composition after a restart (mongodb and mssql on novox). func TestWhatRunsBeforeTheAgentTakesAModuleNamesItsFilledPaths(t *testing.T) { const old, added = "/tmp/r6/power/settings", "/tmp/r6/power/bound" filledFile := func(id string) json.RawMessage { @@ -1039,7 +1042,7 @@ func TestWhatRunsBeforeTheAgentTakesAModuleNamesItsFilledPaths(t *testing.T) { } fp := fpOf("a") ck, _ := delivery.ComposedKey(node, "power", fp) - applied := delivery.Composed{Node: node, Module: "power", FP: fp, Filled: []string{old, added}, + applied := delivery.Composed{Node: node, Module: "power", FP: fp, Filled: []string{old, added}, Data: true, Resources: []json.RawMessage{filledFile("settings"), filledFile("bound")}} _, appliedRaw, appliedDig := signedBy(t, applied, ck, controllerKey) newer := applied @@ -1053,13 +1056,21 @@ func TestWhatRunsBeforeTheAgentTakesAModuleNamesItsFilledPaths(t *testing.T) { bucket []byte history []Update filled, told string + // data is what runs' Data; holdsData the manifest's answer (nil: none read). + data bool + holdsData func(context.Context, string) (bool, bool) }{ {name: "no have: the kept declaration's", filled: old}, {name: "a have naming the bucket's value", have: &Have{Module: "power", FP: fp, Composed: appliedDig}, - bucket: appliedRaw, filled: old + "," + added, told: "the haves were read at the start"}, + bucket: appliedRaw, filled: old + "," + added, told: "the haves were read at the start", data: true}, + {name: "a have applying names nothing that runs yet", have: &Have{Module: "power", FP: fp, Composed: appliedDig, + Phase: string(delivery.PhaseApplying)}, bucket: appliedRaw, filled: old, told: "the haves were read at the start"}, + {name: "a data module from the kept declaration says Data as its manifest does", filled: old, + told: "the haves were read at the start", data: true, + holdsData: func(context.Context, string) (bool, bool) { return true, true }}, {name: "a have naming a revision replaced since", have: &Have{Module: "power", FP: fp, Composed: appliedDig}, bucket: newerRaw, history: []Update{{Key: ck, Value: appliedRaw, Revision: 3}, {Key: ck, Value: newerRaw, Revision: 7}}, - filled: old + "," + added, told: "the haves were read at the start"}, + filled: old + "," + added, told: "the haves were read at the start", data: true}, {name: "a have naming a composition the bucket no longer keeps", have: &Have{Module: "power", FP: fp, Composed: appliedDig}, bucket: newerRaw, filled: old, told: "the haves were read at the start"}, } { @@ -1074,12 +1085,16 @@ func TestWhatRunsBeforeTheAgentTakesAModuleNamesItsFilledPaths(t *testing.T) { b.comps.before(ck, tc.bucket) } b.env.Revisions = func(_ context.Context, key string) ([]Update, error) { return tc.history, nil } + b.env.HoldsData = tc.holdsData var told []string var mu sync.Mutex b.env.RunsChanged = func(why string) { mu.Lock(); told = append(told, why); mu.Unlock() } filledNow := func() string { for _, c := range b.ag.Everything() { if c.Module == "power" { + if c.Data != tc.data { + t.Errorf("what runs says Data %v, want %v", c.Data, tc.data) + } return strings.Join(c.Filled, ",") } } diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 8a596e8..a49734c 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -757,6 +757,27 @@ func (c *Composer) Policy(ctx context.Context, module string) string { return policy } +// HoldsData says whether a module's manifest declares data (catalogue.Manifest.DataItems), as a composition's Data says +// it, and whether the manifest could be read: the agent's Data for what runs of a module from the kept declaration +// (agent.Env.HoldsData, issue 924). +func (c *Composer) HoldsData(ctx context.Context, module string) (data, known bool) { + c.mu.Lock() + m, ok := c.modules[module] + c.mu.Unlock() + if !ok || m.Manifest == "" { + return false, false + } + raw, err := reader{c: c}.Manifest(ctx, m.Manifest) + if err != nil { + return false, false + } + parsed, err := catalogue.ParseManifestForInputs(raw) + if err != nil { + return false, false + } + return len(parsed.DataItems()) > 0, true +} + // reader is the composer's view of the bus as frombus reads it. type reader struct{ c *Composer } -- 2.54.0 From 7d39510450f0377b959fd0b1914342ac0070650d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 02:01:59 +0100 Subject: [PATCH 04/21] Keep the content of every file given by digest on the node, so an object the controller's sweep removed never refuses a composition the node already had (issue 924) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On novox at 01:16 on 2026-10-29, after the node began composing its own modules, mesh-vault's apply failed while the vault ran: its own composition carries the export as it runs (mesh-sdk frombus KeptRuns) by a digest whose object the controller's sweep (mesh-controller cmd/mesh-controller/declaration_sweep.go, sweepNode and sweepDeclarationFiles) had removed, since the sweep keeps only what the controller's compositions in composed- and the wants name. The node may only read its bucket (ADR 0486 §2), so the node keeps what it had: a file here that verifies, or an object fetched, goes to /objects under the object's name, and a kept one is used before the bus is asked. An object unused for 14 days is let go. A fetch that still fails says the keep never held it, before its cause. Quick fix weighed: re-put the object from the controller's store. Thorough (chosen): the node never depends on the bucket for content it once had, for every module, since the sweep and a node composing its own can always disagree. The whole-apply test gains three rows that fail on the old code; the test count stays. --- cmd/mesh-host/file_references_test.go | 31 +++++++ cmd/mesh-host/main.go | 13 ++- internal/apply/objectkeep.go | 111 ++++++++++++++++++++++++++ internal/apply/references.go | 32 ++++++-- 4 files changed, 177 insertions(+), 10 deletions(-) create mode 100644 internal/apply/objectkeep.go diff --git a/cmd/mesh-host/file_references_test.go b/cmd/mesh-host/file_references_test.go index a8303fb..21444ea 100644 --- a/cmd/mesh-host/file_references_test.go +++ b/cmd/mesh-host/file_references_test.go @@ -82,6 +82,37 @@ func TestADeclarationWithAFileByDigestAppliesFromTheObject(t *testing.T) { if err != nil || string(got) != string(body) { t.Errorf("1695/AC1: the file holds %d bytes (%v), want %d", len(got), err, len(body)) } + + // Issue 924 (novox, 2026-10-29 01:16): the controller's sweep removed an object a composition the node ran still named, + // and the module's apply failed while it ran. Content the node once had is its own keep's, and the bus is not asked. + gone := aFilesLink{err: fmt.Errorf("the object %s was deleted from the bucket declaration-files-novox", name)} + never := []byte(strings.Repeat("never had here\n", 6000)) + for _, row := range []struct { + name string + raw []byte + onDisk []byte // nil removes the file + refused string // "" applies + }{ + {"the file here drifted, its object deleted", raw, []byte("the module wrote it since"), ""}, + {"the file here is gone, its object deleted", raw, nil, ""}, + {"never had here, its object deleted", referenceDeclaration(target, never), nil, "never held it"}, + } { + if row.onDisk == nil { + _ = os.Remove(target) + } else if err := os.WriteFile(target, row.onDisk, 0o600); err != nil { + t.Fatal(err) + } + report = applyAndKeep(link.WithFiles(context.Background(), gone), opts, row.raw, nil, nil, nil) + if row.refused == "" { + got, _ := os.ReadFile(target) + if report.Refused != "" || string(got) != string(body) { + t.Errorf("924: %s: refused %q, the file holds %d bytes; want it applied from the node's own keep", + row.name, report.Refused, len(got)) + } + } else if !strings.Contains(report.Refused, row.refused) || !strings.Contains(report.Refused, "was deleted") { + t.Errorf("924: %s: refused %q; want a refusal saying the bucket and the keep both lack it", row.name, report.Refused) + } + } } // 1695/AC8: the hand-run `apply ` and `reconcile` have no bus: a reference whose file here verifies is applied diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 1b6289a..bcff6e6 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -516,7 +516,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw // A file given by digest is had before the plan is drawn from it (novox/hq ADR 0486 §1, issue 1695). This command has // no link to the bus, so a file already here that verifies is used, and any other is refused: never written empty. - if err := apply.ResolveFileReferences(ctx, d, link.FilesFrom(ctx)); err != nil { + if err := apply.ResolveFileReferencesKept(ctx, d, link.FilesFrom(ctx), objectKeep(opts)); err != nil { return fmt.Errorf("a file of this declaration could not be had by its digest: %w", err) } @@ -1664,6 +1664,15 @@ func announceOr(say link.Announce) func(string) { // run by hand beside this service — the lock beside the state does the same (store.Lock). var applying sync.Mutex +// objectKeep is the node's own keep of the objects its compositions name by digest, beside its state (apply.ObjectKeep, +// issue 924): a composition the node once applied never again needs the controller's bucket to hold its files. +func objectKeep(opts options) *apply.ObjectKeep { + if opts.state == "" { + return nil + } + return apply.KeepObjectsIn(filepath.Join(filepath.Dir(opts.state), "objects")) +} + // appliedModule is how the apply of one module's declaration came out. type appliedModule struct { // Refused is why nothing was applied, when nothing was. @@ -1703,7 +1712,7 @@ func applyModuleHeld(ctx context.Context, opts options, declared *declaration.De // **A file given by digest is fetched and verified before anything of the composition is applied** (novox/hq ADR // 0486 §1, issue 1695): one object missing or differing refuses the module whole. - if err := apply.ResolveFileReferences(ctx, declared, link.FilesFrom(ctx)); err != nil { + if err := apply.ResolveFileReferencesKept(ctx, declared, link.FilesFrom(ctx), objectKeep(opts)); err != nil { return appliedModule{Refused: "a file of this composition could not be had by its digest: " + err.Error()} } diff --git a/internal/apply/objectkeep.go b/internal/apply/objectkeep.go new file mode 100644 index 0000000..d5be5df --- /dev/null +++ b/internal/apply/objectkeep.go @@ -0,0 +1,111 @@ +package apply + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "git.novox.be/novox/mesh-sdk/go/declaration" +) + +// ObjectKeep is the node-engine's own keep of the content of files given by digest (novox/hq ADR 0486 §1, issue 924): one +// file per object, named as the object is, beside the node's state. **What a node once had it never asks the bus for +// again**, so neither the controller's sweep of the bucket (an object is removed a day after no composition of the +// controller names it) nor a bus that does not answer can refuse a composition the node already ran. The node's account +// may only read its bucket (ADR 0486 §2), so the keep is the node's, not the bucket's: nothing is written back. +// +// An object unused for KeptObjectsFor is let go when another is kept, so the keep holds what the node applied lately and +// not every export it ever ran. Every failure to keep is silent and costs only a fetch: the keep is a copy. +type ObjectKeep struct { + Dir string + // Now is the clock; nil is time.Now. + Now func() time.Time +} + +// KeptObjectsFor is how long an object of the keep is held after it was last used. +const KeptObjectsFor = 14 * 24 * time.Hour + +// KeepObjectsIn is the keep at dir; an empty dir keeps nothing. +func KeepObjectsIn(dir string) *ObjectKeep { + if dir == "" { + return nil + } + return &ObjectKeep{Dir: dir} +} + +func (k *ObjectKeep) now() time.Time { + if k.Now != nil { + return k.Now() + } + return time.Now() +} + +// get is the kept content of the object name when it is what f names, and marks it used. +func (k *ObjectKeep) get(name string, f *declaration.File) ([]byte, bool) { + if k == nil { + return nil, false + } + path := filepath.Join(k.Dir, name) + body, err := os.ReadFile(path) + if err != nil || int64(len(body)) != f.Size || f.Verify(body) != nil { + return nil, false + } + now := k.now() + _ = os.Chtimes(path, now, now) + return body, true +} + +// put keeps body under name, unless it is kept already (then it is marked used), and lets go of what was unused too long. +func (k *ObjectKeep) put(name string, body []byte) { + if k == nil { + return + } + path := filepath.Join(k.Dir, name) + now := k.now() + if info, err := os.Stat(path); err == nil && info.Size() == int64(len(body)) { + _ = os.Chtimes(path, now, now) + return + } + if err := os.MkdirAll(k.Dir, 0o700); err != nil { + return + } + tmp, err := os.CreateTemp(k.Dir, ".put-*") + if err != nil { + return + } + _, werr := tmp.Write(body) + cerr := tmp.Close() + if werr != nil || cerr != nil || os.Chmod(tmp.Name(), 0o600) != nil || os.Rename(tmp.Name(), path) != nil { + _ = os.Remove(tmp.Name()) + return + } + _ = os.Chtimes(path, now, now) + k.prune(now) +} + +// prune lets go of every object unused for KeptObjectsFor. +func (k *ObjectKeep) prune(now time.Time) { + entries, err := os.ReadDir(k.Dir) + if err != nil { + return + } + for _, e := range entries { + info, err := e.Info() + if err != nil || !info.Mode().IsRegular() { + continue + } + if now.Sub(info.ModTime()) >= KeptObjectsFor { + _ = os.Remove(filepath.Join(k.Dir, e.Name())) + } + } +} + +// never says, in a failed fetch's error before its cause (a transient one's detail keeps the tail), that the node's own +// keep did not hold the object either. +func (k *ObjectKeep) never() string { + if k == nil { + return "" + } + return fmt.Sprintf(" (this node's own keep of objects, %s, never held it)", k.Dir) +} diff --git a/internal/apply/references.go b/internal/apply/references.go index ad6bed5..64ce30c 100644 --- a/internal/apply/references.go +++ b/internal/apply/references.go @@ -27,13 +27,23 @@ type FetchObject func(ctx context.Context, name string, limit int64) ([]byte, er // unchanged declaration, and a node that is asleep and has no bus, apply as they always did. One object is fetched // once however many files name its digest. func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetch FetchObject) error { + return ResolveFileReferencesKept(ctx, d, fetch, nil) +} + +// ResolveFileReferencesKept is ResolveFileReferences with the node's own keep of objects (ObjectKeep): every content had +// (a file here that verifies, or an object of the bus) is kept, and a kept one is used before the bus is asked. **A node +// depends on its bucket only for content it never had** (issue 924, 2026-10-29: novox's own composition named the +// export mesh-vault runs by a digest whose object the controller's sweep had removed, so the vault's apply failed while +// it ran; the bucket's objects are kept only while a composition of the controller names them, and a node composing its +// own names objects the controller no longer knows of). A nil keep keeps nothing. +func ResolveFileReferencesKept(ctx context.Context, d *declaration.Declaration, fetch FetchObject, keep *ObjectKeep) error { fetched := map[string][]byte{} for i, r := range d.Resources { f, ok := r.(*declaration.File) if !ok || !f.IsReference() { continue } - body, err := referencedContent(ctx, f, fetch, fetched) + body, err := referencedContent(ctx, f, fetch, fetched, keep) if err != nil { return err } @@ -50,31 +60,37 @@ func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetc } func referencedContent(ctx context.Context, f *declaration.File, fetch FetchObject, - fetched map[string][]byte) ([]byte, error) { + fetched map[string][]byte, keep *ObjectKeep) ([]byte, error) { if body, ok := fetched[f.Digest]; ok { return body, nil } - if body, ok := alreadyThere(f); ok { - fetched[f.Digest] = body - return body, nil - } name, err := declaration.ObjectName(f.Digest) if err != nil { return nil, fmt.Errorf("file %s (%s): %w", f.ID, f.Path, err) } + if body, ok := alreadyThere(f); ok { + fetched[f.Digest] = body + keep.put(name, body) + return body, nil + } + if body, ok := keep.get(name, f); ok { + fetched[f.Digest] = body + return body, nil + } if fetch == nil { return nil, fmt.Errorf("file %s (%s) is given by digest %s and is not on this machine, and this node-engine "+ "has no link to the bus to fetch it from", f.ID, f.Path, f.Digest) } body, err := fetch(ctx, name, f.Size) if err != nil { - return nil, fmt.Errorf("file %s (%s): fetching the object %s of this node's own object bucket: %w", - f.ID, f.Path, name, err) + return nil, fmt.Errorf("file %s (%s): fetching the object %s of this node's own object bucket%s: %w", + f.ID, f.Path, name, keep.never(), err) } if err := f.Verify(body); err != nil { return nil, fmt.Errorf("%s (%s): %w; nothing was applied", f.ID, f.Path, err) } fetched[f.Digest] = body + keep.put(name, body) return body, nil } -- 2.54.0 From d2645a27e946351a4764295cc566be18c83a40c8 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 02:04:40 +0100 Subject: [PATCH 05/21] Compose a file that runs as the node last wrote it, so a stale kept declaration's export no longer names a swept object (issue 924) novox's mesh-vault has failed since 01:16 on 2026-10-29: its have says failed, so loadHad (fbea777) takes no composition from it, and what runs falls back to the kept declaration, whose export (mesh-vault.kept) names f099eaa1, an object the controller's sweep removed. The node wrote a newer export since. The composer carries that file as it runs (mesh-sdk frombus keptOf), so what runs now names the digest and size of what is on disk, but only where the node's record says it wrote exactly that content at that path (Applied.Wrote, Target). The apply then finds the file in place and needs no object. Quick fix weighed: have the controller put the object back. Thorough (chosen, in the node): what runs is what the node wrote, never a guess from an older declaration. The rows join the existing whole-apply test; the test count stays. --- cmd/mesh-host/file_references_test.go | 42 +++++++++++++ cmd/mesh-host/nodeagent.go | 2 +- cmd/mesh-host/runs_as_written.go | 88 +++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 cmd/mesh-host/runs_as_written.go diff --git a/cmd/mesh-host/file_references_test.go b/cmd/mesh-host/file_references_test.go index 21444ea..0be46f6 100644 --- a/cmd/mesh-host/file_references_test.go +++ b/cmd/mesh-host/file_references_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "fmt" "os" "path/filepath" @@ -10,6 +11,7 @@ import ( "testing" "git.novox.be/novox/mesh-sdk/go/declaration" + "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/nats-io/nats.go" "github.com/novox/mesh-host/internal/link" ) @@ -113,6 +115,46 @@ func TestADeclarationWithAFileByDigestAppliesFromTheObject(t *testing.T) { t.Errorf("924: %s: refused %q; want a refusal saying the bucket and the keep both lack it", row.name, report.Refused) } } + + // Issue 924, the live case: what runs came from the kept declaration, whose export names a digest swept long ago, + // while the node wrote a newer export since. Composed as it runs, the file names what the node wrote, and applies with + // no object; a file someone else changed since is left as named. + newer := []byte(strings.Repeat("the export the node wrote since\n", 3000)) + newerName, _ := declaration.ObjectName(declaration.DigestOf(newer)) + ctx = link.WithFiles(context.Background(), aFilesLink{objects: map[string][]byte{newerName: newer}}) + if report = applyAndKeep(ctx, opts, referenceDeclaration(target, newer), nil, nil, nil); report.Refused != "" { + t.Fatalf("924: the newer export: %s", report.Refused) + } + stale := json.RawMessage(`{"id":"m.big","type":"file","path":"` + target + `","digest":"` + declaration.DigestOf(body) + + `","size":` + strconv.Itoa(len(body)) + `}`) + for _, row := range []struct { + name string + edit []byte // what someone else wrote there since; nil for nobody + digest string + }{ + {"the file is what the node wrote", nil, declaration.DigestOf(newer)}, + {"someone changed the file since", []byte("a hand edit"), declaration.DigestOf(body)}, + } { + if row.edit != nil { + if err := os.WriteFile(target, row.edit, 0o600); err != nil { + t.Fatal(err) + } + } + runs := runsAsWritten([]delivery.Composed{{Node: "n", Module: "m", Resources: []json.RawMessage{stale}}}, opts.state) + var got struct { + Digest string `json:"digest"` + } + _ = json.Unmarshal(runs[0].Resources[0], &got) + if got.Digest != row.digest { + t.Errorf("924: %s: what runs names %s, want %s", row.name, got.Digest, row.digest) + } + if row.edit == nil { + raw := []byte(`{"declaration":1,"resources":[` + string(runs[0].Resources[0]) + `]}`) + if report = applyAndKeep(link.WithFiles(context.Background(), gone), opts, raw, nil, nil, nil); report.Refused != "" { + t.Errorf("924: %s: composed as it runs, refused %q; want applied with no object", row.name, report.Refused) + } + } + } } // 1695/AC8: the hand-run `apply ` and `reconcile` have no bus: a reference whose file here verifies is applied diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 88d9b6b..31b2e59 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -563,7 +563,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus hooks := composerHooks{ running: func() []delivery.Composed { if now := own.get(); now != nil { - return now.Everything() + return runsAsWritten(now.Everything(), opts.state) } return nil }, diff --git a/cmd/mesh-host/runs_as_written.go b/cmd/mesh-host/runs_as_written.go new file mode 100644 index 0000000..a2bca2a --- /dev/null +++ b/cmd/mesh-host/runs_as_written.go @@ -0,0 +1,88 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + + "git.novox.be/novox/mesh-sdk/go/declaration" + "git.novox.be/novox/mesh-sdk/go/delivery" + + "github.com/novox/mesh-host/internal/store" +) + +// runsAsWritten is what the node runs as the composer reads it (selfcompose.Env.Running), with every file given by digest +// named as this node last wrote it, where the two differ (issue 924, novox 2026-10-29 01:16). What runs of a module the +// agent has not applied since its start can come from the kept declaration, older than the compositions applied since: +// its export (mesh-vault.kept) then named a digest the controller's sweep had removed, while the file on disk was the +// newer export the node wrote. A node composing its own carries such a file as it runs (mesh-sdk frombus keptOf), so it +// names what is on disk: **only when the node's record says it wrote exactly that content there** (store Applied.Wrote, +// Target), never a file someone else changed. A file that cannot be read, or that the record does not vouch for, is +// left as the composition names it. +func runsAsWritten(running []delivery.Composed, statePath string) []delivery.Composed { + if len(running) == 0 || statePath == "" { + return running + } + known, err := store.Load(statePath) + if err != nil { + return running + } + out := make([]delivery.Composed, len(running)) + for i, c := range running { + out[i] = c + var changed []json.RawMessage + for j, raw := range c.Resources { + if fixed, ok := asWritten(raw, known); ok { + if changed == nil { + changed = append([]json.RawMessage(nil), c.Resources...) + } + changed[j] = fixed + } + } + if changed != nil { + out[i].Resources = changed + } + } + return out +} + +// asWritten is the file resource raw named by the digest and size of what this node last wrote at its path, when raw is a +// file given by another digest and the file there is what the record says the node wrote. +func asWritten(raw json.RawMessage, known store.State) (json.RawMessage, bool) { + var fields map[string]json.RawMessage + if json.Unmarshal(raw, &fields) != nil { + return nil, false + } + var kind, id, path, digest string + _ = json.Unmarshal(fields["type"], &kind) + _ = json.Unmarshal(fields["id"], &id) + _ = json.Unmarshal(fields["path"], &path) + _ = json.Unmarshal(fields["digest"], &digest) + if kind != string(declaration.TypeFile) || digest == "" || path == "" { + return nil, false + } + rec, ok := known.Find(id) + if !ok || rec.Wrote == "" || rec.Target != path { + return nil, false + } + body, err := os.ReadFile(path) + if err != nil { + return nil, false + } + sum := sha256.Sum256(body) + if hex.EncodeToString(sum[:]) != rec.Wrote { + return nil, false // changed since the node wrote it: not the node's to vouch for + } + if now := declaration.DigestOf(body); now == digest { + return nil, false + } else { + fields["digest"], _ = json.Marshal(now) + } + fields["size"], _ = json.Marshal(len(body)) + fixed, err := json.Marshal(fields) + if err != nil { + return nil, false + } + return fixed, true +} -- 2.54.0 From a7acee3eed371e802455c9ffd37b173c32efb45d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 10:23:37 +0100 Subject: [PATCH 06/21] Refuse an apply that would remove a resource of a module a want still places here, so a composition that lost what runs can never remove the bus server or a held module again (issue 774) --- cmd/mesh-host/nodeagent.go | 15 +++-- internal/agent/agent.go | 14 +++++ internal/apply/apply.go | 46 ++++++++++++++++ internal/apply/module_scope_test.go | 85 ++++++++++++++++++++--------- 4 files changed, 129 insertions(+), 31 deletions(-) diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 31b2e59..0d22773 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -526,7 +526,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus if herr != nil { res = failedWith(herr.Error()) } else { - res = applyComposition(actx, opts, held, others, sched, say) + res = applyComposition(actx, opts, held, others, ag.Placed(scopeName(c)), sched, say) } done() // The mesh's own certificate it carries is kept for this machine's record (ADR 0555 §3). @@ -545,7 +545,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus }, Remove: func(ctx context.Context, module string, others []delivery.Composed) agent.Result { actx, done := applyContext(session.WithLink(ctx)) - res := removeComposition(actx, opts, module, others, sched, say) + res := removeComposition(actx, opts, module, others, ag.Placed(module), sched, say) done() afterApply() sayFacts(ctx, ag) @@ -684,7 +684,10 @@ func (c *agentNow) get() *agent.Agent { } // applyComposition applies one composition: a module's, or the node's own. -func applyComposition(ctx context.Context, opts options, c delivery.Composed, others []delivery.Composed, +// +// placed is the modules a want still places here besides this one: none of their resources is removed by this apply, +// which is refused instead (issue 774). +func applyComposition(ctx context.Context, opts options, c delivery.Composed, others []delivery.Composed, placed map[string]bool, sched *apply.Scheduler, say link.Announce) agent.Result { name := scopeName(c) if c.Module == "" { @@ -700,7 +703,7 @@ func applyComposition(ctx context.Context, opts options, c delivery.Composed, ot if err := refuseEmptyWrites(d); err != nil { return failedWith(err.Error()) } - scope := apply.Scope{Module: name, Others: resourcesOf(others)} + scope := apply.Scope{Module: name, Others: resourcesOf(others), Placed: placed} got := applyModuleHeld(ctx, opts, d, &scope, sched, say) if got.Refused != "" { return failedWith(got.Refused) @@ -719,9 +722,9 @@ func applyComposition(ctx context.Context, opts options, c delivery.Composed, ot } // removeComposition removes one module's own resources and nothing of another's. -func removeComposition(ctx context.Context, opts options, module string, others []delivery.Composed, +func removeComposition(ctx context.Context, opts options, module string, others []delivery.Composed, placed map[string]bool, sched *apply.Scheduler, say link.Announce) agent.Result { - scope := apply.Scope{Module: module, Others: resourcesOf(others)} + scope := apply.Scope{Module: module, Others: resourcesOf(others), Placed: placed} got := applyModuleHeld(ctx, opts, &declaration.Declaration{Version: declaration.Version}, &scope, sched, say) switch { case got.Refused != "": diff --git a/internal/agent/agent.go b/internal/agent/agent.go index acb91db..8525b4b 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -1042,6 +1042,20 @@ func (a *Agent) Everything() []delivery.Composed { return a.others("\x00") } +// Placed is the modules a want still places on the node (present and not removed), held ones included, but except: +// what no apply but the module's own may remove (apply.Scope.Placed, issue 774). Called without a.mu held. +func (a *Agent) Placed(except string) map[string]bool { + a.mu.Lock() + defer a.mu.Unlock() + out := map[string]bool{} + for m, w := range a.wants { + if m != NodeModule && m != except && w.Present && !w.Want.Removed { + out[m] = true + } + } + return out +} + // HoldsNode says whether the agent runs the node's own composition, so the node's parts are its and not the kept // declaration's. func (a *Agent) HoldsNode() bool { diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 1c1023c..24f93c5 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -312,6 +312,27 @@ type Scope struct { Module string // Others are the resources of every other module the node has a composition for. Others []declaration.Resource + // Placed are the modules a want still places on the node (present, not removed), held ones included: no apply but + // the module's own removes a resource of one of them (issue 774: on 2026-10-29 the node's own apply on the bus node, + // composed without what its folded modules ran, removed the bus server and four held modules). Nil guards nothing. + Placed map[string]bool +} + +// placedOwner is the module a want still places on the node that the record belongs to, other than the scope's own: +// the module recorded with it, else the one its id names (a resource the node's own composition carried for a folded +// module is recorded as the node's, its id naming the module). Empty when there is none. +func (sc *Scope) placedOwner(a store.Applied) string { + if sc == nil || len(sc.Placed) == 0 { + return "" + } + m := a.Module + if m == "" || m == store.NodeModule { + m, _ = declaration.ModuleGuess(a.ID) + } + if m == "" || m == sc.Module || !sc.Placed[m] { + return "" + } + return m } // owns reports whether a record belongs to the scope's module. @@ -441,6 +462,31 @@ func applyScoped( } } + // **A module a want still places here is never removed by another's apply** (issue 774): its resources go only with + // its own apply or its want's removal. One that would is refused whole, before anything is changed, naming them: a + // composition that leaves them out lost what runs, and removing what runs is the worst a node can do with that. + if scope != nil { + var refused []string + for _, orphan := range known.Orphans(declared, origin) { + if !scope.owns(orphan) { + continue + } + if _, kept := heldByOthers[orphan.ID]; kept { + continue + } + if m := scope.placedOwner(orphan); m != "" { + refused = append(refused, orphan.ID+" of "+m) + } + } + if len(refused) > 0 { + sort.Strings(refused) + err := fmt.Errorf("the apply of %s would remove %s, of modules a want still places here; a module's resources "+ + "go only with its own apply or its want's removal, so nothing was applied", scope.Module, strings.Join(refused, ", ")) + log(" refused: " + err.Error()) + return report, known, &Error{Resource: "the resources of modules placed here", Err: err, Done: report} + } + } + removeOrphan := func(orphan store.Applied) error { var action, detail string var err error diff --git a/internal/apply/module_scope_test.go b/internal/apply/module_scope_test.go index 67b494a..14b95f4 100644 --- a/internal/apply/module_scope_test.go +++ b/internal/apply/module_scope_test.go @@ -3,6 +3,7 @@ package apply import ( "context" "path/filepath" + "strings" "testing" "git.novox.be/novox/mesh-sdk/go/declaration" @@ -116,32 +117,66 @@ func TestARecordWithoutAnOwnerBelongsToTheModuleItsIdNames(t *testing.T) { // leaves it as it is, whatever its type, until the module's own apply takes it (issue 1945, live on 2026-10-25 at 02:32: // shanks and g14 lost systemd-resolved's guard, its resolver files and sudo's operator file, ace qbittorrent's server, // because the node's own apply removed every record no longer in it, the others holding the very same resources or not). +// +// And a resource the node's own composition carried for a module a want still places here (a folded module of the old +// form, held) is never removed by the node's own apply when a composition leaves it out: the apply is refused whole and +// says why (issue 774, live on 2026-10-29 at 07:54 UTC on the bus node: the node's own composition, composed without what +// it ran once the kept declaration was read no more, removed the bus server, the proxy and three held modules). Only its +// want's removal lets it go. func TestAResourceThatMovedToItsModulesCompositionIsLeftByTheNodesOwnApply(t *testing.T) { - dir := t.TempDir() - fact := filepath.Join(dir, "suffix") - gone := filepath.Join(dir, "gone.conf") - own := declare(t, `{"id":"systemd-resolved.fact-suffix","type":"file","path":"`+fact+`","content":"mesh\n"}, - {"id":"old.gone","type":"file","path":"`+gone+`","content":"g\n"}`) - _, state, err := ApplyModule(context.Background(), archHost(t), own, Scope{Module: store.NodeModule}, - state0(), store.OriginDeclared, noServices, nil, nil, nil, nil) - if err != nil { - t.Fatal(err) - } - // systemd-resolved now has a composition of its own, not applied yet; old.gone is no module's any more. - moved := declare(t, `{"id":"systemd-resolved.fact-suffix","type":"file","path":"`+fact+`","content":"mesh\n","module":"systemd-resolved"}`) - report, state, err := ApplyModule(context.Background(), archHost(t), &declaration.Declaration{}, - Scope{Module: store.NodeModule, Others: moved.Resources}, state, store.OriginDeclared, noServices, nil, nil, nil, nil) - if err != nil { - t.Fatal(err) - } - if !exists(fact) { - t.Fatalf("the moved resource was removed: %+v", outcomeOf(report, "systemd-resolved.fact-suffix")) - } - if exists(gone) { - t.Fatal("a resource no composition holds any more was kept") - } - if _, still := state.Find("systemd-resolved.fact-suffix"); !still { - t.Fatal("the moved resource's record was dropped, so its module's apply would not know it") + for _, tc := range []struct { + name string + id string // the resource the node's own composition carried, and no longer does + moved bool // its module's own composition now declares it (Others) + placed bool // a want still places its module here + refused bool // the node's own apply is refused whole + stays bool // the resource is still on the machine and recorded + }{ + {name: "moved to its module's own composition", id: "systemd-resolved.fact-suffix", moved: true, placed: true, stays: true}, + {name: "a held folded module, in no composition", id: "nats.server-conf", placed: true, refused: true, stays: true}, + {name: "its module's want removed", id: "nats.server-conf"}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + fact := filepath.Join(dir, "carried") + gone := filepath.Join(dir, "gone.conf") + own := declare(t, `{"id":"`+tc.id+`","type":"file","path":"`+fact+`","content":"mesh\n"}, + {"id":"old.gone","type":"file","path":"`+gone+`","content":"g\n"}`) + _, state, err := ApplyModule(context.Background(), archHost(t), own, Scope{Module: store.NodeModule}, + state0(), store.OriginDeclared, noServices, nil, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + module, _ := declaration.ModuleGuess(tc.id) + scope := Scope{Module: store.NodeModule} + if tc.moved { + moved := declare(t, `{"id":"`+tc.id+`","type":"file","path":"`+fact+`","content":"mesh\n","module":"`+module+`"}`) + scope.Others = moved.Resources + } + if tc.placed { + scope.Placed = map[string]bool{module: true} + } + report, after, err := ApplyModule(context.Background(), archHost(t), &declaration.Declaration{}, scope, state, + store.OriginDeclared, noServices, nil, nil, nil, nil) + if tc.refused != (err != nil) { + t.Fatalf("refused %v, want %v: %v", err != nil, tc.refused, err) + } + if tc.refused { + if !strings.Contains(err.Error(), tc.id+" of "+module) { + t.Fatalf("the refusal does not name what it kept: %v", err) + } + after = state + } + if exists(fact) != tc.stays { + t.Fatalf("the carried resource exists %v, want %v: %+v", exists(fact), tc.stays, outcomeOf(report, tc.id)) + } + if _, still := after.Find(tc.id); still != tc.stays { + t.Fatalf("the carried resource's record kept %v, want %v", still, tc.stays) + } + if exists(gone) == !tc.refused { + t.Fatalf("a resource no composition holds any more exists %v, with the apply refused %v", exists(gone), tc.refused) + } + }) } } -- 2.54.0 From acf02d52ff0788c5807a2832cd22b90839b27b54 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:04:53 +0100 Subject: [PATCH 07/21] Pin mesh-sdk 22c9ddd, so a carried or held module serves its tools from what runs and a data flag alone moves no data (issues 1126, 924) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8bbda86..0666c02 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029145421-dbcaa29a1a38 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index 8c32690..043e05f 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029145421-dbcaa29a1a38 h1:gJwMmK8isI9HwFM2WOCZXjiTqZmGQxs2UY38stk1ZOA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029145421-dbcaa29a1a38/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= -- 2.54.0 From 76e214a8b93550b8e7f56d17881853ae8b877642 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:16:38 +0100 Subject: [PATCH 08/21] Keep every composition the engine applied on the machine, so a module whose composition the bus no longer keeps runs as applied and is never composed as nothing (issue 774) On novox on 2026-10-29 at 07:54 UTC the kept declaration was retired and the node's own composition was in no bucket. The composer's first pass reads what runs before the agent reads any have (the agent waits for the composer's caught, sent after that pass), so the node's own was composed with its folded modules running nothing, and the bus server and four held modules were removed. - Quick fix: read the node's own have in loadHad too. It still waits for the composer's first pass, and still finds nothing when the bucket no longer keeps the composition the have names. - Thorough fix, chosen: the engine keeps each composition it applied (the node's own too) in `applied/` beside its state, 0600. The agent reads it at Run's start, before any watch, as what runs ahead of the bucket and the kept declaration. A have naming another running composition wins. An apply keeps; a removal forgets. Test: a case in TestEverythingIsWhatTheAgentRunsAndWhatIsKeptForTheRest (no new test function). It fails on L3's engine 4f9227a (0 compositions at the first own composition) and passes here. --- cmd/mesh-host/applied_store.go | 97 ++++++++++++++++++++++++++++++ cmd/mesh-host/nodeagent.go | 5 +- internal/agent/agent.go | 102 +++++++++++++++++++++++++++++++ internal/agent/agent_test.go | 106 +++++++++++++++++++++++++++++++++ 4 files changed, 309 insertions(+), 1 deletion(-) create mode 100644 cmd/mesh-host/applied_store.go diff --git a/cmd/mesh-host/applied_store.go b/cmd/mesh-host/applied_store.go new file mode 100644 index 0000000..60df951 --- /dev/null +++ b/cmd/mesh-host/applied_store.go @@ -0,0 +1,97 @@ +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" +) + +// appliedStore keeps every composition the agent applied, one file per module (the node's own as `node.json`), in the +// directory `applied` beside the node's state (agent.Env.Applied, issue 774). It is the machine's own record of what runs: +// read at the agent's start before anything is watched, so the node never composes its own as if a module it runs ran +// nothing because the bucket no longer keeps that composition or the kept declaration is retired. Files are 0600 in a +// 0700 directory: a composition carries values sealed to this machine. +type appliedStore struct{ dir string } + +func appliedStoreFor(statePath string) appliedStore { + return appliedStore{dir: filepath.Join(filepath.Dir(statePath), "applied")} +} + +// appliedName is a module name the store takes as a file name: a catalogue module's or the node's own, never a path. +var appliedName = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]*$`) + +func (s appliedStore) path(module string) (string, error) { + if !appliedName.MatchString(module) || strings.Contains(module, "..") { + return "", fmt.Errorf("%q is not a module name the applied store takes", module) + } + return filepath.Join(s.dir, module+".json"), nil +} + +func (s appliedStore) Load() (map[string][]byte, error) { + entries, err := os.ReadDir(s.dir) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + out := map[string][]byte{} + var errs []error + for _, e := range entries { + module, ok := strings.CutSuffix(e.Name(), ".json") + if !ok || e.IsDir() || !appliedName.MatchString(module) { + continue + } + raw, err := os.ReadFile(filepath.Join(s.dir, e.Name())) + if err != nil { + errs = append(errs, err) + continue + } + out[module] = raw + } + return out, errors.Join(errs...) +} + +func (s appliedStore) Keep(module string, raw []byte) error { + path, err := s.path(module) + if err != nil { + return err + } + if was, err := os.ReadFile(path); err == nil && string(was) == string(raw) { + return nil + } + if err := os.MkdirAll(s.dir, 0o700); err != nil { + return err + } + tmp, err := os.CreateTemp(s.dir, "."+module+".*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err := tmp.Write(raw); err != nil { + tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), path) +} + +func (s appliedStore) Drop(module string) error { + path, err := s.path(module) + if err != nil { + return err + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil +} diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 0d22773..07fc79c 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -494,7 +494,10 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus // The same kept declaration's filled paths, so a module still running from it keeps its files filled. KeptFilled: func() map[string][]string { _, filled := kept(); return filled }, Recorded: recordedIn(opts.state, say), - Inactive: inactiveIn(&liveness.Exec{Run: apply.ExecRunner, System: builtFor}, heldBy(apply.WindowsIn(filepath.Dir(opts.state)))), + // What this engine applied, kept on the machine and read at the start, so what runs is known before the node + // composes its own (issue 774). + Applied: appliedStoreFor(opts.state), + Inactive: inactiveIn(&liveness.Exec{Run: apply.ExecRunner, System: builtFor}, heldBy(apply.WindowsIn(filepath.Dir(opts.state)))), Revisions: func(ctx context.Context, key string) ([]agent.Update, error) { got, err := session.KVHistory(ctx, composedBucket, key) if err != nil { diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 8525b4b..0b47ede 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -156,6 +156,22 @@ type Env struct { // Data of its own. A composition says Data, so without it a data module from the kept declaration is judged to move // its data against its first composition after a restart (mongodb and mssql on novox, issue 924). Nil reads none. HoldsData func(ctx context.Context, module string) (data, known bool) + // Applied keeps every composition the agent applied, by module (the node's own under NodeModule), beside the engine's + // state on the machine: read at the start before anything is watched, so what runs is known before the node composes + // its own for the first time, whatever the bucket and the kept declaration still hold (issue 774: on 2026-10-29 the + // bus node, its kept declaration retired and its own composition in no bucket, composed its own as if its folded + // modules ran nothing and removed the bus server and four held modules). Nil keeps none. + Applied AppliedStore +} + +// AppliedStore is where the engine keeps the compositions it applied (Env.Applied): each module's last, as its bytes. +type AppliedStore interface { + // Load is every composition kept, by module; a store that cannot be read says so and loads none. + Load() (map[string][]byte, error) + // Keep keeps the composition the module was applied from, in place of the one before. + Keep(module string, raw []byte) error + // Drop forgets the module's, once it is removed. + Drop(module string) error } // transientFor is how many transient retries say the phase applying before the have says failed: past them the bus has @@ -248,6 +264,9 @@ func New(env Env) *Agent { // before any module's, then each change as it comes. func (a *Agent) Run(ctx context.Context) error { defer close(a.stopped) + // What this engine last applied, before any composition is watched: the node composing its own reads what runs + // (Everything) from its first pass on, and that pass comes before the haves are read below (issue 774). + a.loadApplied(ctx) wants, err := a.env.Wants.Watch(ctx, "want."+a.env.Node+".>") if err != nil { return fmt.Errorf("the agent cannot watch its wants: %w", err) @@ -861,6 +880,10 @@ func (a *Agent) loadHad(ctx context.Context) { continue } a.mu.Lock() + if c, kept := a.had[m]; kept && c.Dig == h.Composed { + a.mu.Unlock() + continue // what this engine applied and kept itself (loadApplied) + } c, found := a.compositionFor(m, h.FP) a.mu.Unlock() if !found || c.Err != nil || c.Dig != h.Composed { @@ -869,6 +892,8 @@ func (a *Agent) loadHad(ctx context.Context) { a.mu.Lock() if found { a.had[m] = c + } else if _, applied := a.had[m]; applied { + // Kept as this engine applied it (loadApplied), never composed as nothing. } else if _, kept := a.keptHas(m); kept { a.sayOnce("had:"+m, fmt.Sprintf("%s's have names the composition %s, which the bucket no longer keeps: it runs "+ "from the kept declaration until it is applied or taken", m, short(h.Composed))) @@ -877,6 +902,76 @@ func (a *Agent) loadHad(ctx context.Context) { } } +// loadApplied takes, once at the start, what this engine last applied of each module as what runs of it, ahead of the +// bucket and the kept declaration (Env.Applied, issue 774): it is what the machine holds, whatever the bus still keeps. +// One whose have names another composition that runs (applied since by another engine) is left to the have; one whose +// have cannot be read, or that has none, is kept as applied, so a module is never composed as running nothing. +func (a *Agent) loadApplied(ctx context.Context) { + if a.env.Applied == nil { + return + } + kept, err := a.env.Applied.Load() + if err != nil { + a.env.Say(fmt.Sprintf("the compositions this engine applied cannot be read, so what runs is read from the haves alone: %v", err)) + } + names := make([]string, 0, len(kept)) + for m := range kept { + names = append(names, m) + } + sort.Strings(names) + for _, m := range names { + if ctx.Err() != nil { + return + } + c, err := delivery.DecodeComposed(kept[m]) + var dig string + if err == nil { + dig, err = delivery.ComposedDigestOf(c) + } + if err != nil { + a.env.Say(fmt.Sprintf("the composition this engine kept of %s cannot be read, so what runs of it is read from its have: %v", m, err)) + continue + } + h, found, unread := a.readHave(ctx, m) + if found && h.Composed != "" && h.Composed != dig && haveRuns(h) { + a.env.Say(fmt.Sprintf("%s's have names the composition %s, not %s this engine applied last: what runs of it is read from its have", + m, short(h.Composed), short(dig))) + continue + } + if m == NodeModule { + c.Module, c.FP = "", "" + } + a.mu.Lock() + a.had[m] = compNow{Raw: kept[m], C: c, Dig: dig} + a.mu.Unlock() + switch { + case unread: + a.env.Say(fmt.Sprintf("%s's have cannot be read: it runs as this engine applied it, %s", m, short(dig))) + case !found: + a.env.Say(fmt.Sprintf("%s has no have: it runs as this engine applied it, %s", m, short(dig))) + } + } +} + +// keepApplied keeps the composition a module was applied from (Env.Applied); one that cannot be kept is said, and the +// apply stands. +func (a *Agent) keepApplied(module string, comp compNow) { + if a.env.Applied == nil { + return + } + raw := comp.Raw + if len(raw) == 0 { + var err error + if raw, err = delivery.EncodeComposed(comp.C); err != nil { + a.env.Say(fmt.Sprintf("the composition %s was applied from cannot be kept: %v", module, err)) + return + } + } + if err := a.env.Applied.Keep(module, raw); err != nil { + a.env.Say(fmt.Sprintf("the composition %s was applied from cannot be kept: %v", module, err)) + } +} + // haveRuns says a have names what runs, in the phases baseline takes as running: running (or empty), held for a failed // backup, or a composer's mark over what still runs. A have backing up, applying, checking, failed or waiting names a // composition not applied, or not yet (issue 924, review of 031e106). @@ -1327,6 +1422,7 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo a.running[module] = delivery.Have{FP: fp, Composed: dig, Phase: delivery.PhaseRunning} a.applied[module] = comp a.mu.Unlock() + a.keepApplied(module, comp) if a.env.Settled != nil { a.env.Settled(module) } @@ -1356,8 +1452,14 @@ func (a *Agent) remove(ctx context.Context, module string, w wantNow) { return } delete(a.attempt, module) + delete(a.had, module) a.running[module] = delivery.Have{Phase: delivery.PhaseRemoved} a.mu.Unlock() + if a.env.Applied != nil { + if err := a.env.Applied.Drop(module); err != nil { + a.env.Say(fmt.Sprintf("the composition %s was applied from cannot be forgotten now it is removed: %v", module, err)) + } + } if a.env.Settled != nil { a.env.Settled(module) } diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index a73faa2..4aa7d9f 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -769,6 +769,112 @@ func TestEverythingIsWhatTheAgentRunsAndWhatIsKeptForTheRest(t *testing.T) { if a.HoldsNode() { t.Fatal("the agent holds no node composition") } + + // Issue 774 (novox, 2026-10-29 07:54 UTC): the kept declaration retired and the node's own composition in no bucket, a + // fresh engine's first own composition was composed from what runs with the node's own missing, so its folded modules + // (the bus server among them) were composed as nothing and removed. What this engine applied is kept on the machine + // and is what runs from before the first composition is watched; an apply keeps, a removal forgets. + t.Run("what this engine applied is what runs before the node composes its own", func(t *testing.T) { + b := newBed(t) + b.env.SelfComposed, b.env.Trusted, b.kept = true, nil, nil + store := &memApplied{kept: map[string][]byte{}} + b.env.Applied = store + // A folded module's resource in the node's own composition: its id names the module, as the controller folds it. + natsConf, _ := json.Marshal(map[string]any{"id": "nats.conf", "type": "file", "path": "/tmp/r6/nats/conf", "content": "listen 4222"}) + own := delivery.Composed{Node: node, Resources: []json.RawMessage{natsConf}} + ownRaw, err := delivery.EncodeComposed(own) + if err != nil { + t.Fatal(err) + } + ownDig, _ := delivery.ComposedDigestOf(own) + web := delivery.Composed{Node: node, Module: "web", FP: fpOf("a"), Resources: []json.RawMessage{file("web", "x", "1")}} + webRaw, _ := delivery.EncodeComposed(web) + webDig, _ := delivery.ComposedDigestOf(web) + store.kept[NodeModule], store.kept["web"] = ownRaw, webRaw + b.haves.Write(context.Background(), Have{Module: NodeModule, Composed: ownDig}) + b.haves.Write(context.Background(), Have{Module: "web", FP: fpOf("a"), Composed: webDig, Phase: string(delivery.PhaseRunning)}) + // The composer's first pass reads what runs when its watch begins, before the agent reads any have. + firstCh := make(chan []delivery.Composed, 1) + comps := b.comps + var ag *Agent + b.env.Composed = watchFunc(func(ctx context.Context, p string) (<-chan Update, error) { + firstCh <- ag.Everything() + return comps.Watch(ctx, p) + }) + ag = New(b.env) + b.ag = ag + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); _ = ag.Run(ctx) }() + defer func() { cancel(); <-done }() + var first []delivery.Composed + select { + case first = <-firstCh: + case <-time.After(5 * time.Second): + t.Fatal("the composer was not watched") + } + var ranOwn, ranWeb bool + for _, c := range first { + ranOwn = ranOwn || (c.Module == "" && len(c.Resources) == 1 && strings.Contains(string(c.Resources[0]), "4222")) + ranWeb = ranWeb || c.Module == "web" + } + if !ranOwn || !ranWeb { + t.Fatalf("what runs at the node's first own composition: own %v, web %v (%d compositions); want both as this engine applied them", + ranOwn, ranWeb, len(first)) + } + // An apply keeps the composition it applied; a removal forgets it. + web2 := delivery.Composed{Node: node, Module: "web", FP: fpOf("b"), Resources: []json.RawMessage{file("web", "x", "2")}} + ck, _ := delivery.ComposedKey(node, "web", fpOf("b")) + raw2, _ := delivery.EncodeComposed(web2) + comps.put(ck, raw2) + wk, _ := delivery.WantKey(node, "web") + wraw, _ := delivery.EncodeWant(delivery.Want{FP: fpOf("b"), Self: true}) + b.wants.put(wk, wraw) + eventually(t, "the applied composition was not kept", func() bool { return string(store.get("web")) == string(raw2) }) + rraw, _ := delivery.EncodeWant(delivery.Want{Removed: true}) + b.wants.put(wk, rraw) + eventually(t, "the removed module's composition was not forgotten", func() bool { return store.get("web") == nil }) + }) +} + +// watchFunc is a Watcher from a function. +type watchFunc func(ctx context.Context, pattern string) (<-chan Update, error) + +func (f watchFunc) Watch(ctx context.Context, pattern string) (<-chan Update, error) { + return f(ctx, pattern) +} + +// memApplied is an AppliedStore in memory. +type memApplied struct { + mu sync.Mutex + kept map[string][]byte +} + +func (m *memApplied) Load() (map[string][]byte, error) { + m.mu.Lock() + defer m.mu.Unlock() + out := map[string][]byte{} + for k, v := range m.kept { + out[k] = v + } + return out, nil +} +func (m *memApplied) Keep(module string, raw []byte) error { + m.mu.Lock() + defer m.mu.Unlock() + m.kept[module] = raw + return nil +} +func (m *memApplied) Drop(module string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.kept, module) + return nil +} +func (m *memApplied) get(module string) []byte { + m.mu.Lock() + defer m.mu.Unlock() + return m.kept[module] } // A failed apply is applied again after a restart even when the kept declaration holds the composition: once this -- 2.54.0 From 81672b183df7d57f814ac181baab4c04f9a7d5c1 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:25:51 +0100 Subject: [PATCH 09/21] Keep the node's own as it runs from an engine's first start, and refuse the node's own apply removing a module no want removed (issue 774, review round 1) The reviewer showed how 76e214a still lets the 07:54 outage happen at the re-landed engine's first start on novox. applied/ is empty there, so loadApplied loads nothing. The kept declaration retires, because every placed module has a have. Nothing then carries the node's own composition. The guard misses nats, which has no want there, so its nats.conf is guessed as nats's and removed. - At the start, when the store has no node's own composition, the node's own runs as the kept declaration has it, and the store keeps it now. The kept declaration retires only once the store holds `node`. - What this engine applied runs until loadHad reads the composition a differing have names. The bucket no longer keeps the one novox's have names, so the stored one stays. - In the node's own apply, a resource of another module goes only when that module's want says removed (Scope.RemovedByWant). A module the node's own composition carried has no want, so Placed never named it. Cost: the node's own apply cannot drop one of its own dotted resources without such a want. It is refused whole and said, never removed silently. Tests, with no new test functions: - A first-start case in the agent's Everything test (empty store, then a start with the declaration retired). It fails on 76e214a. - Two rows in the guard's table: no want refuses; a want saying removed removes. - A case for the store in the authority test. --- cmd/mesh-host/applied_store.go | 10 ++++++ cmd/mesh-host/authority_test.go | 32 ++++++++++++++++++ cmd/mesh-host/nodeagent.go | 24 +++++++++++--- internal/agent/agent.go | 51 ++++++++++++++++++++++++++--- internal/agent/agent_test.go | 50 ++++++++++++++++++++++++++++ internal/apply/apply.go | 17 ++++++++-- internal/apply/module_scope_test.go | 12 +++++++ 7 files changed, 184 insertions(+), 12 deletions(-) diff --git a/cmd/mesh-host/applied_store.go b/cmd/mesh-host/applied_store.go index 60df951..75243ed 100644 --- a/cmd/mesh-host/applied_store.go +++ b/cmd/mesh-host/applied_store.go @@ -30,6 +30,16 @@ func (s appliedStore) path(module string) (string, error) { return filepath.Join(s.dir, module+".json"), nil } +// holds says whether the store keeps the module's composition. +func (s appliedStore) holds(module string) bool { + path, err := s.path(module) + if err != nil { + return false + } + _, err = os.Stat(path) + return err == nil +} + func (s appliedStore) Load() (map[string][]byte, error) { entries, err := os.ReadDir(s.dir) if errors.Is(err, os.ErrNotExist) { diff --git a/cmd/mesh-host/authority_test.go b/cmd/mesh-host/authority_test.go index 84d8ea9..ac7bdcb 100644 --- a/cmd/mesh-host/authority_test.go +++ b/cmd/mesh-host/authority_test.go @@ -127,6 +127,38 @@ func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) { if err := certifyThisNode(context.Background(), state, "n1", trust, ask, func() { written++ }); err == nil || written != 1 { t.Errorf("a certificate of another authority was kept: %v", err) } + + // Issue 774: what the engine applied is kept beside the state too, one file per module, 0600 in a 0700 directory, + // written by a rename so a crash leaves the one before; a name that is no module's is refused, never a path. + t.Run("the compositions the engine applied are kept beside the state", func(t *testing.T) { + applied := appliedStoreFor(filepath.Join(t.TempDir(), "state", "state.json")) + if m, err := applied.Load(); err != nil || len(m) != 0 || applied.holds("node") { + t.Fatalf("an empty store loads %v, %v", m, err) + } + for _, tc := range []struct { + module string + takes bool + }{{"node", true}, {"nats", true}, {"mesh-delivery-view", true}, {"../state", false}, {"a/b", false}, {"..", false}, {"", false}} { + if err := applied.Keep(tc.module, []byte(`{"v":1}`)); (err == nil) != tc.takes { + t.Errorf("keeping %q: %v, want taken %v", tc.module, err, tc.takes) + } + } + if err := applied.Keep("node", []byte(`{"v":2}`)); err != nil { + t.Fatal(err) + } + entries, _ := os.ReadDir(applied.dir) + dirInfo, _ := os.Stat(applied.dir) + fileInfo, _ := os.Stat(filepath.Join(applied.dir, "node.json")) + if len(entries) != 3 || dirInfo.Mode().Perm() != 0o700 || fileInfo.Mode().Perm() != 0o600 { + t.Fatalf("the store holds %d files (want 3, no temporary one left), directory %v, file %v", len(entries), dirInfo.Mode(), fileInfo.Mode()) + } + if err := applied.Drop("nats"); err != nil || applied.Drop("nats") != nil || applied.holds("nats") { + t.Fatalf("a dropped module is still held, or dropping twice fails: %v", err) + } + if m, err := applied.Load(); err != nil || string(m["node"]) != `{"v":2}` || len(m) != 2 { + t.Fatalf("the store loads %v, %v", m, err) + } + }) } // testAuthority is a self-signed authority certificate for key, as the vault makes its own. diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 07fc79c..00531d8 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -454,8 +454,15 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus var ag *agent.Agent // The kept release-5 declaration is no longer read once every module the seat's placement puts here has a have of its // own (ADR 0550 §2: its signature is the controller's, trusted no more after the lease write). - if retired, why := placedAllHave(linked, session, node); retired && !keptRetired.Swap(true) { - say("the kept declaration is no longer read: " + why) + // Nor before this engine keeps the node's own as it runs (applied/node.json): until then the kept declaration is the one + // record of what the node's own composition carries, its folded modules among them (issue 774, review of the + // re-land, round 1: novox's first start would compose its own with nats running nothing). + if retired, why := placedAllHave(linked, session, node); retired && !keptRetired.Load() { + if !appliedStoreFor(opts.state).holds(agent.NodeModule) { + say("the kept declaration is still read: " + why + ", but this engine keeps no node's own composition yet") + } else if !keptRetired.Swap(true) { + say("the kept declaration is no longer read: " + why) + } } kept := keptOnce(opts, signer) // The mesh's own certificate the declaration kept from before carries, kept for the record before this machine @@ -529,7 +536,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus if herr != nil { res = failedWith(herr.Error()) } else { - res = applyComposition(actx, opts, held, others, ag.Placed(scopeName(c)), sched, say) + res = applyComposition(actx, opts, held, others, ag.Placed(scopeName(c)), ag.RemovedByWant(), sched, say) } done() // The mesh's own certificate it carries is kept for this machine's record (ADR 0555 §3). @@ -690,7 +697,10 @@ func (c *agentNow) get() *agent.Agent { // // placed is the modules a want still places here besides this one: none of their resources is removed by this apply, // which is refused instead (issue 774). -func applyComposition(ctx context.Context, opts options, c delivery.Composed, others []delivery.Composed, placed map[string]bool, +// +// removed is the modules whose want says removed: in the node's own apply, a resource of any other module goes only when +// its module is among them (issue 774). +func applyComposition(ctx context.Context, opts options, c delivery.Composed, others []delivery.Composed, placed, removed map[string]bool, sched *apply.Scheduler, say link.Announce) agent.Result { name := scopeName(c) if c.Module == "" { @@ -707,6 +717,12 @@ func applyComposition(ctx context.Context, opts options, c delivery.Composed, ot return failedWith(err.Error()) } scope := apply.Scope{Module: name, Others: resourcesOf(others), Placed: placed} + if name == agent.NodeModule { + scope.RemovedByWant = removed + if scope.RemovedByWant == nil { + scope.RemovedByWant = map[string]bool{} + } + } got := applyModuleHeld(ctx, opts, d, &scope, sched, say) if got.Refused != "" { return failedWith(got.Refused) diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 0b47ede..7b6494c 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -904,7 +904,7 @@ func (a *Agent) loadHad(ctx context.Context) { // loadApplied takes, once at the start, what this engine last applied of each module as what runs of it, ahead of the // bucket and the kept declaration (Env.Applied, issue 774): it is what the machine holds, whatever the bus still keeps. -// One whose have names another composition that runs (applied since by another engine) is left to the have; one whose +// One whose have names another composition runs as applied until loadHad reads that one from the bucket; one whose // have cannot be read, or that has none, is kept as applied, so a module is never composed as running nothing. func (a *Agent) loadApplied(ctx context.Context) { if a.env.Applied == nil { @@ -914,6 +914,30 @@ func (a *Agent) loadApplied(ctx context.Context) { if err != nil { a.env.Say(fmt.Sprintf("the compositions this engine applied cannot be read, so what runs is read from the haves alone: %v", err)) } + // At an engine's first start nothing is kept yet: the node's own runs as the kept declaration has it, which the store + // keeps now, so the node never composes its own as if its folded modules ran nothing once the declaration retires + // (issue 774, review of the re-land, round 1). Taken whatever digest the node's have names: the composition that names + // is in no bucket, and the declaration is the one record of what runs on disk. + seeded := false + if _, own := kept[NodeModule]; !own && err == nil && a.env.Kept != nil { + if rs, ok := a.env.Kept()[NodeModule]; ok && len(rs) > 0 { + c, _ := a.keptComposed(NodeModule, rs) + c.Module = "" + raw, eerr := delivery.EncodeComposed(c) + if eerr == nil { + eerr = a.env.Applied.Keep(NodeModule, raw) + } + if eerr != nil { + a.env.Say(fmt.Sprintf("the node's own as the kept declaration has it cannot be kept: %v", eerr)) + } else { + if kept == nil { + kept = map[string][]byte{} + } + kept[NodeModule], seeded = raw, true + a.env.Say("the node's own runs as the kept declaration has it, kept now as what this engine runs") + } + } + } names := make([]string, 0, len(kept)) for m := range kept { names = append(names, m) @@ -933,10 +957,13 @@ func (a *Agent) loadApplied(ctx context.Context) { continue } h, found, unread := a.readHave(ctx, m) - if found && h.Composed != "" && h.Composed != dig && haveRuns(h) { - a.env.Say(fmt.Sprintf("%s's have names the composition %s, not %s this engine applied last: what runs of it is read from its have", - m, short(h.Composed), short(dig))) - continue + if seeded && m == NodeModule { + found, unread = true, false // what the kept declaration has runs, whatever the have names + } else if found && h.Composed != "" && h.Composed != dig && haveRuns(h) { + // The composition the have names replaces this once loadHad finds it in the bucket; until then, and when + // the bucket keeps it no more, what this engine applied runs (issue 774, review of the re-land, round 1). + a.env.Say(fmt.Sprintf("%s's have names the composition %s, not %s this engine applied last: it runs as applied "+ + "until the one its have names is read", m, short(h.Composed), short(dig))) } if m == NodeModule { c.Module, c.FP = "", "" @@ -1151,6 +1178,20 @@ func (a *Agent) Placed(except string) map[string]bool { return out } +// RemovedByWant is the modules whose want says removed (apply.Scope.RemovedByWant, issue 774): in the node's own apply, +// only their resources may go. Called without a.mu held. +func (a *Agent) RemovedByWant() map[string]bool { + a.mu.Lock() + defer a.mu.Unlock() + out := map[string]bool{} + for m, w := range a.wants { + if m != NodeModule && w.Present && w.Want.Removed { + out[m] = true + } + } + return out +} + // HoldsNode says whether the agent runs the node's own composition, so the node's parts are its and not the kept // declaration's. func (a *Agent) HoldsNode() bool { diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 4aa7d9f..9ec0741 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -835,6 +835,56 @@ func TestEverythingIsWhatTheAgentRunsAndWhatIsKeptForTheRest(t *testing.T) { b.wants.put(wk, rraw) eventually(t, "the removed module's composition was not forgotten", func() bool { return store.get("web") == nil }) }) + + // Issue 774, review of the re-land, round 1: at the re-landed engine's first start nothing is kept yet, and the node's + // have names a composition the node composed itself, in no bucket. The node's own runs as the kept declaration has it + // and is kept then, so the next start, the declaration retired, still runs nats. + t.Run("at the first start nothing is kept: the node's own runs as the kept declaration has it", func(t *testing.T) { + natsConf, _ := json.Marshal(map[string]any{"id": "nats.conf", "type": "file", "path": "/tmp/r6/nats/conf", "content": "listen 4222"}) + store := &memApplied{kept: map[string][]byte{}} + h := &haves{} + h.Write(context.Background(), Have{Module: NodeModule, Composed: "sha256:" + strings.Repeat("c", 64)}) + firstPass := func(kept map[string][]json.RawMessage) []delivery.Composed { + b := newBed(t) + b.env.SelfComposed, b.env.Trusted, b.kept, b.env.Applied, b.env.Haves = true, nil, kept, store, h + firstCh := make(chan []delivery.Composed, 1) + comps := b.comps + var ag *Agent + b.env.Composed = watchFunc(func(ctx context.Context, p string) (<-chan Update, error) { + firstCh <- ag.Everything() + return comps.Watch(ctx, p) + }) + ag = New(b.env) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); _ = ag.Run(ctx) }() + defer func() { cancel(); <-done }() + select { + case got := <-firstCh: + return got + case <-time.After(5 * time.Second): + t.Fatal("the composer was not watched") + } + return nil + } + runsNats := func(cs []delivery.Composed) bool { + for _, c := range cs { + if c.Module == "" && len(c.Resources) == 1 && strings.Contains(string(c.Resources[0]), "4222") { + return true + } + } + return false + } + if got := firstPass(map[string][]json.RawMessage{NodeModule: {natsConf}}); !runsNats(got) { + t.Fatalf("the first start does not run nats in the node's own (%d compositions)", len(got)) + } + if store.get(NodeModule) == nil { + t.Fatal("the first start kept no node's own composition") + } + if got := firstPass(nil); !runsNats(got) { + t.Fatalf("the next start, the kept declaration retired, does not run nats in the node's own (%d compositions)", len(got)) + } + }) } // watchFunc is a Watcher from a function. diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 24f93c5..8eb83d8 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -316,23 +316,34 @@ type Scope struct { // the module's own removes a resource of one of them (issue 774: on 2026-10-29 the node's own apply on the bus node, // composed without what its folded modules ran, removed the bus server and four held modules). Nil guards nothing. Placed map[string]bool + // RemovedByWant are the modules whose want says removed. In the node's own apply (Module NodeModule) with this set + // non-nil, a resource of any other module is removed only when that module is here: a module the node runs folded + // in its own composition has no want, so Placed does not name it (issue 774, review of the re-land, round 1: nats on + // the bus node has none, and its nats.conf was guessed as nats's and removed). Nil guards by Placed alone. + RemovedByWant map[string]bool } // placedOwner is the module a want still places on the node that the record belongs to, other than the scope's own: // the module recorded with it, else the one its id names (a resource the node's own composition carried for a folded // module is recorded as the node's, its id naming the module). Empty when there is none. func (sc *Scope) placedOwner(a store.Applied) string { - if sc == nil || len(sc.Placed) == 0 { + if sc == nil || (len(sc.Placed) == 0 && sc.RemovedByWant == nil) { return "" } m := a.Module if m == "" || m == store.NodeModule { m, _ = declaration.ModuleGuess(a.ID) } - if m == "" || m == sc.Module || !sc.Placed[m] { + if m == "" || m == sc.Module || m == store.NodeModule { return "" } - return m + if sc.Placed[m] { + return m + } + if sc.Module == store.NodeModule && sc.RemovedByWant != nil && !sc.RemovedByWant[m] { + return m // a module the node's own composition carried, which no want removed + } + return "" } // owns reports whether a record belongs to the scope's module. diff --git a/internal/apply/module_scope_test.go b/internal/apply/module_scope_test.go index 14b95f4..51e42a5 100644 --- a/internal/apply/module_scope_test.go +++ b/internal/apply/module_scope_test.go @@ -131,10 +131,16 @@ func TestAResourceThatMovedToItsModulesCompositionIsLeftByTheNodesOwnApply(t *te placed bool // a want still places its module here refused bool // the node's own apply is refused whole stays bool // the resource is still on the machine and recorded + // wants names the modules whose want says removed (Scope.RemovedByWant); nil leaves the guard to Placed alone. + wants []string }{ {name: "moved to its module's own composition", id: "systemd-resolved.fact-suffix", moved: true, placed: true, stays: true}, {name: "a held folded module, in no composition", id: "nats.server-conf", placed: true, refused: true, stays: true}, {name: "its module's want removed", id: "nats.server-conf"}, + // Issue 774, review of the re-land, round 1: a module folded in the node's own has no want, so Placed does not name + // it; in the node's own apply only a module a want removed loses its resources. + {name: "a folded module with no want", id: "nats.server-conf", wants: []string{}, refused: true, stays: true}, + {name: "a folded module a want removed", id: "nats.server-conf", wants: []string{"nats", "old"}}, } { t.Run(tc.name, func(t *testing.T) { dir := t.TempDir() @@ -156,6 +162,12 @@ func TestAResourceThatMovedToItsModulesCompositionIsLeftByTheNodesOwnApply(t *te if tc.placed { scope.Placed = map[string]bool{module: true} } + if tc.wants != nil { + scope.RemovedByWant = map[string]bool{} + for _, m := range tc.wants { + scope.RemovedByWant[m] = true + } + } report, after, err := ApplyModule(context.Background(), archHost(t), &declaration.Declaration{}, scope, state, store.OriginDeclared, noServices, nil, nil, nil, nil) if tc.refused != (err != nil) { -- 2.54.0 From 58aacaad2d97d54dabe8ff4749d469664048b097 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 21:57:09 +0100 Subject: [PATCH 10/21] Pin mesh-sdk 7000bd5, the re-land group's sdk head: a tools bundle declares its own bus account (issue 1952 S5, issue 774) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0666c02..8a59051 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029205252-7000bd5049ec github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index 043e05f..b0b32c9 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a h1:nhRFmtdcI4VkghGUdJhpt1BnPuTQcekioOIRzW8fcG0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029200316-22c9ddd32b6a/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029205252-7000bd5049ec h1:OLXHSQkAHnf03hzJTxRrUsl7vMfMhOx43M5yJpi5L5Y= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029205252-7000bd5049ec/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= -- 2.54.0 From f7814853f9dfb9bf8c615f7cb0a50d75a2f7f4b5 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:18:45 +0100 Subject: [PATCH 11/21] Pin mesh-sdk 2e5cc4f, the re-land group's sdk head (issue 774) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8a59051..60fe963 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029205252-7000bd5049ec + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index b0b32c9..f53fc84 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029205252-7000bd5049ec h1:OLXHSQkAHnf03hzJTxRrUsl7vMfMhOx43M5yJpi5L5Y= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029205252-7000bd5049ec/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 h1:ImOyj4WqH9lwxOr95aH9Wq2iJkUwdtnEyPrASm9TrYA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= -- 2.54.0 From 6b32e0716452927b8d9483420813e05f613ec734 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:36:53 +0100 Subject: [PATCH 12/21] Fill a provision's ${secret:} at the write from the vault's value sealed to this machine, so the provisioner's client never holds it (issue 1954 L6-E1, E2; ADR 0570 clause 6) The provisioner's client fills a handed module's file but for each `${secret:}` (or a whole `secrets` file that is the placeholder alone). - The node-engine reads the provision's key `..[.]` in mesh-provisioner's `provisions`. The key must be `provided` and carry the vault's reference. - It asks the vault's `own` for that pair credential sealed to this machine, at least at the reference's generation, and opens it with the node's sealing key. - It writes the file as declared (its owner), 0600 when the declaration gives no mode. - Anything short of that refuses the write, naming the provision and why. A module the node composes itself already has its `secrets` files written so: its composition carries the vault's value from `own`, written 0600 with the declared owner. Pins mesh-sdk 4a96793 (go/provisions). Tests: rows in TestAWriteAndAnApplyOfTheSameFileSerialise. --- cmd/mesh-host/filling.go | 57 ++++++++++-- cmd/mesh-host/filling_test.go | 73 +++++++++++++++- cmd/mesh-host/main.go | 2 +- cmd/mesh-host/provision_secrets.go | 136 +++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 4 +- internal/apply/filled.go | 30 ++++++- 7 files changed, 291 insertions(+), 13 deletions(-) create mode 100644 cmd/mesh-host/provision_secrets.go diff --git a/cmd/mesh-host/filling.go b/cmd/mesh-host/filling.go index 5257e3e..373aef2 100644 --- a/cmd/mesh-host/filling.go +++ b/cmd/mesh-host/filling.go @@ -3,22 +3,25 @@ package main import ( "context" "crypto/ed25519" + "encoding/json" "errors" "fmt" "path/filepath" "sync/atomic" "git.novox.be/novox/mesh-sdk/go/declaration" + "git.novox.be/novox/mesh-sdk/go/provisions" "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/filling" + "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" ) // serveFilling writes filled files on the provisioner client's request for as long as this engine serves (novox/hq // ADR 0349 §4). A socket that cannot be served is said once, and this engine goes on applying. -func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, say func(string)) { - e := fillingEngine(opts, signer, say) +func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, queue *link.Session, node string, say func(string)) { + e := fillingEngine(opts, signer, provisionSecretsOn(queue, node, opts.state), say) fillingNow.Store(e) defer fillingNow.CompareAndSwap(e, nil) if err := filling.Serve(ctx, filling.Socket, e, say); err != nil { @@ -50,7 +53,10 @@ func knockFilling(after string, say func(string)) { // fillingEngine is the write of a filled file as this engine does it: the declaration it keeps, verified, read under // the apply's lock — the same two locks every apply takes, the process's and the one beside the state — and the file // written through the apply (apply.WriteFilled), its outcome recorded in the node's state as an apply's is. -func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *filling.Engine { +// +// A provision's `${secret:}` the client left in the content is filled first (secrets, novox/hq ADR 0570 clause 6), and a +// file so filled is written 0600 when the declaration gives no mode. +func fillingEngine(opts options, signer ed25519.PublicKey, secrets provisionSecrets, say func(string)) *filling.Engine { return &filling.Engine{ Declared: func() (*declaration.Declaration, error) { return declaredNow(opts.state, signer) @@ -74,8 +80,13 @@ func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *fi if err != nil { return "", "", err } - report, updated, writeErr := apply.WriteFilled(ctx, built, d, known, path, content, apply.ExecRunner, - announceOr(say), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state))) + content, secret, err := secrets.fill(ctx, filledModule(d, path), content) + if err != nil { + return "", "", fmt.Errorf("%s was not written: %w", path, err) + } + report, updated, writeErr := apply.WriteFilledWith(ctx, built, d, known, path, content, apply.ExecRunner, + announceOr(say), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state)), + apply.Filled{Secret: secret}) // Saved whichever way it went, as an apply's: what was written is on the machine either way. if err := store.Save(opts.state, updated); err != nil { return "", "", errors.Join(writeErr, errors.New("written, and the node's state could not be saved: "+ @@ -94,3 +105,39 @@ func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *fi }, } } + +// filledModule is the module whose file the declaration declares at path, or "" when none or several do. +func filledModule(d *declaration.Declaration, path string) string { + module := "" + for _, r := range d.Resources { + if f, ok := r.(*declaration.File); ok && f.Path == path { + if module != "" { + return "" + } + module, _ = d.ModuleOfID(f.ID) + } + } + return module +} + +// provisionSecretsOn reads provisions and asks the vault over the link, and opens with this machine's sealing key. +func provisionSecretsOn(queue *link.Session, node, statePath string) provisionSecrets { + p := provisionSecrets{node: node, open: sealOpener(statePath)} + if queue == nil { + return p + } + p.read = func(ctx context.Context, key string) ([]byte, bool, error) { + h, err := queue.KVHistory(ctx, provisions.ProvisionsBucket, key) + if err != nil { + return nil, false, err + } + if len(h) == 0 || h[len(h)-1].Deleted { + return nil, false, nil + } + return h[len(h)-1].Value, true, nil + } + p.own = func(ctx context.Context, args map[string]any) (json.RawMessage, error) { + return queue.AskVault(ctx, "own", args) + } + return p +} diff --git a/cmd/mesh-host/filling_test.go b/cmd/mesh-host/filling_test.go index 828682a..a25cc55 100644 --- a/cmd/mesh-host/filling_test.go +++ b/cmd/mesh-host/filling_test.go @@ -10,6 +10,8 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/provisions" + "github.com/novox/mesh-host/internal/filling" "github.com/novox/mesh-host/internal/store" ) @@ -45,7 +47,28 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) { store.Declared{Declaration: raw, Signature: ed25519.Sign(private, raw)}); err != nil { t.Fatal(err) } - e := fillingEngine(opts, public, func(string) {}) + // The provision m reads, as mesh-provisioner keeps it, and the vault's value of it, sealed to this machine (faked: + // the opener reads the box's "v"). + provision := map[string]string{} + secrets := provisionSecrets{node: "n1", + read: func(_ context.Context, key string) ([]byte, bool, error) { + v, ok := provision[key] + return []byte(v), ok, nil + }, + own: func(_ context.Context, args map[string]any) (json.RawMessage, error) { + raw, _ := json.Marshal(args) + var a struct{ Credentials []map[string]any } + _ = json.Unmarshal(raw, &a) + c := a.Credentials[0] + c["generation"], c["sealed"] = 2, map[string]string{"v": "s3cret"} + return json.Marshal(map[string]any{"node": "n1", "credentials": []any{c}}) + }, + open: func(sealed string) ([]byte, error) { + var b struct{ V string } + err := json.Unmarshal([]byte(sealed), &b) + return []byte(b.V), err + }} + e := fillingEngine(opts, public, secrets, func(string) {}) uid := uint32(os.Getuid()) if err := e.Accepts(uid); err != nil { t.Fatal(err) @@ -102,4 +125,52 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) { if a := <-done; a.Action != "updated" { t.Fatalf("the second write: %+v", a) } + + // ADR 0570 clause 6: the client leaves a provision's `${secret:}` to the node-engine, which fills it at the write + // from the vault's value of a provided provision, and writes the file 0600; a provision not provided refuses it. + keyed := func(state string, ref bool) string { + p := provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "m", Node: "n1"}, + Provider: provisions.End{Module: "postgres", Node: "n2"}, State: state, At: time.Now()} + if state != provisions.Provided { + p.Why = "the provider has not answered" + } + if ref { + p.Reference = &provisions.Reference{ID: "provision/postgres/db", Generation: 2} + } + raw, err := provisions.Encode(p) + if err != nil { + t.Fatal(err) + } + return string(raw) + } + for _, tc := range []struct { + name, state string + ref bool + content string + want string // the file after; "" keeps what was there + mode os.FileMode + }{ + {name: "a provision not provided is refused", state: provisions.Asked, content: "password=${secret:db}\n"}, + {name: "provided with no reference is refused", state: provisions.Provided, content: "password=${secret:db}\n"}, + {name: "provided: filled at the write, 0600", state: provisions.Provided, ref: true, + content: "password=${secret:db}\n", want: "password=s3cret\n", mode: 0o600}, + {name: "no placeholder is written as it came, as the declaration says", content: "plain\n", want: "plain\n", mode: 0o644}, + } { + before, _ := os.ReadFile(path) + if tc.state != "" { + provision["n1.m.db"] = keyed(tc.state, tc.ref) + } + a := e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte(tc.content)}) + got, _ := os.ReadFile(path) + if tc.want == "" { + if a.Failed == "" && a.Refused == "" || string(got) != string(before) { + t.Errorf("%s: answered %+v, the file holds %q", tc.name, a, got) + } + continue + } + info, _ := os.Stat(path) + if a.Failed != "" || a.Refused != "" || string(got) != tc.want || info.Mode().Perm() != tc.mode { + t.Errorf("%s: answered %+v, the file holds %q, mode %v", tc.name, a, got, info.Mode()) + } + } } diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index bcff6e6..8e52318 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1200,7 +1200,7 @@ func runLink(ctx context.Context, opts options) error { go serveMeshCLI(aside, queue, say) // And a filled file is written on the provisioner client's request, one at a time with the apply (novox/hq ADR // 0349 §4). - go serveFilling(aside, opts, mine.Membership.Signer, say) + go serveFilling(aside, opts, mine.Membership.Signer, queue, mine.Node, say) held := link.HoldRoused(aside, membership, queue, say, opts.timeout, rousedBySignal(ctx)) // **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back diff --git a/cmd/mesh-host/provision_secrets.go b/cmd/mesh-host/provision_secrets.go new file mode 100644 index 0000000..e9c32f7 --- /dev/null +++ b/cmd/mesh-host/provision_secrets.go @@ -0,0 +1,136 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strings" + + "git.novox.be/novox/mesh-sdk/go/provisions" +) + +// provisionSecrets fills a provision's value into a file the provisioner's client hands the node-engine to write +// (novox/hq ADR 0570 clause 6): the client fills the file but for each `${secret:}`, and the node-engine +// fills those at the write, so the client never holds the value. A whole `secrets` file of a handed consumer is the +// placeholder alone. +// +// For each placeholder it reads the provision's key `..` of the seat mesh-provisioner's state +// `provisions` (clause 1), which must be `provided` and carry the vault's reference; asks the vault for that pair +// credential sealed to this machine's key (the verb `own`, ADR 0349 §11 with the node-engine standing for the client, +// ADR 0550 §5); and opens it with this machine's sealing key. Anything short of that refuses the write, naming the +// provision and why: a file written with a placeholder left in it, or with no value, is a module that starts and cannot +// connect. +type provisionSecrets struct { + node string + // read is the provision's key from mesh-provisioner's `provisions`: its value and whether it is there. + read func(ctx context.Context, key string) ([]byte, bool, error) + // own asks the vault's verb `own` in this engine's name. + own func(ctx context.Context, args map[string]any) (json.RawMessage, error) + // open opens a value sealed to this machine. + open func(sealed string) ([]byte, error) +} + +// secretPlaceholder is `${secret:}`, or `${secret:.}` where the consumer keeps several values +// of one provision (ADR 0094). +var secretPlaceholder = regexp.MustCompile(`\$\{secret:([A-Za-z0-9_-]+)(?:\.([A-Za-z0-9_-]+))?\}`) + +// fill is content with each provision placeholder of module's filled, and whether any was. +func (p provisionSecrets) fill(ctx context.Context, module string, content []byte) ([]byte, bool, error) { + found := secretPlaceholder.FindAllSubmatch(content, -1) + if len(found) == 0 { + return content, false, nil + } + if module == "" { + return nil, false, fmt.Errorf("the file names %d provision secret(s) and belongs to no module, so whose they are is not known", len(found)) + } + out := string(content) + done := map[string]bool{} + for _, f := range found { + whole, provision, local := string(f[0]), string(f[1]), string(f[2]) + if done[whole] { + continue + } + done[whole] = true + value, err := p.value(ctx, module, provision, local) + if err != nil { + return nil, false, err + } + out = strings.ReplaceAll(out, whole, string(value)) + } + return []byte(out), true, nil +} + +// value is one provision's value for module on this node, opened. +func (p provisionSecrets) value(ctx context.Context, module, provision, local string) ([]byte, error) { + key, err := provisions.Key(p.node, module, provision, local) + if err != nil { + return nil, err + } + if p.read == nil || p.own == nil || p.open == nil { + return nil, fmt.Errorf("the provision %s cannot be read on this machine: the node-engine is not linked to the mesh", key) + } + raw, ok, err := p.read(ctx, key) + switch { + case err != nil: + return nil, fmt.Errorf("the provision %s cannot be read: %w", key, err) + case !ok: + return nil, fmt.Errorf("the provision %s is not in %s: nothing asks for it", key, provisions.ProvisionsRead) + } + pr, err := provisions.Decode(raw) + if err != nil { + return nil, fmt.Errorf("the provision %s: %w", key, err) + } + if pr.State != provisions.Provided || pr.Reference == nil { + why := pr.State + if pr.Why != "" { + why += ": " + pr.Why + } + if pr.State == provisions.Provided { + why = "provided with no reference to the vault's value" + } + return nil, fmt.Errorf("the provision %s is %s, so its value is not written yet", key, why) + } + type asked struct { + Role string `json:"role"` + Provision string `json:"provision"` + Consumer string `json:"consumer"` + ConsumerModule string `json:"consumer_module"` + Provider string `json:"provider"` + Local string `json:"local,omitempty"` + } + ask := asked{Role: "consumer", Provision: provision, Consumer: p.node, ConsumerModule: module, Provider: pr.Provider.Node, Local: local} + answer, err := p.own(ctx, map[string]any{"credentials": []asked{ask}}) + if err != nil { + return nil, fmt.Errorf("the vault's value of %s: %w", key, err) + } + var a struct { + Node string `json:"node"` + Credentials []struct { + asked + Generation int `json:"generation"` + Sealed json.RawMessage `json:"sealed"` + } `json:"credentials"` + } + if err := json.Unmarshal(answer, &a); err != nil { + return nil, fmt.Errorf("the vault's answer for %s is not one: %w", key, err) + } + if a.Node != p.node { + return nil, fmt.Errorf("the vault answered %s's credentials for %s on %s", a.Node, key, p.node) + } + for _, c := range a.Credentials { + if c.asked != ask { + continue + } + if c.Generation < pr.Reference.Generation { + return nil, fmt.Errorf("the vault holds %s at generation %d and the provision names %d: not written until it has it", + key, c.Generation, pr.Reference.Generation) + } + value, err := p.open(string(c.Sealed)) + if err != nil { + return nil, fmt.Errorf("the vault's value of %s does not open on this machine: %w", key, err) + } + return value, nil + } + return nil, fmt.Errorf("the vault answered no value of %s for this machine", key) +} diff --git a/go.mod b/go.mod index 60fe963..675bca2 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index f53fc84..3c95de2 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 h1:ImOyj4WqH9lwxOr95aH9Wq2iJkUwdtnEyPrASm9TrYA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368 h1:LaBxsyU3HeTIPKYmvpJuw+Wm2plpf6O7qHO91O+mAi0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/internal/apply/filled.go b/internal/apply/filled.go index 5d4d2ca..6bd9b7a 100644 --- a/internal/apply/filled.go +++ b/internal/apply/filled.go @@ -45,6 +45,30 @@ func WriteFilled( log func(string), keep Keep, windows *Windows, +) (report Report, _ store.State, _ error) { + return WriteFilledWith(ctx, sys, d, known, path, content, run, log, keep, windows, Filled{}) +} + +// Filled is what the node-engine did to a filled file's content before its write. +type Filled struct { + // Secret says the node-engine filled a value into the content, a provision's `${secret:}` (novox/hq ADR 0570 + // clause 6): the file is then written 0600 when the declaration gives no mode, as a file carrying a secret is. + Secret bool +} + +// WriteFilledWith is WriteFilled for content the node-engine filled before the write (Filled). +func WriteFilledWith( + ctx context.Context, + sys system.System, + d *declaration.Declaration, + known store.State, + path string, + content []byte, + run Runner, + log func(string), + keep Keep, + windows *Windows, + filled Filled, ) (report Report, _ store.State, _ error) { if log == nil { log = func(string) {} @@ -65,7 +89,7 @@ func WriteFilled( return report, known, &Error{Resource: path, Err: fmt.Errorf("%d resources declare %s, and a filled file "+ "is written for exactly one, so nothing was written", len(declared), path)} } - file := filledAs(declared[0], content) + file := filledAs(declared[0], content, filled.Secret) if held, isHeld := known.HeldAt(file.ID); isHeld { return report, known, &Error{Resource: file.ID, Err: fmt.Errorf("%s is held as found for a module not "+ "yet taken (%s), so nothing was written", path, held.Why)} @@ -163,10 +187,10 @@ func WriteFilled( } // filledAs is the declared file with the filled content in place of the declaration's. -func filledAs(declared *declaration.File, content []byte) *declaration.File { +func filledAs(declared *declaration.File, content []byte, secret bool) *declaration.File { f := *declared f.Content, f.Sealed, f.VaultSealed, f.Bytes, f.Secrets = string(content), "", nil, "", nil - if f.Mode == "" && (declared.Secret() || len(declared.Secrets) > 0) { + if f.Mode == "" && (secret || declared.Secret() || len(declared.Secrets) > 0) { // Written as the declaration would have written it: a file carrying a credential is not world-readable. f.Mode = "0600" } -- 2.54.0 From 5497d362dd5715ef93ff0c1011fe3489f67e6cf8 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:37:44 +0100 Subject: [PATCH 13/21] Refuse a provision's value unless the vault's credential name and generation are the ones its reference names (issue 1954 L6-E1; ADR 0570 clauses 1, 6) The value written is the one the provision names: a1-47 makes Reference.ID the vault's credential name. A mismatch in either refuses the write, naming both. Test: a row in TestAWriteAndAnApplyOfTheSameFileSerialise. --- cmd/mesh-host/filling_test.go | 10 +++++++++- cmd/mesh-host/provision_secrets.go | 8 +++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/cmd/mesh-host/filling_test.go b/cmd/mesh-host/filling_test.go index a25cc55..cc5df0e 100644 --- a/cmd/mesh-host/filling_test.go +++ b/cmd/mesh-host/filling_test.go @@ -50,6 +50,7 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) { // The provision m reads, as mesh-provisioner keeps it, and the vault's value of it, sealed to this machine (faked: // the opener reads the box's "v"). provision := map[string]string{} + vaultName := "provision/postgres/db" secrets := provisionSecrets{node: "n1", read: func(_ context.Context, key string) ([]byte, bool, error) { v, ok := provision[key] @@ -60,7 +61,7 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) { var a struct{ Credentials []map[string]any } _ = json.Unmarshal(raw, &a) c := a.Credentials[0] - c["generation"], c["sealed"] = 2, map[string]string{"v": "s3cret"} + c["name"], c["generation"], c["sealed"] = vaultName, 2, map[string]string{"v": "s3cret"} return json.Marshal(map[string]any{"node": "n1", "credentials": []any{c}}) }, open: func(sealed string) ([]byte, error) { @@ -149,9 +150,12 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) { content string want string // the file after; "" keeps what was there mode os.FileMode + vault string // the vault's credential name, when not the one the provision names }{ {name: "a provision not provided is refused", state: provisions.Asked, content: "password=${secret:db}\n"}, {name: "provided with no reference is refused", state: provisions.Provided, content: "password=${secret:db}\n"}, + {name: "the vault's value is another than the provision names", state: provisions.Provided, ref: true, + vault: "pair/n2/db/n1/m", content: "password=${secret:db}\n"}, {name: "provided: filled at the write, 0600", state: provisions.Provided, ref: true, content: "password=${secret:db}\n", want: "password=s3cret\n", mode: 0o600}, {name: "no placeholder is written as it came, as the declaration says", content: "plain\n", want: "plain\n", mode: 0o644}, @@ -160,6 +164,10 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) { if tc.state != "" { provision["n1.m.db"] = keyed(tc.state, tc.ref) } + vaultName = "provision/postgres/db" + if tc.vault != "" { + vaultName = tc.vault + } a := e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte(tc.content)}) got, _ := os.ReadFile(path) if tc.want == "" { diff --git a/cmd/mesh-host/provision_secrets.go b/cmd/mesh-host/provision_secrets.go index e9c32f7..887cc4c 100644 --- a/cmd/mesh-host/provision_secrets.go +++ b/cmd/mesh-host/provision_secrets.go @@ -108,6 +108,7 @@ func (p provisionSecrets) value(ctx context.Context, module, provision, local st Node string `json:"node"` Credentials []struct { asked + Name string `json:"name"` Generation int `json:"generation"` Sealed json.RawMessage `json:"sealed"` } `json:"credentials"` @@ -122,9 +123,10 @@ func (p provisionSecrets) value(ctx context.Context, module, provision, local st if c.asked != ask { continue } - if c.Generation < pr.Reference.Generation { - return nil, fmt.Errorf("the vault holds %s at generation %d and the provision names %d: not written until it has it", - key, c.Generation, pr.Reference.Generation) + // The value the provision names, by the vault's credential name and its generation (clause 6): another is refused. + if c.Name != pr.Reference.ID || c.Generation != pr.Reference.Generation { + return nil, fmt.Errorf("the vault answered %s generation %d for %s, and the provision names %s generation %d: "+ + "nothing is written until they agree", c.Name, c.Generation, key, pr.Reference.ID, pr.Reference.Generation) } value, err := p.open(string(c.Sealed)) if err != nil { -- 2.54.0 From cbe8528f91f2840576d609f3647d6672006d316a Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:41:44 +0100 Subject: [PATCH 14/21] Resolve ${secret:} through this node's provision keys, and check the reference by the vault's own name (issue 1954 L6-E1, E2; ADR 0570 clauses 1, 6; ADR 0094) - A placeholder is `${secret:}`, the name the module calls the credential (catalogue.SecretLocal): the local name where it keeps several values of one provision, else the provision. The `.` form goes. - The engine finds the one key under `..` of mesh-provisioner's `provisions` going by that name. That is what node. may read, by a watch to its caught-up mark; plans are not read. - The reference's ID must be a credential name of that provision (provisions.VaultName). The vault's answer must carry exactly that name and generation. Anything else refuses the write, naming both. Pins mesh-sdk 66ff33d. Tests: rows in TestAWriteAndAnApplyOfTheSameFileSerialise (a local name, a name no provision goes by, a reference to no credential of the provision, a mismatched answer). --- cmd/mesh-host/filling.go | 30 ++++++++--- cmd/mesh-host/filling_test.go | 40 ++++++++++---- cmd/mesh-host/provision_secrets.go | 84 ++++++++++++++++++++---------- go.mod | 2 +- go.sum | 4 +- 5 files changed, 115 insertions(+), 45 deletions(-) diff --git a/cmd/mesh-host/filling.go b/cmd/mesh-host/filling.go index 373aef2..fa241e8 100644 --- a/cmd/mesh-host/filling.go +++ b/cmd/mesh-host/filling.go @@ -126,15 +126,33 @@ func provisionSecretsOn(queue *link.Session, node, statePath string) provisionSe if queue == nil { return p } - p.read = func(ctx context.Context, key string) ([]byte, bool, error) { - h, err := queue.KVHistory(ctx, provisions.ProvisionsBucket, key) + // What the bus holds under prefix, read by a watch to its caught-up mark: no poll, no list of every key. + p.list = func(ctx context.Context, prefix string) (map[string][]byte, error) { + wctx, cancel := context.WithCancel(ctx) + defer cancel() + updates, err := queue.KVWatch(wctx, provisions.ProvisionsBucket, prefix+">") if err != nil { - return nil, false, err + return nil, err } - if len(h) == 0 || h[len(h)-1].Deleted { - return nil, false, nil + out := map[string][]byte{} + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case u, ok := <-updates: + if !ok { + return nil, fmt.Errorf("the watch of %s ended before it caught up", provisions.ProvisionsBucket) + } + switch { + case u.Caught: + return out, nil + case u.Deleted: + delete(out, u.Key) + default: + out[u.Key] = u.Value + } + } } - return h[len(h)-1].Value, true, nil } p.own = func(ctx context.Context, args map[string]any) (json.RawMessage, error) { return queue.AskVault(ctx, "own", args) diff --git a/cmd/mesh-host/filling_test.go b/cmd/mesh-host/filling_test.go index cc5df0e..7885eb7 100644 --- a/cmd/mesh-host/filling_test.go +++ b/cmd/mesh-host/filling_test.go @@ -7,6 +7,7 @@ import ( "os" "os/user" "path/filepath" + "strings" "testing" "time" @@ -50,11 +51,16 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) { // The provision m reads, as mesh-provisioner keeps it, and the vault's value of it, sealed to this machine (faked: // the opener reads the box's "v"). provision := map[string]string{} - vaultName := "provision/postgres/db" + vaultName := "provision/n2/db" secrets := provisionSecrets{node: "n1", - read: func(_ context.Context, key string) ([]byte, bool, error) { - v, ok := provision[key] - return []byte(v), ok, nil + list: func(_ context.Context, prefix string) (map[string][]byte, error) { + out := map[string][]byte{} + for k, v := range provision { + if strings.HasPrefix(k, prefix) { + out[k] = []byte(v) + } + } + return out, nil }, own: func(_ context.Context, args map[string]any) (json.RawMessage, error) { raw, _ := json.Marshal(args) @@ -129,14 +135,14 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) { // ADR 0570 clause 6: the client leaves a provision's `${secret:}` to the node-engine, which fills it at the write // from the vault's value of a provided provision, and writes the file 0600; a provision not provided refuses it. - keyed := func(state string, ref bool) string { - p := provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "m", Node: "n1"}, + keyed := func(state string, ref bool, id, local string) string { + p := provisions.Provision{Provision: "db", Local: local, Consumer: provisions.End{Module: "m", Node: "n1"}, Provider: provisions.End{Module: "postgres", Node: "n2"}, State: state, At: time.Now()} if state != provisions.Provided { p.Why = "the provider has not answered" } if ref { - p.Reference = &provisions.Reference{ID: "provision/postgres/db", Generation: 2} + p.Reference = &provisions.Reference{ID: id, Generation: 2} } raw, err := provisions.Encode(p) if err != nil { @@ -151,20 +157,36 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) { want string // the file after; "" keeps what was there mode os.FileMode vault string // the vault's credential name, when not the one the provision names + refID string // the reference's id, when not the provision's shared value + local string // the provision's local name (ADR 0094): the placeholder names it }{ {name: "a provision not provided is refused", state: provisions.Asked, content: "password=${secret:db}\n"}, {name: "provided with no reference is refused", state: provisions.Provided, content: "password=${secret:db}\n"}, + {name: "a reference to no credential of the provision is refused", state: provisions.Provided, ref: true, + refID: "provision/n9/db", content: "password=${secret:db}\n"}, + {name: "a name no provision goes by is refused", state: provisions.Provided, ref: true, content: "password=${secret:other}\n"}, {name: "the vault's value is another than the provision names", state: provisions.Provided, ref: true, vault: "pair/n2/db/n1/m", content: "password=${secret:db}\n"}, {name: "provided: filled at the write, 0600", state: provisions.Provided, ref: true, content: "password=${secret:db}\n", want: "password=s3cret\n", mode: 0o600}, + {name: "a local name is the provision's by its key", state: provisions.Provided, ref: true, local: "admin", + content: "admin=${secret:admin}\n", want: "admin=s3cret\n", mode: 0o600}, {name: "no placeholder is written as it came, as the declaration says", content: "plain\n", want: "plain\n", mode: 0o644}, } { before, _ := os.ReadFile(path) if tc.state != "" { - provision["n1.m.db"] = keyed(tc.state, tc.ref) + id := "provision/n2/db" + if tc.refID != "" { + id = tc.refID + } + clear(provision) + key := "n1.m.db" + if tc.local != "" { + key += "." + tc.local + } + provision[key] = keyed(tc.state, tc.ref, id, tc.local) } - vaultName = "provision/postgres/db" + vaultName = "provision/n2/db" if tc.vault != "" { vaultName = tc.vault } diff --git a/cmd/mesh-host/provision_secrets.go b/cmd/mesh-host/provision_secrets.go index 887cc4c..a99fca0 100644 --- a/cmd/mesh-host/provision_secrets.go +++ b/cmd/mesh-host/provision_secrets.go @@ -5,8 +5,10 @@ import ( "encoding/json" "fmt" "regexp" + "sort" "strings" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/provisions" ) @@ -15,25 +17,27 @@ import ( // fills those at the write, so the client never holds the value. A whole `secrets` file of a handed consumer is the // placeholder alone. // -// For each placeholder it reads the provision's key `..` of the seat mesh-provisioner's state -// `provisions` (clause 1), which must be `provided` and carry the vault's reference; asks the vault for that pair +// A placeholder names the credential as the module calls it, `${secret:}` (catalogue.SecretLocal, ADR 0094): the +// local name where the module keeps several values of one provision, else the provision. For each, it finds the one +// provision key `..[.]` of the seat mesh-provisioner's state `provisions` (clause 1) +// going by that name, which must be `provided` and carry the vault's reference; asks the vault for that pair // credential sealed to this machine's key (the verb `own`, ADR 0349 §11 with the node-engine standing for the client, // ADR 0550 §5); and opens it with this machine's sealing key. Anything short of that refuses the write, naming the // provision and why: a file written with a placeholder left in it, or with no value, is a module that starts and cannot // connect. type provisionSecrets struct { node string - // read is the provision's key from mesh-provisioner's `provisions`: its value and whether it is there. - read func(ctx context.Context, key string) ([]byte, bool, error) + // list is every key of mesh-provisioner's `provisions` under prefix (`..`), by key: what this + // engine may read, its own node's (`.>`). + list func(ctx context.Context, prefix string) (map[string][]byte, error) // own asks the vault's verb `own` in this engine's name. own func(ctx context.Context, args map[string]any) (json.RawMessage, error) // open opens a value sealed to this machine. open func(sealed string) ([]byte, error) } -// secretPlaceholder is `${secret:}`, or `${secret:.}` where the consumer keeps several values -// of one provision (ADR 0094). -var secretPlaceholder = regexp.MustCompile(`\$\{secret:([A-Za-z0-9_-]+)(?:\.([A-Za-z0-9_-]+))?\}`) +// secretPlaceholder is `${secret:}`. +var secretPlaceholder = regexp.MustCompile(`\$\{secret:([A-Za-z0-9_-]+)\}`) // fill is content with each provision placeholder of module's filled, and whether any was. func (p provisionSecrets) fill(ctx context.Context, module string, content []byte) ([]byte, bool, error) { @@ -44,15 +48,27 @@ func (p provisionSecrets) fill(ctx context.Context, module string, content []byt if module == "" { return nil, false, fmt.Errorf("the file names %d provision secret(s) and belongs to no module, so whose they are is not known", len(found)) } + if p.list == nil || p.own == nil || p.open == nil { + return nil, false, fmt.Errorf("%s's provisions cannot be read on this machine: the node-engine is not linked to the mesh", module) + } + prefix := p.node + "." + module + "." + keys, err := p.list(ctx, prefix) + if err != nil { + return nil, false, fmt.Errorf("%s's provisions cannot be read: %w", module, err) + } out := string(content) done := map[string]bool{} for _, f := range found { - whole, provision, local := string(f[0]), string(f[1]), string(f[2]) + whole, name := string(f[0]), string(f[1]) if done[whole] { continue } done[whole] = true - value, err := p.value(ctx, module, provision, local) + pr, key, err := byName(keys, name) + if err != nil { + return nil, false, fmt.Errorf("%s's secret %q: %w", module, name, err) + } + value, err := p.value(ctx, key, pr) if err != nil { return nil, false, err } @@ -61,26 +77,31 @@ func (p provisionSecrets) fill(ctx context.Context, module string, content []byt return []byte(out), true, nil } -// value is one provision's value for module on this node, opened. -func (p provisionSecrets) value(ctx context.Context, module, provision, local string) ([]byte, error) { - key, err := provisions.Key(p.node, module, provision, local) - if err != nil { - return nil, err +// byName is the one provision of keys the module calls name (catalogue.SecretLocal), and its key. +func byName(keys map[string][]byte, name string) (provisions.Provision, string, error) { + var hit []string + var got provisions.Provision + for key, raw := range keys { + pr, err := provisions.Decode(raw) + if err != nil { + return provisions.Provision{}, "", fmt.Errorf("the provision %s: %w", key, err) + } + if catalogue.SecretLocal(pr.Provision, pr.Local) == name { + hit, got = append(hit, key), pr + } } - if p.read == nil || p.own == nil || p.open == nil { - return nil, fmt.Errorf("the provision %s cannot be read on this machine: the node-engine is not linked to the mesh", key) - } - raw, ok, err := p.read(ctx, key) - switch { - case err != nil: - return nil, fmt.Errorf("the provision %s cannot be read: %w", key, err) - case !ok: - return nil, fmt.Errorf("the provision %s is not in %s: nothing asks for it", key, provisions.ProvisionsRead) - } - pr, err := provisions.Decode(raw) - if err != nil { - return nil, fmt.Errorf("the provision %s: %w", key, err) + switch len(hit) { + case 0: + return provisions.Provision{}, "", fmt.Errorf("no provision of %s goes by that name: nothing asks for it", provisions.ProvisionsRead) + case 1: + return got, hit[0], nil } + sort.Strings(hit) + return provisions.Provision{}, "", fmt.Errorf("the provisions %s all go by that name", strings.Join(hit, ", ")) +} + +// value is one provision's value on this node, opened. +func (p provisionSecrets) value(ctx context.Context, key string, pr provisions.Provision) ([]byte, error) { if pr.State != provisions.Provided || pr.Reference == nil { why := pr.State if pr.Why != "" { @@ -91,6 +112,15 @@ func (p provisionSecrets) value(ctx context.Context, module, provision, local st } return nil, fmt.Errorf("the provision %s is %s, so its value is not written yet", key, why) } + // The reference names a credential of this provision, as the vault names it (provisions.VaultName). + ours := false + for _, form := range []string{provisions.FormPair, provisions.FormShared, provisions.FormIssued} { + ours = ours || provisions.VaultName(form, pr) == pr.Reference.ID + } + if !ours { + return nil, fmt.Errorf("the provision %s names the vault's %s, which is no credential of it", key, pr.Reference.ID) + } + module, provision, local := pr.Consumer.Module, pr.Provision, pr.Local type asked struct { Role string `json:"role"` Provision string `json:"provision"` diff --git a/go.mod b/go.mod index 675bca2..6a00b6c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index 3c95de2..326dffd 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368 h1:LaBxsyU3HeTIPKYmvpJuw+Wm2plpf6O7qHO91O+mAi0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d h1:J1QAvzlaIvCCyZGBQX0XGJjYWdNPjLXBF4zaAeIkUOg= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= -- 2.54.0 From 65fbc1e6f876bafc53b71f4683c9058fd6ff8e1c Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:56:14 +0100 Subject: [PATCH 15/21] Write each node's health report into its own key of mesh-host_health on a change, so the watchman raises the machine's alerts without the controller (issue 774, the operator's alert table) Until now the six machine alerts came from a health message passing to the controller. The report is now kept in the engine's declared state `health`, key ``, created by the grants part as `acts` is. Only this node's engine writes it, after comparing it on every look and only when what it says changed; it is read first after a restart. It carries: - the network parts (machine-network; network-rewritten through the resolver file part's writer and owner); - the tunnel's peers, each seen within five minutes or not (network-unreachable); - the failed units no module owns (machine-units); - the accounts marked relogin needed (relogin-needed); - the directories used as found (directory-used-as-found). The streaks and the time of the look are left out, so a look alone writes nothing. A peer is reported seen or not, never its handshake time, so a renewed handshake writes nothing. The layout is internal/link's (NodeHealth) until it moves into mesh-sdk. Pins mesh-sdk 58a8324. Tests: - rows in TestOnTheHubAnyFreshPeerIsAHealthyTunnel: the peers, and a change only when one is seen or lost; - a case in TestAHealthStatementIsSaidOnTheLinkOpenNow: what the report takes, and one write per change across a restart; - the manifest's states in TestTheManifestConsumesTheBackupEvent. --- cmd/mesh-host/main.go | 34 ++++- go.mod | 2 +- go.sum | 4 +- internal/link/health_test.go | 68 +++++++++ internal/link/nodehealth.go | 228 +++++++++++++++++++++++++++++++ internal/link/session.go | 3 + internal/network/network.go | 53 ++++++- internal/network/network_test.go | 20 ++- module.json | 3 + packaging/operator_keys_test.go | 5 +- 10 files changed, 406 insertions(+), 14 deletions(-) create mode 100644 internal/link/nodehealth.go diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 8e52318..0be295e 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1353,6 +1353,8 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, var lastSaid time.Time owed := true spaced := 1.0 + // healthUnwritten is why the health report was last not written, said once each time it changes. + var healthUnwritten string // A module whose change ended unjudged is looked at and said at once (review of fix 15): a module with nothing // long-running is then ok within a look of its apply, not at the five-minute statement. var nudged <-chan struct{} @@ -1436,6 +1438,11 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) && (unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy()) && len(found) == 0 && accessesHealthy(accessSt) + h := *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), + unitSt), accountSt), found), accessSt) + // The node's own health report in its key on the bus, compared on every look and written only when what it says + // changed (issue 774): what the watchman raises the machine's alerts from, without the controller. + writeHealthReport(ctx, queue, h, netSt, &healthUnwritten, say) if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway { continue } @@ -1462,13 +1469,36 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, reconcile(inactive...) } } - if queue.SayHealth(ctx, *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), - unitSt), accountSt), found), accessSt)) { + if queue.SayHealth(ctx, h) { lastSaid, owed = time.Now(), false } } } +// writeHealthReport writes this node's health report to its key (link.WriteNodeHealth) when it changed. Why it could +// not is said once per reason, and said again when it is written after all. +func writeHealthReport(ctx context.Context, queue *link.Session, h link.Health, ns *network.Statement, unwritten *string, + say link.Announce) { + if queue == nil { + return + } + var peers []link.PeerSeen + if ns != nil { + for _, p := range ns.Peers { + peers = append(peers, link.PeerSeen{Key: p.Key, Addresses: p.Addresses, Seen: p.Seen}) + } + } + _, err := queue.WriteNodeHealth(ctx, link.NodeHealthOf(queue.Membership.Node, h, peers)) + switch { + case err != nil && err.Error() != *unwritten: + *unwritten = err.Error() + say("this machine's health report is not written to the bus: " + err.Error()) + case err == nil && *unwritten != "": + *unwritten = "" + say("this machine's health report is written to the bus again") + } +} + // healthAsReported is a statement as the report and the event carry it. func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health { // LibraryContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a diff --git a/go.mod b/go.mod index 6a00b6c..e49e5bd 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215315-58a83249c80b github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index 326dffd..e33c19d 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d h1:J1QAvzlaIvCCyZGBQX0XGJjYWdNPjLXBF4zaAeIkUOg= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213929-66ff33dbd35d/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215315-58a83249c80b h1:1b3bqhFHWosEhxlsjMI6NEWFH+vdD3hqXVIysyLhPYU= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215315-58a83249c80b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/internal/link/health_test.go b/internal/link/health_test.go index db7935a..58282a2 100644 --- a/internal/link/health_test.go +++ b/internal/link/health_test.go @@ -50,6 +50,74 @@ func TestAHealthStatementIsSaidOnTheLinkOpenNow(t *testing.T) { if HealthSubject("anchor") != "mesh.control.anchor.health" { t.Fatalf("the subject %s is not inside the host's own grant", HealthSubject("anchor")) } + + // Issue 774: the node's own health report in its key of mesh-host_health, for the watchman's six alerts: written on + // a change only, the streaks and the time of the look not being one, and read first after a restart. + t.Run("the node's health report is written on a change only", func(t *testing.T) { + since := time.Date(2026, 10, 29, 20, 0, 0, 0, time.UTC) + look := func(streak int, writer string) Health { + return Health{At: time.Now(), Network: &NetworkHealth{State: StateUnhealthy, Since: since, Parts: []NetworkPart{ + {Part: "resolv-conf", State: StateUnhealthy, Writer: writer, Owner: "dnsmasq", Since: since, Streak: streak}}}, + Units: &UnitsHealth{State: "degraded", Failed: []FailedUnit{{Unit: "cups.service", Scope: "system", Since: since}}}, + Resources: []ResourceHealth{ + {Module: "brother", Resource: "brother.scanner", Kind: KindAccount, Account: "jochen", State: StateUnhealthy, + Reason: "relogin needed: the session began before scanner", Since: since}, + {Module: "photos", Resource: "photos.library", Kind: KindDirectory, Target: "/srv/photos", State: StateUnhealthy, + Reason: ReasonUsedAsFound + " owned by root", Since: since}, + {Module: "letta", Resource: "letta.server", Kind: "container", State: StateHealthy, Since: since}}} + } + peers := []PeerSeen{{Key: "B=", Seen: true}, {Key: "A=", Addresses: []string{"10.10.0.2/32"}}} + r := NodeHealthOf("anchor", look(1, "NetworkManager"), peers) + if r.Network.Parts[0].Writer != "NetworkManager" || r.Network.Parts[0].Streak != 0 || r.Peers[0].Key != "A=" || + len(r.Units.Failed) != 1 || len(r.Relogin) != 1 || r.Relogin[0].Account != "jochen" || + len(r.Found) != 1 || r.Found[0].Path != "/srv/photos" { + t.Fatalf("the report is %+v", r) + } + store := &memHealth{} + var last *NodeHealth + clock := func() time.Time { return since } + for i, step := range []struct { + h NodeHealth + fresh bool // a restart: nothing written in this run yet + wrote bool + }{ + {h: r, wrote: true}, + {h: NodeHealthOf("anchor", look(2, "NetworkManager"), peers)}, + {h: NodeHealthOf("anchor", look(3, "NetworkManager"), peers), fresh: true}, + {h: NodeHealthOf("anchor", look(3, "openvpn"), peers), wrote: true}, + {h: NodeHealthOf("anchor", look(3, "openvpn"), []PeerSeen{{Key: "A=", Seen: true}}), wrote: true}, + } { + if step.fresh { + last = nil + } + wrote, err := writeNodeHealth(t.Context(), store, &last, step.h, clock) + if err != nil || wrote != step.wrote { + t.Fatalf("step %d: wrote %v (want %v), %v", i, wrote, step.wrote, err) + } + } + got, err := DecodeNodeHealth(store.values["anchor"]) + if err != nil || got.Node != "anchor" || got.Network.Parts[0].Writer != "openvpn" || !got.Peers[0].Seen || got.At.IsZero() { + t.Fatalf("the key holds %+v, %v", got, err) + } + if store.puts != 3 { + t.Fatalf("%d writes, want 3: one per change", store.puts) + } + }) +} + +// memHealth is a HealthStore in memory. +type memHealth struct { + values map[string][]byte + puts int +} + +func (m *memHealth) Get(_ context.Context, key string) ([]byte, error) { return m.values[key], nil } +func (m *memHealth) Put(_ context.Context, key string, value []byte) error { + if m.values == nil { + m.values = map[string][]byte{} + } + m.values[key], m.puts = value, m.puts+1 + return nil } // A declared tool check reads the node tools' reply envelope: healthy only when the tool says so (novox/hq diff --git a/internal/link/nodehealth.go b/internal/link/nodehealth.go new file mode 100644 index 0000000..4ec92c9 --- /dev/null +++ b/internal/link/nodehealth.go @@ -0,0 +1,228 @@ +package link + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + "github.com/nats-io/nats.go" +) + +// The node-engine's own health report in the bus's own storage (issue 774, the operator's alert table, 2026-10-29): one +// key per node, ``, of the engine's declared state `health` (HealthBucket, created by the grants part from the +// engine's manifest as its state `acts` is), written by this node's engine alone and only when what it says changed. +// The watchman (seat mesh-watchdogs) reads it for six alerts the controller raised from the passing health message: +// +// - machine-network: Network, the machine's route, tunnel, name servers, resolver file and bus, each a part; +// - network-rewritten: the resolver file's part, unhealthy with Writer, the program that rewrote it, and Owner, the +// module whose file it is; +// - network-unreachable: Peers, each peer of the mesh's tunnel and whether this machine saw it within five minutes, +// read across every node's report; +// - machine-units: Units, the failed system units no module owns; +// - relogin-needed: Relogin, each account whose running session lacks a group it was given; +// - directory-used-as-found: Found, each directory a module declares that the engine uses as found. +// +// The layout moves into mesh-sdk before the push (the lead, 2026-10-29); until then it is this package's. + +// HealthBucket is the node-engine's declared state health: key ``, value NodeHealth. +const HealthBucket = "mesh-host_health" + +// NodeHealthVersion is the version of NodeHealth's layout. +const NodeHealthVersion = 1 + +// NodeHealth is one node's health report, as its engine last found it. +type NodeHealth struct { + V int `json:"v"` + Node string `json:"node"` + // Network is the machine's networking, its parts as Health.Network says them, without the streaks; nil where this + // engine does not judge it yet. + Network *NetworkHealth `json:"network,omitempty"` + // Peers are the mesh tunnel's peers on the last look that read them, sorted by key. + Peers []PeerSeen `json:"peers,omitempty"` + // Units is the machine's service managers: each failed unit no module places (Health.Units). + Units *UnitsHealth `json:"units,omitempty"` + // Relogin are the accounts whose running session needs a new login for a group it was given. + Relogin []NodeAccount `json:"relogin,omitempty"` + // Found are the directories used as found. + Found []NodeDirectory `json:"found,omitempty"` + // At is when the engine wrote this, on a change: never a reason to write. + At time.Time `json:"at"` +} + +// PeerSeen is one peer of the mesh's tunnel: its public key, its allowed addresses, and whether this machine saw a +// handshake with it within five minutes. +type PeerSeen struct { + Key string `json:"key"` + Addresses []string `json:"addresses,omitempty"` + Seen bool `json:"seen"` +} + +// NodeAccount is one account marked relogin needed, with the module whose resource it is and why. +type NodeAccount struct { + Account string `json:"account"` + Module string `json:"module,omitempty"` + Resource string `json:"resource,omitempty"` + Reason string `json:"reason"` + Since time.Time `json:"since"` +} + +// NodeDirectory is one directory used as found: the module declaring it, its resource and path, and why. +type NodeDirectory struct { + Module string `json:"module"` + Resource string `json:"resource"` + Path string `json:"path"` + Reason string `json:"reason"` + Since time.Time `json:"since"` +} + +// ReasonReloginNeeded starts an account's reason when its session needs a new login (accounts.ReasonRelogin). +const ReasonReloginNeeded = "relogin needed" + +// NodeHealthOf is the report a health statement and the tunnel's peers make: the statement's network without its +// streaks, its machine units, and each account and directory of its resources that the six alerts read. +func NodeHealthOf(node string, h Health, peers []PeerSeen) NodeHealth { + out := NodeHealth{V: NodeHealthVersion, Node: node, Units: h.Units} + if len(peers) > 0 { + out.Peers = append([]PeerSeen(nil), peers...) + sort.Slice(out.Peers, func(i, k int) bool { return out.Peers[i].Key < out.Peers[k].Key }) + } + if h.Network != nil { + n := NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: make([]NetworkPart, 0, len(h.Network.Parts))} + for _, p := range h.Network.Parts { + p.Streak = 0 // a look's count moves every look; the report says a part's state, not how often it was seen + n.Parts = append(n.Parts, p) + } + out.Network = &n + } + for _, r := range h.Resources { + switch { + case r.Kind == KindAccount && r.State == StateUnhealthy && strings.HasPrefix(r.Reason, ReasonReloginNeeded): + out.Relogin = append(out.Relogin, NodeAccount{Account: r.Account, Module: r.Module, Resource: r.Resource, + Reason: r.Reason, Since: r.Since}) + case r.Kind == KindDirectory && strings.HasPrefix(r.Reason, ReasonUsedAsFound): + out.Found = append(out.Found, NodeDirectory{Module: r.Module, Resource: r.Resource, Path: r.Target, + Reason: r.Reason, Since: r.Since}) + } + } + sort.Slice(out.Relogin, func(i, k int) bool { + return out.Relogin[i].Account+out.Relogin[i].Module < out.Relogin[k].Account+out.Relogin[k].Module + }) + sort.Slice(out.Found, func(i, k int) bool { return out.Found[i].Path < out.Found[k].Path }) + return out +} + +// EncodeNodeHealth is h as JSON at its version. +func EncodeNodeHealth(h NodeHealth) ([]byte, error) { + h.V = NodeHealthVersion + if h.Node == "" { + return nil, errors.New("a node's health report names its node") + } + return json.Marshal(h) +} + +// DecodeNodeHealth reads a report. +func DecodeNodeHealth(raw []byte) (NodeHealth, error) { + var h NodeHealth + if err := json.Unmarshal(raw, &h); err != nil { + return NodeHealth{}, fmt.Errorf("a node's health report is not readable: %w", err) + } + if h.V != NodeHealthVersion { + return NodeHealth{}, fmt.Errorf("a node's health report of version %d, and this reads %d", h.V, NodeHealthVersion) + } + return h, nil +} + +// SameNodeHealth says two reports say the same, whenever each was written. +func SameNodeHealth(a, b NodeHealth) bool { + a.At, b.At = time.Time{}, time.Time{} + ra, errA := json.Marshal(a) + rb, errB := json.Marshal(b) + return errA == nil && errB == nil && bytes.Equal(ra, rb) +} + +// HealthStore is where a node's report is kept: its own key of HealthBucket. +type HealthStore interface { + // Get is the key's value, nil when it has none. + Get(ctx context.Context, key string) ([]byte, error) + Put(ctx context.Context, key string, value []byte) error +} + +// healthStore binds HealthBucket over the link open now: bound, never created. +func (l *natsLink) healthStore() (HealthStore, error) { + kv, err := l.js.KeyValue(HealthBucket) + if err != nil { + return nil, fmt.Errorf("the node-engine's state %s cannot be bound (it is created from the node-engine's "+ + "manifest; written again once it is): %w", HealthBucket, err) + } + return natsHealth{kv}, nil +} + +type natsHealth struct{ kv nats.KeyValue } + +func (n natsHealth) Get(_ context.Context, key string) ([]byte, error) { + e, err := n.kv.Get(key) + if errors.Is(err, nats.ErrKeyNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return e.Value(), nil +} + +func (n natsHealth) Put(_ context.Context, key string, value []byte) error { + _, err := n.kv.Put(key, value) + return err +} + +// WriteNodeHealth writes this node's report to its key when it says something other than the key holds, and says +// whether it wrote. The key is read before the first write of a run, so a restart that finds the same report writes +// nothing; a store that cannot be bound or written is an error, and the next look tries again. +func (q *Session) WriteNodeHealth(ctx context.Context, h NodeHealth) (bool, error) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + l, ok := bus.(interface{ healthStore() (HealthStore, error) }) + if bus == nil || !ok { + return false, ErrNoLink + } + store, err := l.healthStore() + if err != nil { + return false, err + } + return writeNodeHealth(ctx, store, &q.healthWritten, h, time.Now) +} + +// writeNodeHealth is WriteNodeHealth over a store: last is the report last written or read, nil before the first. +func writeNodeHealth(ctx context.Context, store HealthStore, last **NodeHealth, h NodeHealth, now func() time.Time) (bool, error) { + if *last == nil { + raw, err := store.Get(ctx, h.Node) + if err != nil { + return false, fmt.Errorf("the health report of %s cannot be read: %w", h.Node, err) + } + if raw != nil { + if was, err := DecodeNodeHealth(raw); err == nil { + *last = &was + } + } + } + if *last != nil && SameNodeHealth(**last, h) { + return false, nil + } + h.At = now().UTC() + raw, err := EncodeNodeHealth(h) + if err != nil { + return false, err + } + if err := store.Put(ctx, h.Node, raw); err != nil { + return false, fmt.Errorf("the health report of %s cannot be written: %w", h.Node, err) + } + *last = &h + return true, nil +} diff --git a/internal/link/session.go b/internal/link/session.go index e33314d..5b2fb89 100644 --- a/internal/link/session.go +++ b/internal/link/session.go @@ -86,6 +86,9 @@ type Session struct { // haves is what this node-engine writes beside its facts (novox/hq ADR 0429 §4, issue 1132). haves haveState + // healthWritten is this node's health report as last written or read (WriteNodeHealth): the health loop alone + // writes it, one look at a time. + healthWritten *NodeHealth } func (q *Session) init() { diff --git a/internal/network/network.go b/internal/network/network.go index bd65d52..f1f511a 100644 --- a/internal/network/network.go +++ b/internal/network/network.go @@ -32,6 +32,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "sort" "strconv" "strings" @@ -94,6 +95,18 @@ type Finding struct { Writer string // Toward is what the failure points at: TowardHub, or each resolver's address that failed. Toward []string + // Peers is, for the tunnel, each peer of the mesh's interface and whether it was seen (Peer). + Peers []Peer +} + +// Peer is one peer of the mesh's tunnel as this machine sees it: its public key, the addresses it is allowed, and +// whether it handshook within StaleHandshake. What the watchman reads to tell a machine nobody reaches from one whose +// tunnel alone is down (network-unreachable). Seen, not the handshake's time: the time moves every two minutes, the +// fact only when a peer comes or goes. +type Peer struct { + Key string `json:"key"` + Addresses []string `json:"addresses,omitempty"` + Seen bool `json:"seen"` } // Part is one part's state, as the statement says it. @@ -115,6 +128,8 @@ type Statement struct { Since time.Time `json:"since"` At time.Time `json:"at"` Parts []Part `json:"parts"` + // Peers are the tunnel's peers on the last look that read them, sorted by key; nil where the tunnel is not read. + Peers []Peer `json:"peers,omitempty"` } // Machine is what a look reads, replaced in tests. @@ -266,6 +281,9 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) { changed = true } st.State, st.Since = worst, j.overall.since + // The tunnel's peers as this look read them; a peer seen or lost is a change of its own (network-unreachable). + st.Peers = findings[PartTunnel].Peers + changed = changed || !samePeers(j.said.Peers, st.Peers) j.said = st return st, changed } @@ -435,14 +453,22 @@ func (j *Judge) lookTunnel(ctx context.Context, now time.Time) Finding { } } hub := "" + allowed := map[string][]string{} for _, line := range strings.Split(ips, "\n") { f := strings.Fields(line) for _, prefix := range f[min(1, len(f)):] { + allowed[f[0]] = append(allowed[f[0]], prefix) if _, bits, ok := strings.Cut(prefix, "/"); ok && bits != "32" && bits != "128" { hub = f[0] } } } + peers := make([]Peer, 0, len(handshakes)) + for key, at := range handshakes { + peers = append(peers, Peer{Key: key, Addresses: allowed[key], Seen: at > 0 && now.Sub(time.Unix(at, 0)) <= StaleHandshake}) + } + sort.Slice(peers, func(i, k int) bool { return peers[i].Key < peers[k].Key }) + found := func(f Finding) Finding { f.Peers = peers; return f } age := func(at int64) string { if at == 0 { return "never" @@ -452,23 +478,36 @@ func (j *Judge) lookTunnel(ctx context.Context, now time.Time) Finding { if hub != "" { at := handshakes[hub] if at > 0 && now.Sub(time.Unix(at, 0)) <= StaleHandshake { - return Finding{OK: true} + return found(Finding{OK: true}) } - return Finding{Reason: "the tunnel to the hub has not handshaken for over five minutes", - Said: fmt.Sprintf("%s's newest handshake with the hub was %s", Interface, age(at)), Toward: []string{TowardHub}} + return found(Finding{Reason: "the tunnel to the hub has not handshaken for over five minutes", + Said: fmt.Sprintf("%s's newest handshake with the hub was %s", Interface, age(at)), Toward: []string{TowardHub}}) } if len(handshakes) == 0 { - return Finding{OK: true} + return found(Finding{OK: true}) } var newest int64 for _, at := range handshakes { newest = max(newest, at) } if newest > 0 && now.Sub(time.Unix(newest, 0)) <= StaleHandshake { - return Finding{OK: true} + return found(Finding{OK: true}) } - return Finding{Reason: "no machine has handshaken with the hub's tunnel for over five minutes", - Said: fmt.Sprintf("%s's newest handshake with any of its %d peers was %s", Interface, len(handshakes), age(newest))} + return found(Finding{Reason: "no machine has handshaken with the hub's tunnel for over five minutes", + Said: fmt.Sprintf("%s's newest handshake with any of its %d peers was %s", Interface, len(handshakes), age(newest))}) +} + +// samePeers says two looks saw the same peers, each seen or not alike. +func samePeers(a, b []Peer) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i].Key != b[i].Key || a[i].Seen != b[i].Seen || !slices.Equal(a[i].Addresses, b[i].Addresses) { + return false + } + } + return true } func (j *Judge) lookBus() Finding { diff --git a/internal/network/network_test.go b/internal/network/network_test.go index f27caa0..67155e7 100644 --- a/internal/network/network_test.go +++ b/internal/network/network_test.go @@ -302,9 +302,27 @@ func TestOnTheHubAnyFreshPeerIsAHealthyTunnel(t *testing.T) { f.hs = "A=\t" + itoa(f.now.Add(-time.Hour).Unix()) + "\nB=\t" + itoa(f.now.Unix()) + "\nC=\t0\n" f.ips = "A=\t10.10.0.2/32\nB=\t10.10.0.3/32\nC=\t10.10.0.4/32\n" j := f.judge(t) - if st := f.look(t, j); st.State != Healthy { + st := f.look(t, j) + if st.State != Healthy { t.Fatalf("the hub with one fresh peer is %+v", st.Parts) } + // Each peer and whether it was seen within five minutes, for the watchman's network-unreachable (issue 774); a + // peer seen again is a change, a handshake renewed alone is not. + want := []Peer{{Key: "A=", Addresses: []string{"10.10.0.2/32"}}, {Key: "B=", Addresses: []string{"10.10.0.3/32"}, Seen: true}, + {Key: "C=", Addresses: []string{"10.10.0.4/32"}}} + if !samePeers(st.Peers, want) { + t.Fatalf("the peers are %+v", st.Peers) + } + f.hs = "A=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nB=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nC=\t0\n" + f.now = f.now.Add(LookEvery) + if st, changed := j.Look(t.Context()); !changed || !st.Peers[0].Seen { + t.Fatalf("a peer seen again is no change: changed %v, %+v", changed, st.Peers) + } + f.hs = "A=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nB=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nC=\t0\n" + f.now = f.now.Add(LookEvery) + if _, changed := j.Look(t.Context()); changed { + t.Fatal("handshakes renewed alone are a change") + } } func TestAnUnreadableTunnelIsSaidAndAMissingToolSkipsIt(t *testing.T) { diff --git a/module.json b/module.json index d10f063..146d6a5 100644 --- a/module.json +++ b/module.json @@ -21,6 +21,9 @@ { "name": "acts", "ttl-seconds": 7776000 + }, + { + "name": "health" } ], "config": { diff --git a/packaging/operator_keys_test.go b/packaging/operator_keys_test.go index 6794b9e..ecf8ed1 100644 --- a/packaging/operator_keys_test.go +++ b/packaging/operator_keys_test.go @@ -74,7 +74,10 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) { if err != nil { t.Fatal(err) } - if len(parsed.State) != 1 || parsed.State[0].Name != "acts" || parsed.State[0].TTLSeconds != 90*24*60*60 { + // 774 (the alert table): beside it the state health, one key per node, the node's own health report, kept until + // written again (no TTL: a node that stops writing keeps its last word, which the watchman reads with its time). + if len(parsed.State) != 2 || parsed.State[0].Name != "acts" || parsed.State[0].TTLSeconds != 90*24*60*60 || + parsed.State[1].Name != "health" || parsed.State[1].TTLSeconds != 0 { t.Fatalf("the manifest declares the state %+v", parsed.State) } } -- 2.54.0 From 552f97c91914430d22604363970d69cb17cc3df4 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:59:22 +0100 Subject: [PATCH 16/21] Take the node's health report layout from mesh-sdk 94d0de5, so the engine and the watchman read one form (issue 774) delivery.NodeHealth and its parts replace this package's copies. The message's network and units parts are aliases of the same types, so its JSON is unchanged. The engine keeps the derivation from its statement and the write on a change. Test: the key's bytes, decoded and encoded again by the sdk, are the same bytes. --- go.mod | 2 +- go.sum | 4 +- internal/link/health_test.go | 8 ++- internal/link/messages.go | 58 ++++----------------- internal/link/nodehealth.go | 99 ++++++------------------------------ 5 files changed, 34 insertions(+), 137 deletions(-) diff --git a/go.mod b/go.mod index e49e5bd..a7c393c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215315-58a83249c80b + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215805-94d0de5c19b6 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index e33c19d..903dac8 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215315-58a83249c80b h1:1b3bqhFHWosEhxlsjMI6NEWFH+vdD3hqXVIysyLhPYU= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215315-58a83249c80b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215805-94d0de5c19b6 h1:eQIsSB3Z72MNjirW3TbKbvFgkhwjCAgOAqWd0U4ldCE= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215805-94d0de5c19b6/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/internal/link/health_test.go b/internal/link/health_test.go index 58282a2..88ace64 100644 --- a/internal/link/health_test.go +++ b/internal/link/health_test.go @@ -6,6 +6,8 @@ import ( "strings" "testing" "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" ) // A health statement is said on the link open now, as the machine's own word on its own subject, and @@ -95,10 +97,14 @@ func TestAHealthStatementIsSaidOnTheLinkOpenNow(t *testing.T) { t.Fatalf("step %d: wrote %v (want %v), %v", i, wrote, step.wrote, err) } } - got, err := DecodeNodeHealth(store.values["anchor"]) + got, err := delivery.DecodeNodeHealth(store.values["anchor"]) if err != nil || got.Node != "anchor" || got.Network.Parts[0].Writer != "openvpn" || !got.Peers[0].Seen || got.At.IsZero() { t.Fatalf("the key holds %+v, %v", got, err) } + // The key's bytes are mesh-sdk's layout: decoded and encoded again by it, the same bytes. + if again, err := delivery.EncodeNodeHealth(got); err != nil || string(again) != string(store.values["anchor"]) { + t.Fatalf("the key's bytes are not the sdk's layout: %s\n%s (%v)", store.values["anchor"], again, err) + } if store.puts != 3 { t.Fatalf("%d writes, want 3: one per change", store.puts) } diff --git a/internal/link/messages.go b/internal/link/messages.go index 2582562..23376d4 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -313,58 +313,18 @@ const KindRunner = "runner" // ReasonUsedAsFound starts the reason of a directory used as found. const ReasonUsedAsFound = "used as found:" -// UnitsHealth is the machine's service managers in one statement (issue 315). -type UnitsHealth struct { - // State is running, degraded — a unit failed, the modules' or the machine's — or unknown when no - // manager could be read. - State string `json:"state"` - // Failed is every unit failed on two looks in a row that no module places: the machine's own. - Failed []FailedUnit `json:"failed"` - // Unread names a manager that could not be read, with why. - Unread []string `json:"unread,omitempty"` - // Away names an account's own manager not running, so not read: nothing in it is failed (issue 764). - Away []string `json:"away,omitempty"` -} -// FailedUnit is one failed unit no module places. -type FailedUnit struct { - Unit string `json:"unit"` - // Scope is system, or user: an account's own manager. - Scope string `json:"scope"` - // Load is loaded, not-found — a unit whose file is gone and whose failure its manager still holds… - Load string `json:"load,omitempty"` - // Result is how it failed: exit-code, timeout, start-limit-hit… - Result string `json:"result,omitempty"` - // Resource is the mesh's own resource that names it, when the mesh placed it in its own right. - Resource string `json:"resource,omitempty"` - Since time.Time `json:"since"` -} -// NetworkHealth is the machine's networking in one statement (ADR 0241): the worst of its parts, since -// when, and each part. -type NetworkHealth struct { - State string `json:"state"` - Since time.Time `json:"since"` - Parts []NetworkPart `json:"parts"` -} -// NetworkPart is one part of a machine's networking, as its engine judged it on its second look. -type NetworkPart struct { - // Part is resolv-conf, names, tunnel, bus or route. - Part string `json:"part"` - State string `json:"state"` - // Reason is why it is not healthy, in words with no address, path or domain; Said the detail. - Reason string `json:"reason,omitempty"` - Said string `json:"said,omitempty"` - // Writer is the program that rewrote the resolver file, when it can be named; Owner the module whose - // file it is — the uplink's holder. - Writer string `json:"writer,omitempty"` - Owner string `json:"owner,omitempty"` - // Toward is what the failure points at: "hub", or each resolver's address that failed. - Toward []string `json:"toward,omitempty"` - Since time.Time `json:"since"` - Streak int `json:"streak,omitempty"` -} + +// The machine's networking and its service managers in a statement are the node's health report's parts (mesh-sdk +// delivery, issue 774): one layout for the message, the report and the watchman that reads it. +type ( + NetworkHealth = delivery.NetworkHealth + NetworkPart = delivery.NetworkPart + UnitsHealth = delivery.UnitsHealth + FailedUnit = delivery.FailedUnit +) // The states a long-running resource is said in (ADR 0240 §4). const ( diff --git a/internal/link/nodehealth.go b/internal/link/nodehealth.go index 4ec92c9..862a2a0 100644 --- a/internal/link/nodehealth.go +++ b/internal/link/nodehealth.go @@ -1,15 +1,14 @@ package link import ( - "bytes" "context" - "encoding/json" "errors" "fmt" "sort" "strings" "time" + "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/nats-io/nats.go" ) @@ -27,58 +26,19 @@ import ( // - relogin-needed: Relogin, each account whose running session lacks a group it was given; // - directory-used-as-found: Found, each directory a module declares that the engine uses as found. // -// The layout moves into mesh-sdk before the push (the lead, 2026-10-29); until then it is this package's. -// HealthBucket is the node-engine's declared state health: key ``, value NodeHealth. -const HealthBucket = "mesh-host_health" +// The report's layout is mesh-sdk's (delivery.NodeHealth): the bucket, its version and its parts. +const ( + HealthBucket = delivery.NodeHealthBucket + NodeHealthVersion = delivery.NodeHealthVersion +) -// NodeHealthVersion is the version of NodeHealth's layout. -const NodeHealthVersion = 1 - -// NodeHealth is one node's health report, as its engine last found it. -type NodeHealth struct { - V int `json:"v"` - Node string `json:"node"` - // Network is the machine's networking, its parts as Health.Network says them, without the streaks; nil where this - // engine does not judge it yet. - Network *NetworkHealth `json:"network,omitempty"` - // Peers are the mesh tunnel's peers on the last look that read them, sorted by key. - Peers []PeerSeen `json:"peers,omitempty"` - // Units is the machine's service managers: each failed unit no module places (Health.Units). - Units *UnitsHealth `json:"units,omitempty"` - // Relogin are the accounts whose running session needs a new login for a group it was given. - Relogin []NodeAccount `json:"relogin,omitempty"` - // Found are the directories used as found. - Found []NodeDirectory `json:"found,omitempty"` - // At is when the engine wrote this, on a change: never a reason to write. - At time.Time `json:"at"` -} - -// PeerSeen is one peer of the mesh's tunnel: its public key, its allowed addresses, and whether this machine saw a -// handshake with it within five minutes. -type PeerSeen struct { - Key string `json:"key"` - Addresses []string `json:"addresses,omitempty"` - Seen bool `json:"seen"` -} - -// NodeAccount is one account marked relogin needed, with the module whose resource it is and why. -type NodeAccount struct { - Account string `json:"account"` - Module string `json:"module,omitempty"` - Resource string `json:"resource,omitempty"` - Reason string `json:"reason"` - Since time.Time `json:"since"` -} - -// NodeDirectory is one directory used as found: the module declaring it, its resource and path, and why. -type NodeDirectory struct { - Module string `json:"module"` - Resource string `json:"resource"` - Path string `json:"path"` - Reason string `json:"reason"` - Since time.Time `json:"since"` -} +type ( + NodeHealth = delivery.NodeHealth + PeerSeen = delivery.PeerSeen + NodeAccount = delivery.NodeAccount + NodeDirectory = delivery.NodeDirectory +) // ReasonReloginNeeded starts an account's reason when its session needs a new login (accounts.ReasonRelogin). const ReasonReloginNeeded = "relogin needed" @@ -116,35 +76,6 @@ func NodeHealthOf(node string, h Health, peers []PeerSeen) NodeHealth { return out } -// EncodeNodeHealth is h as JSON at its version. -func EncodeNodeHealth(h NodeHealth) ([]byte, error) { - h.V = NodeHealthVersion - if h.Node == "" { - return nil, errors.New("a node's health report names its node") - } - return json.Marshal(h) -} - -// DecodeNodeHealth reads a report. -func DecodeNodeHealth(raw []byte) (NodeHealth, error) { - var h NodeHealth - if err := json.Unmarshal(raw, &h); err != nil { - return NodeHealth{}, fmt.Errorf("a node's health report is not readable: %w", err) - } - if h.V != NodeHealthVersion { - return NodeHealth{}, fmt.Errorf("a node's health report of version %d, and this reads %d", h.V, NodeHealthVersion) - } - return h, nil -} - -// SameNodeHealth says two reports say the same, whenever each was written. -func SameNodeHealth(a, b NodeHealth) bool { - a.At, b.At = time.Time{}, time.Time{} - ra, errA := json.Marshal(a) - rb, errB := json.Marshal(b) - return errA == nil && errB == nil && bytes.Equal(ra, rb) -} - // HealthStore is where a node's report is kept: its own key of HealthBucket. type HealthStore interface { // Get is the key's value, nil when it has none. @@ -207,16 +138,16 @@ func writeNodeHealth(ctx context.Context, store HealthStore, last **NodeHealth, return false, fmt.Errorf("the health report of %s cannot be read: %w", h.Node, err) } if raw != nil { - if was, err := DecodeNodeHealth(raw); err == nil { + if was, err := delivery.DecodeNodeHealth(raw); err == nil { *last = &was } } } - if *last != nil && SameNodeHealth(**last, h) { + if *last != nil && delivery.SameNodeHealth(**last, h) { return false, nil } h.At = now().UTC() - raw, err := EncodeNodeHealth(h) + raw, err := delivery.EncodeNodeHealth(h) if err != nil { return false, err } -- 2.54.0 From 3ed9abf658e08ef6024713d6a655e8b44d677e7d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 22:59:38 +0100 Subject: [PATCH 17/21] Format the health report's aliases as gofmt does (issue 774) --- internal/link/messages.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/link/messages.go b/internal/link/messages.go index 23376d4..7a64191 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -313,10 +313,6 @@ const KindRunner = "runner" // ReasonUsedAsFound starts the reason of a directory used as found. const ReasonUsedAsFound = "used as found:" - - - - // The machine's networking and its service managers in a statement are the node's health report's parts (mesh-sdk // delivery, issue 774): one layout for the message, the report and the watchman that reads it. type ( -- 2.54.0 From c3ff18de4e84f23e3e0e97f6b0ffc03e55117b38 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:05:02 +0100 Subject: [PATCH 18/21] Name the machines a failing network part points at in the node's health report, so the watchman holds it where the controller does (issue 774) delivery.NetworkPart.Targets (mesh-sdk 6eae076) is filled where the report is made, by the controller's own rule: - the hub's node for "hub"; - each resolver's node by its address on the private network; - none when the part points at nothing, at this machine, or at an address no other mesh machine has. The names come from the placements the node's own composer holds (Decided.Address, Decided.Hub). A spoke's tunnel names only the hub, so the tunnel cannot name a resolver on another spoke. A node that does not compose its own names no target. Test: rows in TestAHealthStatementIsSaidOnTheLinkOpenNow (hub, two resolvers, a VPN's resolver, its own, a healthy part). --- cmd/mesh-host/main.go | 6 +++- cmd/mesh-host/nodeagent.go | 5 ++++ go.mod | 2 +- go.sum | 4 +-- internal/link/health_test.go | 31 +++++++++++++++++---- internal/link/nodehealth.go | 43 +++++++++++++++++++++++++++-- internal/selfcompose/selfcompose.go | 18 ++++++++++++ 7 files changed, 98 insertions(+), 11 deletions(-) diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 0be295e..505bc93 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1488,7 +1488,11 @@ func writeHealthReport(ctx context.Context, queue *link.Session, h link.Health, peers = append(peers, link.PeerSeen{Key: p.Key, Addresses: p.Addresses, Seen: p.Seen}) } } - _, err := queue.WriteNodeHealth(ctx, link.NodeHealthOf(queue.Membership.Node, h, peers)) + var machines link.Machines + if c := machinesNow.Load(); c != nil { + machines.ByAddress, machines.Hub = c.Machines() + } + _, err := queue.WriteNodeHealth(ctx, link.NodeHealthOf(queue.Membership.Node, h, peers, machines)) switch { case err != nil && err.Error() != *unwritten: *unwritten = err.Error() diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 00531d8..a67f6a0 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -626,6 +626,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus continue } if composer != nil { + machinesNow.Store(composer) // the machines a failing network part points at, by the placements (issue 774) composes.set(delivery.ComposesOwn) env.Composed, env.SelfComposed, env.Trusted, env.Policy = composer, true, nil, composer.Policy env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does @@ -768,6 +769,10 @@ func busyModules() []string { return out } +// machinesNow is the composer whose placements name the mesh's machines by address, for the health report's targets +// (writeHealthReport); nil on a node that does not compose its own, whose parts then name no target. +var machinesNow atomic.Pointer[selfcompose.Composer] + // keptRetired says the kept declaration is read no more: every module the seat's placement puts on this node has a have of // its own (placedAllHave), so what runs is the agent's alone (ADR 0550 §2, design 47's L3 s9). var keptRetired atomic.Bool diff --git a/go.mod b/go.mod index a7c393c..5e975e7 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215805-94d0de5c19b6 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220254-6eae0765f4d3 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index 903dac8..5d9076b 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215805-94d0de5c19b6 h1:eQIsSB3Z72MNjirW3TbKbvFgkhwjCAgOAqWd0U4ldCE= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029215805-94d0de5c19b6/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220254-6eae0765f4d3 h1:4qfwx2bv2vosFM6tEbBZUeaQ/pfRauXZuNI3whul30g= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220254-6eae0765f4d3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/internal/link/health_test.go b/internal/link/health_test.go index 88ace64..7748566 100644 --- a/internal/link/health_test.go +++ b/internal/link/health_test.go @@ -3,6 +3,7 @@ package link import ( "context" "encoding/json" + "slices" "strings" "testing" "time" @@ -69,12 +70,32 @@ func TestAHealthStatementIsSaidOnTheLinkOpenNow(t *testing.T) { {Module: "letta", Resource: "letta.server", Kind: "container", State: StateHealthy, Since: since}}} } peers := []PeerSeen{{Key: "B=", Seen: true}, {Key: "A=", Addresses: []string{"10.10.0.2/32"}}} - r := NodeHealthOf("anchor", look(1, "NetworkManager"), peers) + r := NodeHealthOf("anchor", look(1, "NetworkManager"), peers, Machines{}) if r.Network.Parts[0].Writer != "NetworkManager" || r.Network.Parts[0].Streak != 0 || r.Peers[0].Key != "A=" || len(r.Units.Failed) != 1 || len(r.Relogin) != 1 || r.Relogin[0].Account != "jochen" || len(r.Found) != 1 || r.Found[0].Path != "/srv/photos" { t.Fatalf("the report is %+v", r) } + // The machines an unhealthy part points at, as the controller holds it there; none for the machine's own. + machines := Machines{ByAddress: map[string]string{"10.10.0.1": "novox", "10.10.0.4": "ace", "10.10.0.9": "anchor"}, Hub: "novox"} + for _, tc := range []struct { + name string + toward []string + state string + want []string + }{ + {"the tunnel to the hub", []string{"hub"}, StateUnhealthy, []string{"novox"}}, + {"two mesh resolvers", []string{"10.10.0.1", "10.10.0.4"}, StateUnhealthy, []string{"novox", "ace"}}, + {"a resolver a VPN client wrote in is the machine's own", []string{"10.10.0.1", "192.0.2.53"}, StateUnhealthy, nil}, + {"its own resolver is its own", []string{"10.10.0.9"}, StateUnhealthy, nil}, + {"a healthy part points at nothing", []string{"hub"}, StateHealthy, nil}, + } { + h := Health{Network: &NetworkHealth{State: tc.state, Parts: []NetworkPart{{Part: "names", State: tc.state, Toward: tc.toward}}}} + got := NodeHealthOf("anchor", h, nil, machines).Network.Parts[0].Targets + if !slices.Equal(got, tc.want) { + t.Errorf("%s: targets %v, want %v", tc.name, got, tc.want) + } + } store := &memHealth{} var last *NodeHealth clock := func() time.Time { return since } @@ -84,10 +105,10 @@ func TestAHealthStatementIsSaidOnTheLinkOpenNow(t *testing.T) { wrote bool }{ {h: r, wrote: true}, - {h: NodeHealthOf("anchor", look(2, "NetworkManager"), peers)}, - {h: NodeHealthOf("anchor", look(3, "NetworkManager"), peers), fresh: true}, - {h: NodeHealthOf("anchor", look(3, "openvpn"), peers), wrote: true}, - {h: NodeHealthOf("anchor", look(3, "openvpn"), []PeerSeen{{Key: "A=", Seen: true}}), wrote: true}, + {h: NodeHealthOf("anchor", look(2, "NetworkManager"), peers, Machines{})}, + {h: NodeHealthOf("anchor", look(3, "NetworkManager"), peers, Machines{}), fresh: true}, + {h: NodeHealthOf("anchor", look(3, "openvpn"), peers, Machines{}), wrote: true}, + {h: NodeHealthOf("anchor", look(3, "openvpn"), []PeerSeen{{Key: "A=", Seen: true}}, Machines{}), wrote: true}, } { if step.fresh { last = nil diff --git a/internal/link/nodehealth.go b/internal/link/nodehealth.go index 862a2a0..48b7234 100644 --- a/internal/link/nodehealth.go +++ b/internal/link/nodehealth.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "slices" "sort" "strings" "time" @@ -43,9 +44,43 @@ type ( // ReasonReloginNeeded starts an account's reason when its session needs a new login (accounts.ReasonRelogin). const ReasonReloginNeeded = "relogin needed" +// Machines names the mesh's machines for a report's targets: each node by its address on the private network, and the +// hub's node. The zero value names none, and every part is then the machine's own. +type Machines struct { + ByAddress map[string]string + Hub string +} + +// targets is the machines a part points at, as the controller holds it (mesh-controller machine_network.go): the hub's +// node for TowardHub, each resolver's node by its address; none when it points at nothing, or at this machine or an +// address no other mesh machine has, as one of them makes the part the machine's own. +func (m Machines) targets(node string, toward []string) []string { + if len(toward) == 0 { + return nil + } + var out []string + for _, t := range toward { + x := m.ByAddress[t] + if t == TowardHub { + x = m.Hub + } + if x == "" || x == node { + return nil + } + if !slices.Contains(out, x) { + out = append(out, x) + } + } + return out +} + +// TowardHub is what a part failing toward the hub names in Toward (network.TowardHub). +const TowardHub = "hub" + // NodeHealthOf is the report a health statement and the tunnel's peers make: the statement's network without its -// streaks, its machine units, and each account and directory of its resources that the six alerts read. -func NodeHealthOf(node string, h Health, peers []PeerSeen) NodeHealth { +// streaks, each unhealthy part naming the machines it points at (machines), its machine units, and each account and +// directory of its resources that the six alerts read. +func NodeHealthOf(node string, h Health, peers []PeerSeen, machines Machines) NodeHealth { out := NodeHealth{V: NodeHealthVersion, Node: node, Units: h.Units} if len(peers) > 0 { out.Peers = append([]PeerSeen(nil), peers...) @@ -55,6 +90,10 @@ func NodeHealthOf(node string, h Health, peers []PeerSeen) NodeHealth { n := NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: make([]NetworkPart, 0, len(h.Network.Parts))} for _, p := range h.Network.Parts { p.Streak = 0 // a look's count moves every look; the report says a part's state, not how often it was seen + p.Targets = nil + if p.State == StateUnhealthy { + p.Targets = machines.targets(node, p.Toward) + } n.Parts = append(n.Parts, p) } out.Network = &n diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index a49734c..44ca041 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -423,6 +423,24 @@ func (c *Composer) takeModuleHave(u agent.Update) bool { } // inOwnOf is, of the modules p places, those the node runs inside its own composition (c.inOwn), sorted; c.mu held. +// Machines is, from the placements this composer holds, each node by its address on the private network, and the hub's +// node: what names the machine a failing network part points at (delivery.NetworkPart.Targets, issue 774). Empty +// before any placement is read. +func (c *Composer) Machines() (byAddress map[string]string, hub string) { + c.mu.Lock() + defer c.mu.Unlock() + byAddress = map[string]string{} + for node, p := range c.placements { + if a, _, _ := strings.Cut(p.Decided.Address, "/"); a != "" { + byAddress[a] = node + } + if p.Decided.Hub { + hub = node + } + } + return byAddress, hub +} + func (c *Composer) inOwnOf(p delivery.Placement) []string { var out []string for _, m := range p.Modules { -- 2.54.0 From e5e8ff57833f1f8a4a81a46acbf06fdd2d58b4ca Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 29 Oct 2026 23:49:36 +0100 Subject: [PATCH 19/21] Say each account's root verdict and the engine's contract in the node's health report, so the self-check's DA and D-root judge without the controller (issue 1952, ADR 0266) Every account resource marked root never goes into NodeHealth.Root; the statement's contract into Contract. Relogin, root and found are judged each on its own, so an account may be both. Pins mesh-sdk f3677eb. Test: the report's row in TestAHealthStatementIsSaidOnTheLinkOpenNow carries the agent account's verdict and the contract. --- go.mod | 2 +- go.sum | 4 ++-- internal/link/health_test.go | 9 ++++++--- internal/link/nodehealth.go | 18 ++++++++++++++---- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 5e975e7..e64967e 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220254-6eae0765f4d3 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029224808-f3677eb6f36e github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index 5d9076b..139d8ea 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220254-6eae0765f4d3 h1:4qfwx2bv2vosFM6tEbBZUeaQ/pfRauXZuNI3whul30g= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029220254-6eae0765f4d3/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029224808-f3677eb6f36e h1:h8bh7ZyAgPuFC9/lc6iFBD/hduzomLodZ0a1oJZ5X4U= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029224808-f3677eb6f36e/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/internal/link/health_test.go b/internal/link/health_test.go index 7748566..449f0cb 100644 --- a/internal/link/health_test.go +++ b/internal/link/health_test.go @@ -59,7 +59,7 @@ func TestAHealthStatementIsSaidOnTheLinkOpenNow(t *testing.T) { t.Run("the node's health report is written on a change only", func(t *testing.T) { since := time.Date(2026, 10, 29, 20, 0, 0, 0, time.UTC) look := func(streak int, writer string) Health { - return Health{At: time.Now(), Network: &NetworkHealth{State: StateUnhealthy, Since: since, Parts: []NetworkPart{ + return Health{Contract: 7, At: time.Now(), Network: &NetworkHealth{State: StateUnhealthy, Since: since, Parts: []NetworkPart{ {Part: "resolv-conf", State: StateUnhealthy, Writer: writer, Owner: "dnsmasq", Since: since, Streak: streak}}}, Units: &UnitsHealth{State: "degraded", Failed: []FailedUnit{{Unit: "cups.service", Scope: "system", Since: since}}}, Resources: []ResourceHealth{ @@ -67,13 +67,16 @@ func TestAHealthStatementIsSaidOnTheLinkOpenNow(t *testing.T) { Reason: "relogin needed: the session began before scanner", Since: since}, {Module: "photos", Resource: "photos.library", Kind: KindDirectory, Target: "/srv/photos", State: StateUnhealthy, Reason: ReasonUsedAsFound + " owned by root", Since: since}, - {Module: "letta", Resource: "letta.server", Kind: "container", State: StateHealthy, Since: since}}} + {Module: "letta", Resource: "letta.server", Kind: "container", State: StateHealthy, Since: since}, + {Module: "claude-code", Resource: "claude-code.agent", Kind: KindAccount, Account: "agent", Root: "never", + State: StateHealthy, Since: since}}} } peers := []PeerSeen{{Key: "B=", Seen: true}, {Key: "A=", Addresses: []string{"10.10.0.2/32"}}} r := NodeHealthOf("anchor", look(1, "NetworkManager"), peers, Machines{}) if r.Network.Parts[0].Writer != "NetworkManager" || r.Network.Parts[0].Streak != 0 || r.Peers[0].Key != "A=" || len(r.Units.Failed) != 1 || len(r.Relogin) != 1 || r.Relogin[0].Account != "jochen" || - len(r.Found) != 1 || r.Found[0].Path != "/srv/photos" { + len(r.Found) != 1 || r.Found[0].Path != "/srv/photos" || + len(r.Root) != 1 || r.Root[0].Account != "agent" || r.Root[0].State != StateHealthy || r.Contract != 7 { t.Fatalf("the report is %+v", r) } // The machines an unhealthy part points at, as the controller holds it there; none for the machine's own. diff --git a/internal/link/nodehealth.go b/internal/link/nodehealth.go index 48b7234..b938dbb 100644 --- a/internal/link/nodehealth.go +++ b/internal/link/nodehealth.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-sdk/go/declaration" "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/nats-io/nats.go" ) @@ -81,7 +82,7 @@ const TowardHub = "hub" // streaks, each unhealthy part naming the machines it points at (machines), its machine units, and each account and // directory of its resources that the six alerts read. func NodeHealthOf(node string, h Health, peers []PeerSeen, machines Machines) NodeHealth { - out := NodeHealth{V: NodeHealthVersion, Node: node, Units: h.Units} + out := NodeHealth{V: NodeHealthVersion, Node: node, Units: h.Units, Contract: h.Contract} if len(peers) > 0 { out.Peers = append([]PeerSeen(nil), peers...) sort.Slice(out.Peers, func(i, k int) bool { return out.Peers[i].Key < out.Peers[k].Key }) @@ -99,11 +100,17 @@ func NodeHealthOf(node string, h Health, peers []PeerSeen, machines Machines) No out.Network = &n } for _, r := range h.Resources { - switch { - case r.Kind == KindAccount && r.State == StateUnhealthy && strings.HasPrefix(r.Reason, ReasonReloginNeeded): + // Each alert reads its own: an account may need a new login and be declared never to become root at once. + if r.Kind == KindAccount && r.State == StateUnhealthy && strings.HasPrefix(r.Reason, ReasonReloginNeeded) { out.Relogin = append(out.Relogin, NodeAccount{Account: r.Account, Module: r.Module, Resource: r.Resource, Reason: r.Reason, Since: r.Since}) - case r.Kind == KindDirectory && strings.HasPrefix(r.Reason, ReasonUsedAsFound): + } + if r.Kind == KindAccount && r.Root == declaration.RootNever { + // The account declared never to become root without a person, judged (ADR 0266): what DA and D-root read. + out.Root = append(out.Root, delivery.RootVerdict{Account: r.Account, Module: r.Module, Resource: r.Resource, + State: r.State, Reason: r.Reason, Since: r.Since}) + } + if r.Kind == KindDirectory && strings.HasPrefix(r.Reason, ReasonUsedAsFound) { out.Found = append(out.Found, NodeDirectory{Module: r.Module, Resource: r.Resource, Path: r.Target, Reason: r.Reason, Since: r.Since}) } @@ -112,6 +119,9 @@ func NodeHealthOf(node string, h Health, peers []PeerSeen, machines Machines) No return out.Relogin[i].Account+out.Relogin[i].Module < out.Relogin[k].Account+out.Relogin[k].Module }) sort.Slice(out.Found, func(i, k int) bool { return out.Found[i].Path < out.Found[k].Path }) + sort.Slice(out.Root, func(i, k int) bool { + return out.Root[i].Module+out.Root[i].Resource < out.Root[k].Module+out.Root[k].Resource + }) return out } -- 2.54.0 From 192633edb8e0087ec814feb2dfe44ef2a51f9d66 Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 30 Oct 2026 00:07:07 +0100 Subject: [PATCH 20/21] Pin mesh-sdk 007b6fd, the release's final sdk (issue 774) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e64967e..e1a2421 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029224808-f3677eb6f36e + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index 139d8ea..22bcaae 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029224808-f3677eb6f36e h1:h8bh7ZyAgPuFC9/lc6iFBD/hduzomLodZ0a1oJZ5X4U= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029224808-f3677eb6f36e/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 h1:9FZLjq5H+FShqsaSvZ5vGGjDFaQkdhGCna6ofiBd+p4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= -- 2.54.0 From 02f78c99497c54c008d39dc600e8d1dea3351eed Mon Sep 17 00:00:00 2001 From: jochen Date: Fri, 30 Oct 2026 00:17:00 +0100 Subject: [PATCH 21/21] Pin mesh-sdk 6e4a301, the release's final sdk (issue 774) The whole integration suite passes at this head against a local bus (JetStream, and a bus with the witness grants) and the release's mesh-catalog and mesh-controller beside it: 1260 tests, 7 skipped for a container or a throwaway machine. --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e1a2421..99b7f01 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index 22bcaae..eb0efcb 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7 h1:9FZLjq5H+FShqsaSvZ5vGGjDFaQkdhGCna6ofiBd+p4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029230207-007b6fd45fc7/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde h1:ddf/jKI9c87sQKDmZLoG6PMnvXwsNH16ByVveyuFME4= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029231453-6e4a3014cbde/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= -- 2.54.0