diff --git a/cmd/mesh-host/applied_store.go b/cmd/mesh-host/applied_store.go new file mode 100644 index 0000000..75243ed --- /dev/null +++ b/cmd/mesh-host/applied_store.go @@ -0,0 +1,107 @@ +package main + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" +) + +// appliedStore keeps every composition the agent applied, one file per module (the node's own as `node.json`), in the +// directory `applied` beside the node's state (agent.Env.Applied, issue 774). It is the machine's own record of what runs: +// read at the agent's start before anything is watched, so the node never composes its own as if a module it runs ran +// nothing because the bucket no longer keeps that composition or the kept declaration is retired. Files are 0600 in a +// 0700 directory: a composition carries values sealed to this machine. +type appliedStore struct{ dir string } + +func appliedStoreFor(statePath string) appliedStore { + return appliedStore{dir: filepath.Join(filepath.Dir(statePath), "applied")} +} + +// appliedName is a module name the store takes as a file name: a catalogue module's or the node's own, never a path. +var appliedName = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]*$`) + +func (s appliedStore) path(module string) (string, error) { + if !appliedName.MatchString(module) || strings.Contains(module, "..") { + return "", fmt.Errorf("%q is not a module name the applied store takes", module) + } + return filepath.Join(s.dir, module+".json"), nil +} + +// holds says whether the store keeps the module's composition. +func (s appliedStore) holds(module string) bool { + path, err := s.path(module) + if err != nil { + return false + } + _, err = os.Stat(path) + return err == nil +} + +func (s appliedStore) Load() (map[string][]byte, error) { + entries, err := os.ReadDir(s.dir) + if errors.Is(err, os.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + out := map[string][]byte{} + var errs []error + for _, e := range entries { + module, ok := strings.CutSuffix(e.Name(), ".json") + if !ok || e.IsDir() || !appliedName.MatchString(module) { + continue + } + raw, err := os.ReadFile(filepath.Join(s.dir, e.Name())) + if err != nil { + errs = append(errs, err) + continue + } + out[module] = raw + } + return out, errors.Join(errs...) +} + +func (s appliedStore) Keep(module string, raw []byte) error { + path, err := s.path(module) + if err != nil { + return err + } + if was, err := os.ReadFile(path); err == nil && string(was) == string(raw) { + return nil + } + if err := os.MkdirAll(s.dir, 0o700); err != nil { + return err + } + tmp, err := os.CreateTemp(s.dir, "."+module+".*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err := tmp.Write(raw); err != nil { + tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), path) +} + +func (s appliedStore) Drop(module string) error { + path, err := s.path(module) + if err != nil { + return err + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil +} diff --git a/cmd/mesh-host/authority_test.go b/cmd/mesh-host/authority_test.go index 48ce708..ac7bdcb 100644 --- a/cmd/mesh-host/authority_test.go +++ b/cmd/mesh-host/authority_test.go @@ -1,16 +1,27 @@ package main import ( + "context" + "crypto/ed25519" + "crypto/x509" + "encoding/base64" "encoding/json" + "encoding/pem" "os" "path/filepath" "testing" + "time" + "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" + + "github.com/novox/mesh-host/internal/identity" ) // Issue 1126 (ADR 0555 §3): the mesh's own certificate a composition carries is kept for the machine's record, and the -// record's keys read it; a composition carrying none changes nothing. +// record's keys read it; a composition carrying none changes nothing. And (L3 s6, s7; ADR 0550 §2) the record says what +// the node trusts a mesh signature from, and the node keeps the certificate the vault signed for it. func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) { state := filepath.Join(t.TempDir(), "state", "state.json") res := func(id, content string) json.RawMessage { @@ -26,7 +37,137 @@ func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) { if err := keepAuthority(state, delivery.Composed{Resources: []json.RawMessage{res("ca-trust.certificate-authority", "CA-1")}}); err != nil { t.Fatal(err) } - if got := recordKeys(state)(); got.Authority != "CA-1" { + if got := recordKeys(state, nil)(); got.Authority != "CA-1" { t.Errorf("the record's keys read the authority %q", got.Authority) } + + // L3 s6: before the vault's signers are read the node trusts the controller's key alone; once read, a message the + // vault signed is accepted and the record names both; once the lease is handed away the controller's key is refused + // and the record names the vault's alone. Each change tells the record's writer. + controllerPub, controller, _ := ed25519.GenerateKey(nil) + vaultPub, vault, _ := ed25519.GenerateKey(nil) + trust := newMeshTrust(controllerPub) + told := 0 + trust.onChange(func() { told++ }) + keys := recordKeys(state, trust.Trusts) + ctrlFP, vaultFP := delivery.SignerFingerprint(controllerPub), delivery.SignerFingerprint(vaultPub) + if got := keys().Trusts; !delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || delivery.TrustsKey(got, delivery.SignerVault, vaultFP) { + t.Errorf("before the signers are read the record trusts %v", got) + } + enrol, _ := delivery.SignContext(delivery.KindEnrolment) + msg := []byte("a reply") + if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(vault, enrol, msg)); err == nil { + t.Error("a vault-signed message is accepted before the vault's key is read") + } + vaultCA := testAuthority(t, vaultPub, vault) + keyValue, _ := json.Marshal(delivery.NewSignerKey(vaultPub, time.Now())) + fp, _ := delivery.AuthorityFingerprint(vaultCA) + authValue, _ := json.Marshal(delivery.SignerAuthority{Certificate: vaultCA, Fingerprint: fp, Made: time.Now()}) + trust.heardSigner(delivery.SignersKeyKey, keyValue, false, func(string) {}) + trust.heardSigner(delivery.SignersAuthorityKey, authValue, false, func(string) {}) + if s, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(vault, enrol, msg)); err != nil || s.Name != delivery.SignerVault { + t.Errorf("a vault-signed message on a node that read signers: %+v, %v", s, err) + } + if got := keys().Trusts; !delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) || + trust.VaultAuthority() != vaultCA || told != 2 { + t.Errorf("after the signers are read the record trusts %v (told %d)", got, told) + } + handed, _ := json.Marshal(buscompose.LeaseHolder{HandedTo: "mesh-vault", HandedBy: "person.operator"}) + trust.heardLease(handed, true, nil) + if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(controller, enrol, msg)); err == nil || trust.Controller() != nil { + t.Error("after the lease is handed away the controller's key is still trusted") + } + if got := keys().Trusts; delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) || told != 3 { + t.Errorf("after the lease is handed away the record trusts %v (told %d)", got, told) + } + trust.heardLease(handed, true, nil) + if told != 3 { + t.Error("a lease heard again unchanged wrote the record again") + } + // A signers' key naming another fingerprint is not trusted. + bad, _ := json.Marshal(delivery.SignerKey{Key: base64.StdEncoding.EncodeToString(controllerPub), Fingerprint: vaultFP}) + trust.heardSigner(delivery.SignersKeyKey, bad, false, func(string) {}) + if !delivery.TrustsKey(keys().Trusts, delivery.SignerVault, vaultFP) { + t.Error("a signers' key that does not check replaced the vault's") + } + + // L3 s7: the certificate the vault signs for this node's internal name and serving key is kept where the record reads + // it, and the record is written again; one signed by another authority is refused and nothing is kept. + serving, err := identity.GenerateServingKey() + if err != nil { + t.Fatal(err) + } + if err := identity.WriteServingKey(identity.ServingKeyPath(state), serving); err != nil { + t.Fatal(err) + } + servingPub, _ := base64.StdEncoding.DecodeString(serving.Public) + signedBy := func(ca string, key ed25519.PrivateKey) string { + parent, _ := delivery.ParseCertificate(ca) + der, err := x509.CreateCertificate(nil, delivery.NodeCertificateTemplate(overlay.InternalName("n1"), servingPub, time.Now().Add(-time.Hour)), + parent, ed25519.PublicKey(servingPub), key) + if err != nil { + t.Fatal(err) + } + return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) + } + var asked []any + answer := signedBy(vaultCA, vault) + ask := func(_ context.Context, verb string, args any) (json.RawMessage, error) { + asked = append(asked, verb, args) + return json.Marshal(delivery.SignAnswer{Kind: delivery.KindNodeCertificate, Node: "n1", Certificate: answer}) + } + written := 0 + if err := certifyThisNode(context.Background(), state, "n1", trust, ask, func() { written++ }); err != nil { + t.Fatal(err) + } + if got := recordKeys(state, nil)(); got.Certificate != answer || written != 1 || len(asked) != 2 || asked[0] != delivery.SignVerb { + t.Errorf("the vault's certificate: kept %v, written %d, asked %v", got.Certificate == answer, written, asked) + } + answer = signedBy(testAuthority(t, controllerPub, controller), controller) + if err := certifyThisNode(context.Background(), state, "n1", trust, ask, func() { written++ }); err == nil || written != 1 { + t.Errorf("a certificate of another authority was kept: %v", err) + } + + // Issue 774: what the engine applied is kept beside the state too, one file per module, 0600 in a 0700 directory, + // written by a rename so a crash leaves the one before; a name that is no module's is refused, never a path. + t.Run("the compositions the engine applied are kept beside the state", func(t *testing.T) { + applied := appliedStoreFor(filepath.Join(t.TempDir(), "state", "state.json")) + if m, err := applied.Load(); err != nil || len(m) != 0 || applied.holds("node") { + t.Fatalf("an empty store loads %v, %v", m, err) + } + for _, tc := range []struct { + module string + takes bool + }{{"node", true}, {"nats", true}, {"mesh-delivery-view", true}, {"../state", false}, {"a/b", false}, {"..", false}, {"", false}} { + if err := applied.Keep(tc.module, []byte(`{"v":1}`)); (err == nil) != tc.takes { + t.Errorf("keeping %q: %v, want taken %v", tc.module, err, tc.takes) + } + } + if err := applied.Keep("node", []byte(`{"v":2}`)); err != nil { + t.Fatal(err) + } + entries, _ := os.ReadDir(applied.dir) + dirInfo, _ := os.Stat(applied.dir) + fileInfo, _ := os.Stat(filepath.Join(applied.dir, "node.json")) + if len(entries) != 3 || dirInfo.Mode().Perm() != 0o700 || fileInfo.Mode().Perm() != 0o600 { + t.Fatalf("the store holds %d files (want 3, no temporary one left), directory %v, file %v", len(entries), dirInfo.Mode(), fileInfo.Mode()) + } + if err := applied.Drop("nats"); err != nil || applied.Drop("nats") != nil || applied.holds("nats") { + t.Fatalf("a dropped module is still held, or dropping twice fails: %v", err) + } + if m, err := applied.Load(); err != nil || string(m["node"]) != `{"v":2}` || len(m) != 2 { + t.Fatalf("the store loads %v, %v", m, err) + } + }) +} + +// testAuthority is a self-signed authority certificate for key, as the vault makes its own. +func testAuthority(t *testing.T, pub ed25519.PublicKey, key ed25519.PrivateKey) string { + t.Helper() + tpl := delivery.AuthorityTemplate(pub, time.Now().Add(-time.Hour)) + der, err := x509.CreateCertificate(nil, tpl, tpl, pub, key) + if err != nil { + t.Fatal(err) + } + return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})) } diff --git a/cmd/mesh-host/engineseat.go b/cmd/mesh-host/engineseat.go new file mode 100644 index 0000000..4c11ece --- /dev/null +++ b/cmd/mesh-host/engineseat.go @@ -0,0 +1,277 @@ +package main + +import ( + "bufio" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync/atomic" + "syscall" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + meshasker "git.novox.be/novox/mesh-sdk/go/asker" + "git.novox.be/novox/mesh-sdk/go/asks" + "github.com/novox/mesh-host/internal/accounts" + "github.com/novox/mesh-host/internal/apply" + "github.com/novox/mesh-host/internal/link" + "github.com/novox/mesh-host/internal/store" +) + +// The node seat node-engine's judgements on this machine (novox/hq ADR 0563): what a hand-over and a library's take +// would set and what the disk holds now, and whether this machine is root-free. + +// diskOf is what is on disk at a directory, as an act compares it when the operator's yes arrives: its owner and +// group, its mode, and which file it is. +func diskOf(info os.FileInfo) string { + owner := "unknown" + if uid, gid, ok := apply.OwnerOf(info); ok { + owner = fmt.Sprintf("%d:%d", uid, gid) + } + which := "" + if st, ok := info.Sys().(*syscall.Stat_t); ok { + which = fmt.Sprintf(" %v:%v", st.Dev, st.Ino) + } + return fmt.Sprintf("%s %o%s", owner, info.Mode().Perm(), which) +} + +// handOverJudgeFrom judges a hand-over through the node seat as the controller's ask is judged (judgeFound): the +// declared owner and mode it will set, and the disk as it is. +func handOverJudgeFrom(statePath string) func(string) (link.Judged, error) { + return func(path string) (link.Judged, error) { + found, info, err := judgeFound(statePath, path) + if err != nil { + return link.Judged{}, err + } + return link.Judged{Subject: path, Path: path, Sets: orUnsaid(found.Declared), Disk: diskOf(info), + Words: "A directory of " + moduleOf(found.ID) + " is used as found: the mesh left its owner and mode."}, nil + } +} + +func moduleOf(id string) string { + if at := strings.LastIndex(id, "."); at > 0 { + return id[:at] + } + return id +} + +// accessJudgeFrom judges the take act on a library through the node seat (ADR 0328 §7, ADR 0563 §4): declared on this +// machine, its home, and its root holding data the mesh has not taken; what the take sets, and the disk as it is. +func accessJudgeFrom(statePath string) func(string, string) (link.Judged, error) { + return func(act, access string) (link.Judged, error) { + d, err := declaredNow(statePath, nil) + if errors.Is(err, store.ErrNothingDeclared) { + return link.Judged{}, errors.New("this node keeps no declaration, so it is declared no library") + } + if err != nil { + return link.Judged{}, fmt.Errorf("what runs on this node does not read: %w", err) + } + for _, l := range d.Libraries { + if apply.LibraryAccess(l.Library) != access { + continue + } + if !l.Home { + return link.Judged{}, fmt.Errorf("this machine is not the home of the library %s, and a library is "+ + "taken on its home", l.Library) + } + ctx, cancel := context.WithTimeout(context.Background(), lookBound) + defer cancel() + if v := apply.JudgeLibrary(ctx, l); v.Kind != apply.VerdictWaitKeep { + return link.Judged{}, fmt.Errorf("the library %s does not wait for the operator's word to be kept "+ + "(it is judged %s), so there is nothing to take", l.Library, v.Kind) + } + info, err := os.Lstat(l.Root) + if err != nil { + return link.Judged{}, fmt.Errorf("the library %s's root cannot be read: %v", l.Library, err) + } + return link.Judged{Subject: access, Path: filepath.Clean(l.Root), + Sets: "the group " + l.Group + ", group write and setgid on everything under its root", + Disk: diskOf(info), Words: "The library " + l.Library + " holds data the mesh has not taken."}, nil + } + return link.Judged{}, fmt.Errorf("the access %s names no library declared on this machine", access) + } +} + +// askTakes asks the take of every library the look found waiting for the operator's word, once per library (ADR 0563 +// §4): asked by the engine itself, never a second time while one is open or after one was asked; access-act asks again +// by hand. +func askTakes(acts *link.HandActs, waiting []string, say func(string)) { + if acts == nil { + return + } + for _, access := range waiting { + if acts.AskedFor(link.ActAccess, access) { + continue + } + if _, err := acts.AskAccess(link.EngineSeat+"-"+acts.Node, link.ActTake, access, + "its root holds data outside its group", true); err != nil && say != nil { + say("the take of " + access + " could not be asked: " + err.Error()) + } + } +} + +// lastAccounts is the newest look at every account (the health loop's), which root-free judges from. +var lastAccounts atomic.Pointer[accounts.Statement] + +// rootFreshFor is how old the verdict root-free judges from may be (ADR 0563 §6, the controller's verdictFreshFor). +const rootFreshFor = 15 * time.Minute + +// judgeRootFree is this machine's root-free (ADR 0259 §8, ADR 0563 §6): free only on the engine's own verdict, at most +// rootFreshFor old, that every account declared never to become root without a person cannot, at least one such +// account declared, and the login shell's execute not served here. Anything else is no pass, and says why. +func judgeRootFree(node string, st *accounts.Statement, now time.Time, served bool, servedWhy string) link.RootVerdict { + v := link.RootVerdict{Machine: node, Judged: now.UTC()} + var not []string + confined := "" + switch { + case st == nil: + not = append(not, "this machine's engine has judged no account yet") + case now.Sub(st.At) > rootFreshFor: + not = append(not, fmt.Sprintf("the engine's newest judgement of its accounts was made at %s, more than %d "+ + "minutes ago, and a verdict that old is not a verdict on now", st.At.UTC().Format(time.RFC3339), + int(rootFreshFor.Minutes()))) + default: + var named []string + for _, a := range st.Accounts { + if !a.Root { + continue + } + named = append(named, a.Name) + if a.State != accounts.Healthy { + not = append(not, "the account "+a.Name+" "+orNoneKnownWord(a.Reason)) + } + } + if len(named) == 0 { + not = append(not, "no account here is declared never to become root, so agents run as the operator's "+ + "account, which may become root") + } else if len(not) == 0 { + confined = "the account " + strings.Join(named, ", ") + " cannot become root without a person (judged " + + st.At.UTC().Format(time.RFC3339) + ")" + } + } + if served { + not = append(not, "the login shell runs any command an agent gives it as the machine's runtime account, which "+ + "can become root ("+servedWhy+")") + } + if len(not) > 0 { + v.Why = strings.Join(not, "; ") + return v + } + v.Free, v.Why = true, confined+"; the login shell's execute is not served here" + return v +} + +func orNoneKnownWord(s string) string { + if s == "" { + return "is not judged" + } + return s +} + +// settingFileSuffix is the resource the login shell's holder renders its setting execute into (ADR 0268: zsh's +// tools-conf, `execute=serve` or `execute=withhold`), read beside the bus's discovery. +const settingFileSuffix = ".tools-conf" + +// executeServedHere is whether the login shell's execute is served on this machine (ADR 0268), failing closed: served +// while the bus's discovery hears it answered here or could not be asked, or while the holder's setting file applied +// here says anything but withhold or cannot be read. why says which. +func executeServedHere(statePath string, heard func() (bool, error)) (bool, string) { + var why []string + if heard == nil { + why = append(why, "the bus could not be asked whether execute is answered here") + } else if h, err := heard(); err != nil { + why = append(why, "the bus could not be asked whether execute is answered here: "+err.Error()) + } else if h { + why = append(why, "the bus hears execute answered here") + } + known, err := store.Load(statePath) + if err != nil { + why = append(why, "what this machine applied could not be read: "+err.Error()) + } + for _, r := range known.Resources { + if r.Type != "file" || !strings.HasSuffix(r.ID, settingFileSuffix) { + continue + } + if value, err := settingIn(r.Target); err != nil { + why = append(why, moduleOf(r.ID)+"'s setting execute cannot be read: "+err.Error()) + } else if value != "withhold" { + why = append(why, moduleOf(r.ID)+"'s setting execute here is "+value) + } + } + return len(why) > 0, strings.Join(why, "; ") +} + +// settingIn is the value of execute in a setting file, "" when it says none. +func settingIn(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + value := "" + lines := bufio.NewScanner(f) + for lines.Scan() { + t := strings.TrimSpace(lines.Text()) + if key, v, ok := strings.Cut(t, "="); ok && !strings.HasPrefix(t, "#") && strings.TrimSpace(key) == "execute" { + value = strings.TrimSpace(v) + } + } + if value == "" { + value = "not said" + } + return value, lines.Err() +} + +// keepWaiting is every library a look found waiting for the operator's word to keep it (ADR 0328 §8). +func keepWaiting(vs []link.ResourceHealth) []string { + var out []string + for _, v := range vs { + for _, w := range v.AccessWaits { + if w.Reason == link.WaitKeep { + out = append(out, w.Access) + } + } + } + return out +} + +// engineActs is the node seat this engine holds (ADR 0563): judged and performed by the same code as the controller's +// signed asks, its asks to the operator over the session's link, its acts kept beside the state. +func engineActs(opts options, node string, queue *link.Session, search *accounts.SetuidCache) *link.HandActs { + state := link.BusState{Session: queue, Node: node} + h := &link.HandActs{Node: node, State: state, + JudgeHandOver: handOverJudgeFrom(opts.state), HandOver: handOverFrom(opts.state), + JudgeAccess: accessJudgeFrom(opts.state), AccessAct: accessActFrom(opts.state, opts.timeout), + JudgeDirectories: judgeDirectoriesFrom(opts.state), SetuidSearch: queue.SetuidSearch, Searching: search.Running, + RootFree: func() link.RootVerdict { + served, why := executeServedHere(opts.state, func() (bool, error) { + return queue.HeardServed("node-login-shell", "execute") + }) + return judgeRootFree(node, lastAccounts.Load(), time.Now(), served, why) + }, + Say: queue.SayEngine} + asker := asks.EngineAsker(node) + h.Asker = &meshasker.Client{Module: asker, State: state, Publish: queue.PublishAsk, + Record: func(id string) (*stdio.StateEntry, error) { return queue.RouterRecord(asker + "." + id) }} + return h +} + +// reconcileEvery is how often the engine catches up its open asks from the router's records (a warrant heard while no +// link was open, an ask past its expiry): besides each link's opening. +const reconcileEvery = 5 * time.Minute + +func reconcileActs(ctx context.Context, h *link.HandActs) { + t := time.NewTicker(reconcileEvery) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + h.Reconcile() + } + } +} diff --git a/cmd/mesh-host/file_references_test.go b/cmd/mesh-host/file_references_test.go index a8303fb..0be46f6 100644 --- a/cmd/mesh-host/file_references_test.go +++ b/cmd/mesh-host/file_references_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "fmt" "os" "path/filepath" @@ -10,6 +11,7 @@ import ( "testing" "git.novox.be/novox/mesh-sdk/go/declaration" + "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/nats-io/nats.go" "github.com/novox/mesh-host/internal/link" ) @@ -82,6 +84,77 @@ func TestADeclarationWithAFileByDigestAppliesFromTheObject(t *testing.T) { if err != nil || string(got) != string(body) { t.Errorf("1695/AC1: the file holds %d bytes (%v), want %d", len(got), err, len(body)) } + + // Issue 924 (novox, 2026-10-29 01:16): the controller's sweep removed an object a composition the node ran still named, + // and the module's apply failed while it ran. Content the node once had is its own keep's, and the bus is not asked. + gone := aFilesLink{err: fmt.Errorf("the object %s was deleted from the bucket declaration-files-novox", name)} + never := []byte(strings.Repeat("never had here\n", 6000)) + for _, row := range []struct { + name string + raw []byte + onDisk []byte // nil removes the file + refused string // "" applies + }{ + {"the file here drifted, its object deleted", raw, []byte("the module wrote it since"), ""}, + {"the file here is gone, its object deleted", raw, nil, ""}, + {"never had here, its object deleted", referenceDeclaration(target, never), nil, "never held it"}, + } { + if row.onDisk == nil { + _ = os.Remove(target) + } else if err := os.WriteFile(target, row.onDisk, 0o600); err != nil { + t.Fatal(err) + } + report = applyAndKeep(link.WithFiles(context.Background(), gone), opts, row.raw, nil, nil, nil) + if row.refused == "" { + got, _ := os.ReadFile(target) + if report.Refused != "" || string(got) != string(body) { + t.Errorf("924: %s: refused %q, the file holds %d bytes; want it applied from the node's own keep", + row.name, report.Refused, len(got)) + } + } else if !strings.Contains(report.Refused, row.refused) || !strings.Contains(report.Refused, "was deleted") { + t.Errorf("924: %s: refused %q; want a refusal saying the bucket and the keep both lack it", row.name, report.Refused) + } + } + + // Issue 924, the live case: what runs came from the kept declaration, whose export names a digest swept long ago, + // while the node wrote a newer export since. Composed as it runs, the file names what the node wrote, and applies with + // no object; a file someone else changed since is left as named. + newer := []byte(strings.Repeat("the export the node wrote since\n", 3000)) + newerName, _ := declaration.ObjectName(declaration.DigestOf(newer)) + ctx = link.WithFiles(context.Background(), aFilesLink{objects: map[string][]byte{newerName: newer}}) + if report = applyAndKeep(ctx, opts, referenceDeclaration(target, newer), nil, nil, nil); report.Refused != "" { + t.Fatalf("924: the newer export: %s", report.Refused) + } + stale := json.RawMessage(`{"id":"m.big","type":"file","path":"` + target + `","digest":"` + declaration.DigestOf(body) + + `","size":` + strconv.Itoa(len(body)) + `}`) + for _, row := range []struct { + name string + edit []byte // what someone else wrote there since; nil for nobody + digest string + }{ + {"the file is what the node wrote", nil, declaration.DigestOf(newer)}, + {"someone changed the file since", []byte("a hand edit"), declaration.DigestOf(body)}, + } { + if row.edit != nil { + if err := os.WriteFile(target, row.edit, 0o600); err != nil { + t.Fatal(err) + } + } + runs := runsAsWritten([]delivery.Composed{{Node: "n", Module: "m", Resources: []json.RawMessage{stale}}}, opts.state) + var got struct { + Digest string `json:"digest"` + } + _ = json.Unmarshal(runs[0].Resources[0], &got) + if got.Digest != row.digest { + t.Errorf("924: %s: what runs names %s, want %s", row.name, got.Digest, row.digest) + } + if row.edit == nil { + raw := []byte(`{"declaration":1,"resources":[` + string(runs[0].Resources[0]) + `]}`) + if report = applyAndKeep(link.WithFiles(context.Background(), gone), opts, raw, nil, nil, nil); report.Refused != "" { + t.Errorf("924: %s: composed as it runs, refused %q; want applied with no object", row.name, report.Refused) + } + } + } } // 1695/AC8: the hand-run `apply ` and `reconcile` have no bus: a reference whose file here verifies is applied diff --git a/cmd/mesh-host/filling.go b/cmd/mesh-host/filling.go index d77d2e9..fa241e8 100644 --- a/cmd/mesh-host/filling.go +++ b/cmd/mesh-host/filling.go @@ -3,22 +3,25 @@ package main import ( "context" "crypto/ed25519" + "encoding/json" "errors" "fmt" "path/filepath" "sync/atomic" "git.novox.be/novox/mesh-sdk/go/declaration" + "git.novox.be/novox/mesh-sdk/go/provisions" "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/filling" + "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" ) // serveFilling writes filled files on the provisioner client's request for as long as this engine serves (novox/hq // ADR 0349 §4). A socket that cannot be served is said once, and this engine goes on applying. -func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, say func(string)) { - e := fillingEngine(opts, signer, say) +func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, queue *link.Session, node string, say func(string)) { + e := fillingEngine(opts, signer, provisionSecretsOn(queue, node, opts.state), say) fillingNow.Store(e) defer fillingNow.CompareAndSwap(e, nil) if err := filling.Serve(ctx, filling.Socket, e, say); err != nil { @@ -30,9 +33,10 @@ func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, s var fillingNow atomic.Pointer[filling.Engine] // knockFilling tells the provisioner's client to fill again what the filling refused or failed and the declaration -// applied now may let through (filling.Engine.Knock), once a module's apply or removal settled (agent.Env.Settled). -// What it told, or why it could not, is said. -func knockFilling(module string, say func(string)) { +// applied now may let through (filling.Engine.Knock), after what runs changed: a module's apply or removal settled +// (agent.Env.Settled, " settled"), the haves were read at the start, or a module was taken as running +// (agent.Env.RunsChanged, issue 924). What it told, or why it could not, is said. +func knockFilling(after string, say func(string)) { e := fillingNow.Load() if e == nil { return @@ -40,16 +44,19 @@ func knockFilling(module string, say func(string)) { told, err := e.Knock(filling.Socket) switch { case err != nil: - say(fmt.Sprintf("after %s settled: %v", module, err)) + say(fmt.Sprintf("after %s: %v", after, err)) case told != "": - say(fmt.Sprintf("after %s settled, the provisioner's client is told to fill again: %s", module, told)) + say(fmt.Sprintf("after %s, the provisioner's client is told to fill again: %s", after, told)) } } // fillingEngine is the write of a filled file as this engine does it: the declaration it keeps, verified, read under // the apply's lock — the same two locks every apply takes, the process's and the one beside the state — and the file // written through the apply (apply.WriteFilled), its outcome recorded in the node's state as an apply's is. -func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *filling.Engine { +// +// A provision's `${secret:}` the client left in the content is filled first (secrets, novox/hq ADR 0570 clause 6), and a +// file so filled is written 0600 when the declaration gives no mode. +func fillingEngine(opts options, signer ed25519.PublicKey, secrets provisionSecrets, say func(string)) *filling.Engine { return &filling.Engine{ Declared: func() (*declaration.Declaration, error) { return declaredNow(opts.state, signer) @@ -73,8 +80,13 @@ func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *fi if err != nil { return "", "", err } - report, updated, writeErr := apply.WriteFilled(ctx, built, d, known, path, content, apply.ExecRunner, - announceOr(say), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state))) + content, secret, err := secrets.fill(ctx, filledModule(d, path), content) + if err != nil { + return "", "", fmt.Errorf("%s was not written: %w", path, err) + } + report, updated, writeErr := apply.WriteFilledWith(ctx, built, d, known, path, content, apply.ExecRunner, + announceOr(say), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state)), + apply.Filled{Secret: secret}) // Saved whichever way it went, as an apply's: what was written is on the machine either way. if err := store.Save(opts.state, updated); err != nil { return "", "", errors.Join(writeErr, errors.New("written, and the node's state could not be saved: "+ @@ -93,3 +105,57 @@ func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *fi }, } } + +// filledModule is the module whose file the declaration declares at path, or "" when none or several do. +func filledModule(d *declaration.Declaration, path string) string { + module := "" + for _, r := range d.Resources { + if f, ok := r.(*declaration.File); ok && f.Path == path { + if module != "" { + return "" + } + module, _ = d.ModuleOfID(f.ID) + } + } + return module +} + +// provisionSecretsOn reads provisions and asks the vault over the link, and opens with this machine's sealing key. +func provisionSecretsOn(queue *link.Session, node, statePath string) provisionSecrets { + p := provisionSecrets{node: node, open: sealOpener(statePath)} + if queue == nil { + return p + } + // What the bus holds under prefix, read by a watch to its caught-up mark: no poll, no list of every key. + p.list = func(ctx context.Context, prefix string) (map[string][]byte, error) { + wctx, cancel := context.WithCancel(ctx) + defer cancel() + updates, err := queue.KVWatch(wctx, provisions.ProvisionsBucket, prefix+">") + if err != nil { + return nil, err + } + out := map[string][]byte{} + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case u, ok := <-updates: + if !ok { + return nil, fmt.Errorf("the watch of %s ended before it caught up", provisions.ProvisionsBucket) + } + switch { + case u.Caught: + return out, nil + case u.Deleted: + delete(out, u.Key) + default: + out[u.Key] = u.Value + } + } + } + } + p.own = func(ctx context.Context, args map[string]any) (json.RawMessage, error) { + return queue.AskVault(ctx, "own", args) + } + return p +} diff --git a/cmd/mesh-host/filling_test.go b/cmd/mesh-host/filling_test.go index 828682a..7885eb7 100644 --- a/cmd/mesh-host/filling_test.go +++ b/cmd/mesh-host/filling_test.go @@ -7,9 +7,12 @@ import ( "os" "os/user" "path/filepath" + "strings" "testing" "time" + "git.novox.be/novox/mesh-sdk/go/provisions" + "github.com/novox/mesh-host/internal/filling" "github.com/novox/mesh-host/internal/store" ) @@ -45,7 +48,34 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) { store.Declared{Declaration: raw, Signature: ed25519.Sign(private, raw)}); err != nil { t.Fatal(err) } - e := fillingEngine(opts, public, func(string) {}) + // The provision m reads, as mesh-provisioner keeps it, and the vault's value of it, sealed to this machine (faked: + // the opener reads the box's "v"). + provision := map[string]string{} + vaultName := "provision/n2/db" + secrets := provisionSecrets{node: "n1", + list: func(_ context.Context, prefix string) (map[string][]byte, error) { + out := map[string][]byte{} + for k, v := range provision { + if strings.HasPrefix(k, prefix) { + out[k] = []byte(v) + } + } + return out, nil + }, + own: func(_ context.Context, args map[string]any) (json.RawMessage, error) { + raw, _ := json.Marshal(args) + var a struct{ Credentials []map[string]any } + _ = json.Unmarshal(raw, &a) + c := a.Credentials[0] + c["name"], c["generation"], c["sealed"] = vaultName, 2, map[string]string{"v": "s3cret"} + return json.Marshal(map[string]any{"node": "n1", "credentials": []any{c}}) + }, + open: func(sealed string) ([]byte, error) { + var b struct{ V string } + err := json.Unmarshal([]byte(sealed), &b) + return []byte(b.V), err + }} + e := fillingEngine(opts, public, secrets, func(string) {}) uid := uint32(os.Getuid()) if err := e.Accepts(uid); err != nil { t.Fatal(err) @@ -102,4 +132,75 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) { if a := <-done; a.Action != "updated" { t.Fatalf("the second write: %+v", a) } + + // ADR 0570 clause 6: the client leaves a provision's `${secret:}` to the node-engine, which fills it at the write + // from the vault's value of a provided provision, and writes the file 0600; a provision not provided refuses it. + keyed := func(state string, ref bool, id, local string) string { + p := provisions.Provision{Provision: "db", Local: local, Consumer: provisions.End{Module: "m", Node: "n1"}, + Provider: provisions.End{Module: "postgres", Node: "n2"}, State: state, At: time.Now()} + if state != provisions.Provided { + p.Why = "the provider has not answered" + } + if ref { + p.Reference = &provisions.Reference{ID: id, Generation: 2} + } + raw, err := provisions.Encode(p) + if err != nil { + t.Fatal(err) + } + return string(raw) + } + for _, tc := range []struct { + name, state string + ref bool + content string + want string // the file after; "" keeps what was there + mode os.FileMode + vault string // the vault's credential name, when not the one the provision names + refID string // the reference's id, when not the provision's shared value + local string // the provision's local name (ADR 0094): the placeholder names it + }{ + {name: "a provision not provided is refused", state: provisions.Asked, content: "password=${secret:db}\n"}, + {name: "provided with no reference is refused", state: provisions.Provided, content: "password=${secret:db}\n"}, + {name: "a reference to no credential of the provision is refused", state: provisions.Provided, ref: true, + refID: "provision/n9/db", content: "password=${secret:db}\n"}, + {name: "a name no provision goes by is refused", state: provisions.Provided, ref: true, content: "password=${secret:other}\n"}, + {name: "the vault's value is another than the provision names", state: provisions.Provided, ref: true, + vault: "pair/n2/db/n1/m", content: "password=${secret:db}\n"}, + {name: "provided: filled at the write, 0600", state: provisions.Provided, ref: true, + content: "password=${secret:db}\n", want: "password=s3cret\n", mode: 0o600}, + {name: "a local name is the provision's by its key", state: provisions.Provided, ref: true, local: "admin", + content: "admin=${secret:admin}\n", want: "admin=s3cret\n", mode: 0o600}, + {name: "no placeholder is written as it came, as the declaration says", content: "plain\n", want: "plain\n", mode: 0o644}, + } { + before, _ := os.ReadFile(path) + if tc.state != "" { + id := "provision/n2/db" + if tc.refID != "" { + id = tc.refID + } + clear(provision) + key := "n1.m.db" + if tc.local != "" { + key += "." + tc.local + } + provision[key] = keyed(tc.state, tc.ref, id, tc.local) + } + vaultName = "provision/n2/db" + if tc.vault != "" { + vaultName = tc.vault + } + a := e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte(tc.content)}) + got, _ := os.ReadFile(path) + if tc.want == "" { + if a.Failed == "" && a.Refused == "" || string(got) != string(before) { + t.Errorf("%s: answered %+v, the file holds %q", tc.name, a, got) + } + continue + } + info, _ := os.Stat(path) + if a.Failed != "" || a.Refused != "" || string(got) != tc.want || info.Mode().Perm() != tc.mode { + t.Errorf("%s: answered %+v, the file holds %q, mode %v", tc.name, a, got, info.Mode()) + } + } } diff --git a/cmd/mesh-host/found.go b/cmd/mesh-host/found.go index 48774d3..eeebed0 100644 --- a/cmd/mesh-host/found.go +++ b/cmd/mesh-host/found.go @@ -68,14 +68,23 @@ func foundDirectories(known store.State) []store.Applied { // known (novox/hq ADR 0272): typed on the control-node, from the operator's own account. The directory's // path is the resource's target beside the reason, never in it: the controller says a reason on the // operator's channel, which withholds a path (ADR 0234 §6), and keeps the target as evidence. +// +// From the lease write, heard as the engine hears it (meshTrust), the line has no answerer: the way out is the node +// seat's hand-over with the directory's path, which asks the operator on their channel (novox/hq ADR 0563 §5). func handOverLine() string { node, _ := foundNode.Load().(string) if node == "" { node = "" } + if t := foundTrust.Load(); t != nil && t.HandedAway() { + return fmt.Sprintf("`%s/node-engine.hand-over` with the directory's path, then your yes on your channel", node) + } return fmt.Sprintf("`nox node hand-over %s ` on the control-node", node) } +// foundTrust is what the engine trusts, whose lease decides the way out a directory used as found names. +var foundTrust atomic.Pointer[meshTrust] + // withFoundDirectories adds to a statement every directory used as found, as its module's resource of kind // directory, unhealthy: the reason names its owner and mode as they are now, what was declared, and the line // that hands it over. The path is the target, which the controller keeps as evidence. @@ -129,16 +138,30 @@ func foundNow() []store.Applied { // never follows a path it did not state: a hand-over is root giving a directory to an account at the next // apply, and a path that resolves elsewhere would give that elsewhere. func handOverFound(statePath, path, by string) (string, error) { + found, _, err := judgeFound(statePath, path) + if err != nil { + return "", err + } + if err := store.AddHandedOver(statePath, store.HandOver{ID: found.ID, Path: path, At: time.Now().UTC(), By: by}); err != nil { + return "", err + } + return fmt.Sprintf("%s (%s) is handed to the mesh by %s: the next apply gives it %s, and it is the mesh's from "+ + "then on — removed when empty and no longer declared", path, found.ID, by, orUnsaid(found.Declared)), nil +} + +// judgeFound is the judgement of a hand-over's path, recording nothing: the directory this machine uses as found at +// that path, as the engine states it, and what is on disk there. +func judgeFound(statePath, path string) (store.Applied, os.FileInfo, error) { if !filepath.IsAbs(path) { - return "", fmt.Errorf("hand-over takes the directory's absolute path, and %q is not one; nothing was handed over", path) + return store.Applied{}, nil, fmt.Errorf("hand-over takes the directory's absolute path, and %q is not one; nothing was handed over", path) } if filepath.Clean(path) != path { - return "", fmt.Errorf("%q is not the directory's path as the engine states it (no `..`, no doubled or trailing "+ + return store.Applied{}, nil, fmt.Errorf("%q is not the directory's path as the engine states it (no `..`, no doubled or trailing "+ "separator); nothing was handed over", path) } known, err := store.Load(statePath) if err != nil { - return "", err + return store.Applied{}, nil, err } var found *store.Applied for _, r := range foundDirectories(known) { @@ -149,31 +172,27 @@ func handOverFound(statePath, path, by string) (string, error) { } } if found == nil { - return "", errors.New(path + " is not a directory this machine uses as found (the module's condition names " + + return store.Applied{}, nil, errors.New(path + " is not a directory this machine uses as found (the module's condition names " + "the ones that are); nothing was handed over") } info, err := os.Lstat(path) switch { case err != nil: - return "", fmt.Errorf("%s cannot be read on this machine: %v; nothing was handed over", path, err) + return store.Applied{}, nil, fmt.Errorf("%s cannot be read on this machine: %v; nothing was handed over", path, err) case info.Mode()&os.ModeSymlink != 0: - return "", fmt.Errorf("%s is a link, and the mesh hands over no link (novox/hq ADR 0012); nothing was handed over", path) + return store.Applied{}, nil, fmt.Errorf("%s is a link, and the mesh hands over no link (novox/hq ADR 0012); nothing was handed over", path) case !info.IsDir(): - return "", fmt.Errorf("%s is not a directory on this machine; nothing was handed over", path) + return store.Applied{}, nil, fmt.Errorf("%s is not a directory on this machine; nothing was handed over", path) } resolved, err := filepath.EvalSymlinks(path) if err != nil { - return "", fmt.Errorf("%s cannot be resolved on this machine: %v; nothing was handed over", path, err) + return store.Applied{}, nil, fmt.Errorf("%s cannot be resolved on this machine: %v; nothing was handed over", path, err) } if resolved != path { - return "", fmt.Errorf("%s is reached through a link (it is %s), and the mesh hands over nothing through one "+ + return store.Applied{}, nil, fmt.Errorf("%s is reached through a link (it is %s), and the mesh hands over nothing through one "+ "(novox/hq issue 339); nothing was handed over", path, resolved) } - if err := store.AddHandedOver(statePath, store.HandOver{ID: found.ID, Path: path, At: time.Now().UTC(), By: by}); err != nil { - return "", err - } - return fmt.Sprintf("%s (%s) is handed to the mesh by %s: the next apply gives it %s, and it is the mesh's from "+ - "then on — removed when empty and no longer declared", path, found.ID, by, orUnsaid(found.Declared)), nil + return *found, info, nil } // handOverFrom is the engine's hand-over for the controller's ask: the state beside statePath, the path as asked. diff --git a/cmd/mesh-host/found_test.go b/cmd/mesh-host/found_test.go index 4a13d2c..f756bfc 100644 --- a/cmd/mesh-host/found_test.go +++ b/cmd/mesh-host/found_test.go @@ -40,6 +40,21 @@ func TestAFoundDirectoryIsStatedUnhealthy(t *testing.T) { strings.Contains(r.Reason, "mesh-host") || strings.Contains(r.Reason, dir) { t.Fatalf("the way out is not the operator's line, or carries the path: %q", r.Reason) } + // ADR 0563 §5: from the lease write the remedy is the node seat's hand-over, answered on the operator's channel, and + // no longer the controller's terminal line; the lease is the one the engine hears. + handed := newMeshTrust(nil) + handed.handedAway = true + foundTrust.Store(handed) + if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason, + "`laptop/node-engine.hand-over`") || strings.Contains(r.Reason, "nox node hand-over") || strings.Contains(r.Reason, dir) { + t.Fatalf("after the lease write the way out is %q", r.Reason) + } + foundTrust.Store(newMeshTrust(nil)) + if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason, + "`nox node hand-over laptop `") { + t.Fatalf("while the controller holds its lease the way out is %q", r.Reason) + } + foundTrust.Store((*meshTrust)(nil)) foundNode.Store("") if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason, "`nox node hand-over `") { diff --git a/cmd/mesh-host/libraries_test.go b/cmd/mesh-host/libraries_test.go index 2f3aafb..10910c4 100644 --- a/cmd/mesh-host/libraries_test.go +++ b/cmd/mesh-host/libraries_test.go @@ -8,6 +8,8 @@ import ( "testing" "time" + meshasker "git.novox.be/novox/mesh-sdk/go/asker" + "git.novox.be/novox/mesh-sdk/go/asks" "git.novox.be/novox/mesh-sdk/go/declaration" "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/link" @@ -53,4 +55,42 @@ func TestALibrarySaysItsWaitWithoutWordsToParse(t *testing.T) { if strings.Contains(media.Reason, dir) || len(books.AccessWaits) != 1 || books.AccessWaits[0].Reason != link.WaitDisk { t.Fatalf("a missing root does not wait for its disk, or a reason names a path: %+v %+v", books, media) } + // 774 (ADR 0563 §4): the library's home engine asks its take of the operator itself, once: no second ask while one is + // open, nor after it ended; and a library waiting for its disk is not asked. + t.Run("the home engine asks the take once", func(t *testing.T) { + var asked []asks.Ask + state := &link.MemState{} + h := &link.HandActs{Node: "n1", State: state, + JudgeAccess: func(act, access string) (link.Judged, error) { + return link.Judged{Subject: access, Path: root, Sets: "the group mesh-media", Disk: "0:0 755"}, nil + }, + AccessAct: func(link.AccessAct) (string, error) { return "taken", nil }} + h.Asker = &meshasker.Client{Module: asks.EngineAsker("n1"), State: state, + Publish: func(subject string, body any, id string) error { + if a, ok := body.(asks.Ask); ok { + asked = append(asked, a) + } + return nil + }} + waiting := keepWaiting(vs) + askTakes(h, waiting, nil) + askTakes(h, waiting, nil) + if len(waiting) != 1 || waiting[0] != "library:media" || len(asked) != 1 { + t.Fatalf("waiting %v asked %d times", waiting, len(asked)) + } + h.Heard(asks.Warrant{Ask: asked[0].ID, Asker: asks.EngineAsker("n1"), Outcome: asks.OutcomeExpired, At: time.Now()}) + askTakes(h, waiting, nil) + if len(asked) != 1 { + t.Fatalf("the take was asked again after its ask ended: %d", len(asked)) + } + // Kept in a key of its own (rev-l3 round 1): the acts gone from the record never make it ask again. + keys, _ := state.Keys("act.") + for _, k := range keys { + _ = state.Delete(k) + } + askTakes(h, waiting, nil) + if len(asked) != 1 { + t.Fatalf("the take was asked again with the acts gone: %d", len(asked)) + } + }) } diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index d8b3610..505bc93 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -516,7 +516,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw // A file given by digest is had before the plan is drawn from it (novox/hq ADR 0486 §1, issue 1695). This command has // no link to the bus, so a file already here that verifies is used, and any other is refused: never written empty. - if err := apply.ResolveFileReferences(ctx, d, link.FilesFrom(ctx)); err != nil { + if err := apply.ResolveFileReferencesKept(ctx, d, link.FilesFrom(ctx), objectKeep(opts)); err != nil { return fmt.Errorf("a file of this declaration could not be had by its digest: %w", err) } @@ -1105,10 +1105,17 @@ func runLink(ctx context.Context, opts options) error { // the bus, answer what the controller asks of this machine, and say its facts: when a link opens, and when the // controller asks. proof := &proving{statePath: opts.state, version: runningVersion(), say: say} + // What this engine trusts a mesh signature from: the controller's key until its lease is handed away, and the vault's + // once read from its signers (novox/hq ADR 0550 §2). + trust := newMeshTrust(mine.Membership.Signer) + // And the way out a directory used as found names follows the lease it hears (novox/hq ADR 0563 §5). + foundTrust.Store(trust) queue := &link.Session{ Membership: membership, // And the public halves and certificate its record says (novox/hq ADR 0555 §3), read at each facts report. - Keys: recordKeys(opts.state), + Keys: recordKeys(opts.state, trust.Trusts), + // And the controller's key, read per signed ask: none once its lease is handed away (novox/hq ADR 0563 §7). + ControllerKey: trust.Controller, // And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8). HaveWritten: proof.haveWritten, // And the operator's hand-over of a directory used as found, carried by the controller from the @@ -1129,6 +1136,10 @@ func runLink(ctx context.Context, opts options) error { Say: say, Timeout: opts.timeout, } + // And the node seat node-engine this engine holds (novox/hq ADR 0563): its verbs on every link, its own asks to the + // operator under node-engine-, and its acts kept beside its state. + queue.Acts = engineActs(opts, mine.Node, queue, setuidSearch) + go reconcileActs(ctx, queue.Acts) current := &agentNow{} setRunsHere(runsFromAgent(current)) factsOfNow := func(ctx context.Context, ag *agent.Agent) { @@ -1173,7 +1184,10 @@ func runLink(ctx context.Context, opts options) error { } // The node follows its wants: one module at a time from the composition its want names (novox/hq ADR 0539), and the // busy containers are asked for again soon (ADR 0360 §11). - go nodeAgent(aside, opts, mine.Membership.Signer, queue, sched, say, afterApply, current, factsOfNow) + // And what it trusts, followed on the bus, said in its record whenever it changes (ADR 0550 §2). + trust.onChange(func() { factsOfNow(aside, current.get()) }) + go followTrust(aside, queue, trust, say) + go nodeAgent(aside, opts, mine.Membership.Signer, trust, queue, sched, say, afterApply, current, factsOfNow) go busyPasses(aside, func() { if ag := current.get(); ag != nil { ag.Retry(busyModules()...) @@ -1186,7 +1200,7 @@ func runLink(ctx context.Context, opts options) error { go serveMeshCLI(aside, queue, say) // And a filled file is written on the provisioner client's request, one at a time with the apply (novox/hq ADR // 0349 §4). - go serveFilling(aside, opts, mine.Membership.Signer, say) + go serveFilling(aside, opts, mine.Membership.Signer, queue, mine.Node, say) held := link.HoldRoused(aside, membership, queue, say, opts.timeout, rousedBySignal(ctx)) // **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back @@ -1339,6 +1353,8 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, var lastSaid time.Time owed := true spaced := 1.0 + // healthUnwritten is why the health report was last not written, said once each time it changes. + var healthUnwritten string // A module whose change ended unjudged is looked at and said at once (review of fix 15): a module with nothing // long-running is then ok within a look of its apply, not at the five-minute statement. var nudged <-chan struct{} @@ -1370,6 +1386,7 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, if a := accountJudge; a != nil { as, accountsChanged := a.Look(ctx) accountSt = &as + lastAccounts.Store(&as) owed = owed || accountsChanged if accountsChanged { for _, v := range as.Accounts { @@ -1384,6 +1401,9 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, accessSt, accessesChanged := accesses.look(time.Now()) keeping.poke(ctx) owed = owed || accessesChanged + // And a library whose root holds data outside its group is asked of the operator by this engine, once (novox/hq + // ADR 0563 §4). + askTakes(queue.Acts, keepWaiting(accessSt), say) if accessesChanged { for _, v := range accessSt { if v.State != link.StateHealthy { @@ -1418,6 +1438,11 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) && (unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy()) && len(found) == 0 && accessesHealthy(accessSt) + h := *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), + unitSt), accountSt), found), accessSt) + // The node's own health report in its key on the bus, compared on every look and written only when what it says + // changed (issue 774): what the watchman raises the machine's alerts from, without the controller. + writeHealthReport(ctx, queue, h, netSt, &healthUnwritten, say) if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway { continue } @@ -1444,13 +1469,40 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session, reconcile(inactive...) } } - if queue.SayHealth(ctx, *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt), - unitSt), accountSt), found), accessSt)) { + if queue.SayHealth(ctx, h) { lastSaid, owed = time.Now(), false } } } +// writeHealthReport writes this node's health report to its key (link.WriteNodeHealth) when it changed. Why it could +// not is said once per reason, and said again when it is written after all. +func writeHealthReport(ctx context.Context, queue *link.Session, h link.Health, ns *network.Statement, unwritten *string, + say link.Announce) { + if queue == nil { + return + } + var peers []link.PeerSeen + if ns != nil { + for _, p := range ns.Peers { + peers = append(peers, link.PeerSeen{Key: p.Key, Addresses: p.Addresses, Seen: p.Seen}) + } + } + var machines link.Machines + if c := machinesNow.Load(); c != nil { + machines.ByAddress, machines.Hub = c.Machines() + } + _, err := queue.WriteNodeHealth(ctx, link.NodeHealthOf(queue.Membership.Node, h, peers, machines)) + switch { + case err != nil && err.Error() != *unwritten: + *unwritten = err.Error() + say("this machine's health report is not written to the bus: " + err.Error()) + case err == nil && *unwritten != "": + *unwritten = "" + say("this machine's health report is written to the bus again") + } +} + // healthAsReported is a statement as the report and the event carry it. func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health { // LibraryContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a @@ -1646,6 +1698,15 @@ func announceOr(say link.Announce) func(string) { // run by hand beside this service — the lock beside the state does the same (store.Lock). var applying sync.Mutex +// objectKeep is the node's own keep of the objects its compositions name by digest, beside its state (apply.ObjectKeep, +// issue 924): a composition the node once applied never again needs the controller's bucket to hold its files. +func objectKeep(opts options) *apply.ObjectKeep { + if opts.state == "" { + return nil + } + return apply.KeepObjectsIn(filepath.Join(filepath.Dir(opts.state), "objects")) +} + // appliedModule is how the apply of one module's declaration came out. type appliedModule struct { // Refused is why nothing was applied, when nothing was. @@ -1685,7 +1746,7 @@ func applyModuleHeld(ctx context.Context, opts options, declared *declaration.De // **A file given by digest is fetched and verified before anything of the composition is applied** (novox/hq ADR // 0486 §1, issue 1695): one object missing or differing refuses the module whole. - if err := apply.ResolveFileReferences(ctx, declared, link.FilesFrom(ctx)); err != nil { + if err := apply.ResolveFileReferencesKept(ctx, declared, link.FilesFrom(ctx), objectKeep(opts)); err != nil { return appliedModule{Refused: "a file of this composition could not be had by its digest: " + err.Error()} } @@ -2167,9 +2228,12 @@ func withMissing(st liveness.Statement) liveness.Statement { // recordKeys reads the public halves of the node's keys and the certificate the mesh issued for its internal name from // its state directory (novox/hq ADR 0555 §3): what the node's record says of them. A key that cannot be read is left // empty, never invented; the certificate is there only after the first declaration that carried it. -func recordKeys(state string) func() link.RecordKeys { +func recordKeys(state string, trusts func() []string) func() link.RecordKeys { return func() link.RecordKeys { var k link.RecordKeys + if trusts != nil { + k.Trusts = trusts() + } if id, err := identity.Load(identity.Path(state)); err == nil { k.TunnelKey = id.Overlay.Public } diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 430be23..a67f6a0 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -5,12 +5,14 @@ import ( "crypto/ed25519" "encoding/base64" "encoding/json" + "errors" "fmt" "os" "path/filepath" "sort" "strings" "sync" + "sync/atomic" "time" "git.novox.be/novox/mesh-sdk/go/compose/frombus" @@ -343,13 +345,14 @@ func resourcesOf(cs []delivery.Composed) []declaration.Resource { // controller's key the node was enrolled with (novox/hq ADR 0539 §3, §11); a node adopted later has it at its adoption. const operatorKeysFile = "operator-keys" -// trustedKeys are the keys a composition may be signed by: the controller's, which the node was enrolled with, and the -// operator's, read from operatorKeysFile each time it is asked, so a key that arrives while the agent runs is trusted at -// the next composition. A file that cannot be read trusts the controller's alone, and says so. -func trustedKeys(opts options, controller ed25519.PublicKey) func() []ed25519.PublicKey { +// trustedKeys are the keys a composition may be signed by: the controller's, which the node was enrolled with, while the +// controller holds its lease (ADR 0550 §2: trusted no more once it is handed away), and the operator's, read from +// operatorKeysFile each time it is asked, so a key that arrives while the agent runs is trusted at the next composition. +// A file that cannot be read trusts the controller's alone, and says so. +func trustedKeys(opts options, controllerNow func() ed25519.PublicKey) func() []ed25519.PublicKey { return func() []ed25519.PublicKey { keys := []ed25519.PublicKey{} - if len(controller) == ed25519.PublicKeySize { + if controller := controllerNow(); len(controller) == ed25519.PublicKeySize { keys = append(keys, controller) } raw, err := os.ReadFile(filepath.Join(filepath.Dir(opts.state), operatorKeysFile)) @@ -373,13 +376,16 @@ func trustedKeys(opts options, controller ed25519.PublicKey) func() []ed25519.Pu } // keptOnce reads the kept declaration once and splits it by module: its resources (the node's own are those that say -// none, ADR 0539 §9) and its filled paths. +// none, ADR 0539 §9) and its filled paths. Once the read is retired (keptRetired) nothing is read. func keptOnce(opts options, signer ed25519.PublicKey) func() (map[string][]json.RawMessage, map[string][]string) { var once sync.Once var by map[string][]json.RawMessage var filled map[string][]string return func() (map[string][]json.RawMessage, map[string][]string) { once.Do(func() { + if keptRetired.Load() { + return + } raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer) if err != nil { return @@ -429,7 +435,7 @@ func splitKept(raw []byte) (map[string][]json.RawMessage, map[string][]string) { // nodeAgent runs the node agent under the session's links for as long as ctx lasts: it waits for a link, follows the // node's wants and compositions on it, and begins again with the next link. A watch it cannot make (a bucket this node's // account may not read) is said, and tried again with a growing wait, as the link is. -func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, session *link.Session, sched *apply.Scheduler, +func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trust *meshTrust, session *link.Session, sched *apply.Scheduler, say link.Announce, afterApply func(), current *agentNow, sayFacts func(ctx context.Context, ag *agent.Agent)) { wait := 2 * time.Second for ctx.Err() == nil { @@ -446,6 +452,18 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess return } var ag *agent.Agent + // The kept release-5 declaration is no longer read once every module the seat's placement puts here has a have of its + // own (ADR 0550 §2: its signature is the controller's, trusted no more after the lease write). + // Nor before this engine keeps the node's own as it runs (applied/node.json): until then the kept declaration is the one + // record of what the node's own composition carries, its folded modules among them (issue 774, review of the + // re-land, round 1: novox's first start would compose its own with nats running nothing). + if retired, why := placedAllHave(linked, session, node); retired && !keptRetired.Load() { + if !appliedStoreFor(opts.state).holds(agent.NodeModule) { + say("the kept declaration is still read: " + why + ", but this engine keeps no node's own composition yet") + } else if !keptRetired.Swap(true) { + say("the kept declaration is no longer read: " + why) + } + } kept := keptOnce(opts, signer) // The mesh's own certificate the declaration kept from before carries, kept for the record before this machine // composes its own (ADR 0555 §3): without it a module that asks for a certificate would be stood in for. @@ -458,7 +476,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess } // The bus's user list, written alone from the newest composition of the bus module (ADR 0547 §8): on every // node, and a node whose compositions carry no list writes nothing. - users := userListFollower(opts, session, composedBucket, trustedKeys(opts, signer), say, func() { + users := userListFollower(opts, session, composedBucket, trustedKeys(opts, trust.Controller), say, func() { // This engine wrote the list, so the bus reloads its accounts and answers no JetStream request for a few // seconds: what failed in them is tried again once that is over, not at its next wait. time.AfterFunc(reloadSettles, func() { @@ -483,7 +501,10 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess // The same kept declaration's filled paths, so a module still running from it keeps its files filled. KeptFilled: func() map[string][]string { _, filled := kept(); return filled }, Recorded: recordedIn(opts.state, say), - Inactive: inactiveIn(&liveness.Exec{Run: apply.ExecRunner, System: builtFor}, heldBy(apply.WindowsIn(filepath.Dir(opts.state)))), + // What this engine applied, kept on the machine and read at the start, so what runs is known before the node + // composes its own (issue 774). + Applied: appliedStoreFor(opts.state), + Inactive: inactiveIn(&liveness.Exec{Run: apply.ExecRunner, System: builtFor}, heldBy(apply.WindowsIn(filepath.Dir(opts.state)))), Revisions: func(ctx context.Context, key string) ([]agent.Update, error) { got, err := session.KVHistory(ctx, composedBucket, key) if err != nil { @@ -495,12 +516,15 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess } return out, nil }, - Trusted: trustedKeys(opts, signer), + Trusted: trustedKeys(opts, trust.Controller), Backup: backupNode(session), // What the filling refused before this apply may be named now: the provisioner's client is told to fill // again (ace, 2026-10-25 21:17). - Settled: func(module string) { knockFilling(module, say) }, - Say: say, + Settled: func(module string) { knockFilling(module+" settled", say) }, + // And once the haves are read at the start, and whenever a module is taken as running without an apply, so + // a refusal from the engine's start does not wait for some module to settle (issue 924). + RunsChanged: func(why string) { knockFilling(why, say) }, + Say: say, // A failure of the bus not answering for a moment is tried again: 2 s, 5 s, 15 s, 60 s, then every 5 min. Backoff: link.Backoff, Apply: func(ctx context.Context, c delivery.Composed, others []delivery.Composed) agent.Result { @@ -512,7 +536,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess if herr != nil { res = failedWith(herr.Error()) } else { - res = applyComposition(actx, opts, held, others, sched, say) + res = applyComposition(actx, opts, held, others, ag.Placed(scopeName(c)), ag.RemovedByWant(), sched, say) } done() // The mesh's own certificate it carries is kept for this machine's record (ADR 0555 §3). @@ -531,7 +555,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess }, Remove: func(ctx context.Context, module string, others []delivery.Composed) agent.Result { actx, done := applyContext(session.WithLink(ctx)) - res := removeComposition(actx, opts, module, others, sched, say) + res := removeComposition(actx, opts, module, others, ag.Placed(module), sched, say) done() afterApply() sayFacts(ctx, ag) @@ -549,7 +573,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess hooks := composerHooks{ running: func() []delivery.Composed { if now := own.get(); now != nil { - return now.Everything() + return runsAsWritten(now.Everything(), opts.state) } return nil }, @@ -572,11 +596,17 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess now.NotComposedEach(run, why, held) // a module held alone marks that module, never the others (issue 1126) } }, + // And the vault's authority beside the controller's, and this node's certificate asked of the vault when a + // module here asks for one (ADR 0550 §2). + vaultAuthority: trust.VaultAuthority, + certify: func(ctx context.Context) error { + return certifyThisNode(ctx, opts.state, node, trust, session.AskVault, func() { sayFacts(ctx, ag) }) + }, } // And its answer to every placement candidate, placed or not, applying nothing (novox/hq ADR 0559 §4): written into // its own key, `.node`, under candidates. shadowHooks := hooks - shadowHooks.refused = nil + shadowHooks.refused, shadowHooks.certify = nil, nil shadowEnv := composerEnv(session, node, say, shadowHooks) // And, after each answer, what runs of an own secret the answer says the vault holds another value of, handed to // the vault while the node applies the controller's compositions (issue 1126, cause C4; ownresync). @@ -596,8 +626,10 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess continue } if composer != nil { + machinesNow.Store(composer) // the machines a failing network part points at, by the placements (issue 774) composes.set(delivery.ComposesOwn) env.Composed, env.SelfComposed, env.Trusted, env.Policy = composer, true, nil, composer.Policy + env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does session.NoteComposes(linked, delivery.ComposesOwn, why) // And once the catalogue no longer places it, the agent begins again on the controller's compositions: the // placement key deleted ends this run, the way back (issue 1126, the seed's revert). @@ -663,7 +695,13 @@ func (c *agentNow) get() *agent.Agent { } // applyComposition applies one composition: a module's, or the node's own. -func applyComposition(ctx context.Context, opts options, c delivery.Composed, others []delivery.Composed, +// +// placed is the modules a want still places here besides this one: none of their resources is removed by this apply, +// which is refused instead (issue 774). +// +// removed is the modules whose want says removed: in the node's own apply, a resource of any other module goes only when +// its module is among them (issue 774). +func applyComposition(ctx context.Context, opts options, c delivery.Composed, others []delivery.Composed, placed, removed map[string]bool, sched *apply.Scheduler, say link.Announce) agent.Result { name := scopeName(c) if c.Module == "" { @@ -679,7 +717,13 @@ func applyComposition(ctx context.Context, opts options, c delivery.Composed, ot if err := refuseEmptyWrites(d); err != nil { return failedWith(err.Error()) } - scope := apply.Scope{Module: name, Others: resourcesOf(others)} + scope := apply.Scope{Module: name, Others: resourcesOf(others), Placed: placed} + if name == agent.NodeModule { + scope.RemovedByWant = removed + if scope.RemovedByWant == nil { + scope.RemovedByWant = map[string]bool{} + } + } got := applyModuleHeld(ctx, opts, d, &scope, sched, say) if got.Refused != "" { return failedWith(got.Refused) @@ -698,9 +742,9 @@ func applyComposition(ctx context.Context, opts options, c delivery.Composed, ot } // removeComposition removes one module's own resources and nothing of another's. -func removeComposition(ctx context.Context, opts options, module string, others []delivery.Composed, +func removeComposition(ctx context.Context, opts options, module string, others []delivery.Composed, placed map[string]bool, sched *apply.Scheduler, say link.Announce) agent.Result { - scope := apply.Scope{Module: module, Others: resourcesOf(others)} + scope := apply.Scope{Module: module, Others: resourcesOf(others), Placed: placed} got := applyModuleHeld(ctx, opts, &declaration.Declaration{Version: declaration.Version}, &scope, sched, say) switch { case got.Refused != "": @@ -725,8 +769,23 @@ func busyModules() []string { return out } -// keptDeclaration is the declaration the node kept before the switch, verified with the mesh's key when it is given. +// machinesNow is the composer whose placements name the mesh's machines by address, for the health report's targets +// (writeHealthReport); nil on a node that does not compose its own, whose parts then name no target. +var machinesNow atomic.Pointer[selfcompose.Composer] + +// keptRetired says the kept declaration is read no more: every module the seat's placement puts on this node has a have of +// its own (placedAllHave), so what runs is the agent's alone (ADR 0550 §2, design 47's L3 s9). +var keptRetired atomic.Bool + +// errKeptRetired is keptDeclaration's answer once the read is retired. +var errKeptRetired = errors.New("the kept declaration is read no more: every module placed on this node has a have of its own") + +// keptDeclaration is the declaration the node kept before the switch, verified with the mesh's key when it is given; +// once the read is retired, nothing is read. func keptDeclaration(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) { + if keptRetired.Load() { + return nil, errKeptRetired + } var raw []byte if len(signer) > 0 { var err error @@ -903,6 +962,9 @@ type composerHooks struct { unseal func(string) ([]byte, error) refused func(why string, held map[string]string) held func(module string) (delivery.Seals, []string, error) + // vaultAuthority and certify are selfcompose.Env's VaultAuthority and Certify. + vaultAuthority func() string + certify func(ctx context.Context) error } func selfComposer(ctx context.Context, session *link.Session, node string, say func(string), hooks composerHooks) (*selfcompose.Composer, string) { @@ -966,7 +1028,8 @@ func composerEnv(session *link.Session, node string, say func(string), hooks com Hear: session.Hear, // And a moved module's templates archive, in the artifact store, as every archive is fetched (issue 1126). Fetch: apply.Fetch, - Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held} + Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held, + VaultAuthority: hooks.vaultAuthority, Certify: hooks.certify} } // endWhenUnplaced calls stop once this node's placement key is deleted from the seat's placement (after the watch diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index b214352..852447b 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -159,7 +159,8 @@ func (b *itBed) start(trustOperator bool) (stop func()) { done := make(chan struct{}) go func() { defer close(done) - nodeAgent(ctx, b.opts, b.controll.Public().(ed25519.PublicKey), b.session, nil, b.say, func() {}, + controller := b.controll.Public().(ed25519.PublicKey) + nodeAgent(ctx, b.opts, controller, newMeshTrust(controller), b.session, nil, b.say, func() {}, &agentNow{}, func(context.Context, *agent.Agent) {}) }() return func() { cancel(); <-done } diff --git a/cmd/mesh-host/provision_secrets.go b/cmd/mesh-host/provision_secrets.go new file mode 100644 index 0000000..a99fca0 --- /dev/null +++ b/cmd/mesh-host/provision_secrets.go @@ -0,0 +1,168 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/provisions" +) + +// provisionSecrets fills a provision's value into a file the provisioner's client hands the node-engine to write +// (novox/hq ADR 0570 clause 6): the client fills the file but for each `${secret:}`, and the node-engine +// fills those at the write, so the client never holds the value. A whole `secrets` file of a handed consumer is the +// placeholder alone. +// +// A placeholder names the credential as the module calls it, `${secret:}` (catalogue.SecretLocal, ADR 0094): the +// local name where the module keeps several values of one provision, else the provision. For each, it finds the one +// provision key `..[.]` of the seat mesh-provisioner's state `provisions` (clause 1) +// going by that name, which must be `provided` and carry the vault's reference; asks the vault for that pair +// credential sealed to this machine's key (the verb `own`, ADR 0349 §11 with the node-engine standing for the client, +// ADR 0550 §5); and opens it with this machine's sealing key. Anything short of that refuses the write, naming the +// provision and why: a file written with a placeholder left in it, or with no value, is a module that starts and cannot +// connect. +type provisionSecrets struct { + node string + // list is every key of mesh-provisioner's `provisions` under prefix (`..`), by key: what this + // engine may read, its own node's (`.>`). + list func(ctx context.Context, prefix string) (map[string][]byte, error) + // own asks the vault's verb `own` in this engine's name. + own func(ctx context.Context, args map[string]any) (json.RawMessage, error) + // open opens a value sealed to this machine. + open func(sealed string) ([]byte, error) +} + +// secretPlaceholder is `${secret:}`. +var secretPlaceholder = regexp.MustCompile(`\$\{secret:([A-Za-z0-9_-]+)\}`) + +// fill is content with each provision placeholder of module's filled, and whether any was. +func (p provisionSecrets) fill(ctx context.Context, module string, content []byte) ([]byte, bool, error) { + found := secretPlaceholder.FindAllSubmatch(content, -1) + if len(found) == 0 { + return content, false, nil + } + if module == "" { + return nil, false, fmt.Errorf("the file names %d provision secret(s) and belongs to no module, so whose they are is not known", len(found)) + } + if p.list == nil || p.own == nil || p.open == nil { + return nil, false, fmt.Errorf("%s's provisions cannot be read on this machine: the node-engine is not linked to the mesh", module) + } + prefix := p.node + "." + module + "." + keys, err := p.list(ctx, prefix) + if err != nil { + return nil, false, fmt.Errorf("%s's provisions cannot be read: %w", module, err) + } + out := string(content) + done := map[string]bool{} + for _, f := range found { + whole, name := string(f[0]), string(f[1]) + if done[whole] { + continue + } + done[whole] = true + pr, key, err := byName(keys, name) + if err != nil { + return nil, false, fmt.Errorf("%s's secret %q: %w", module, name, err) + } + value, err := p.value(ctx, key, pr) + if err != nil { + return nil, false, err + } + out = strings.ReplaceAll(out, whole, string(value)) + } + return []byte(out), true, nil +} + +// byName is the one provision of keys the module calls name (catalogue.SecretLocal), and its key. +func byName(keys map[string][]byte, name string) (provisions.Provision, string, error) { + var hit []string + var got provisions.Provision + for key, raw := range keys { + pr, err := provisions.Decode(raw) + if err != nil { + return provisions.Provision{}, "", fmt.Errorf("the provision %s: %w", key, err) + } + if catalogue.SecretLocal(pr.Provision, pr.Local) == name { + hit, got = append(hit, key), pr + } + } + switch len(hit) { + case 0: + return provisions.Provision{}, "", fmt.Errorf("no provision of %s goes by that name: nothing asks for it", provisions.ProvisionsRead) + case 1: + return got, hit[0], nil + } + sort.Strings(hit) + return provisions.Provision{}, "", fmt.Errorf("the provisions %s all go by that name", strings.Join(hit, ", ")) +} + +// value is one provision's value on this node, opened. +func (p provisionSecrets) value(ctx context.Context, key string, pr provisions.Provision) ([]byte, error) { + if pr.State != provisions.Provided || pr.Reference == nil { + why := pr.State + if pr.Why != "" { + why += ": " + pr.Why + } + if pr.State == provisions.Provided { + why = "provided with no reference to the vault's value" + } + return nil, fmt.Errorf("the provision %s is %s, so its value is not written yet", key, why) + } + // The reference names a credential of this provision, as the vault names it (provisions.VaultName). + ours := false + for _, form := range []string{provisions.FormPair, provisions.FormShared, provisions.FormIssued} { + ours = ours || provisions.VaultName(form, pr) == pr.Reference.ID + } + if !ours { + return nil, fmt.Errorf("the provision %s names the vault's %s, which is no credential of it", key, pr.Reference.ID) + } + module, provision, local := pr.Consumer.Module, pr.Provision, pr.Local + type asked struct { + Role string `json:"role"` + Provision string `json:"provision"` + Consumer string `json:"consumer"` + ConsumerModule string `json:"consumer_module"` + Provider string `json:"provider"` + Local string `json:"local,omitempty"` + } + ask := asked{Role: "consumer", Provision: provision, Consumer: p.node, ConsumerModule: module, Provider: pr.Provider.Node, Local: local} + answer, err := p.own(ctx, map[string]any{"credentials": []asked{ask}}) + if err != nil { + return nil, fmt.Errorf("the vault's value of %s: %w", key, err) + } + var a struct { + Node string `json:"node"` + Credentials []struct { + asked + Name string `json:"name"` + Generation int `json:"generation"` + Sealed json.RawMessage `json:"sealed"` + } `json:"credentials"` + } + if err := json.Unmarshal(answer, &a); err != nil { + return nil, fmt.Errorf("the vault's answer for %s is not one: %w", key, err) + } + if a.Node != p.node { + return nil, fmt.Errorf("the vault answered %s's credentials for %s on %s", a.Node, key, p.node) + } + for _, c := range a.Credentials { + if c.asked != ask { + continue + } + // The value the provision names, by the vault's credential name and its generation (clause 6): another is refused. + if c.Name != pr.Reference.ID || c.Generation != pr.Reference.Generation { + return nil, fmt.Errorf("the vault answered %s generation %d for %s, and the provision names %s generation %d: "+ + "nothing is written until they agree", c.Name, c.Generation, key, pr.Reference.ID, pr.Reference.Generation) + } + value, err := p.open(string(c.Sealed)) + if err != nil { + return nil, fmt.Errorf("the vault's value of %s does not open on this machine: %w", key, err) + } + return value, nil + } + return nil, fmt.Errorf("the vault answered no value of %s for this machine", key) +} diff --git a/cmd/mesh-host/runs_as_written.go b/cmd/mesh-host/runs_as_written.go new file mode 100644 index 0000000..a2bca2a --- /dev/null +++ b/cmd/mesh-host/runs_as_written.go @@ -0,0 +1,88 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + + "git.novox.be/novox/mesh-sdk/go/declaration" + "git.novox.be/novox/mesh-sdk/go/delivery" + + "github.com/novox/mesh-host/internal/store" +) + +// runsAsWritten is what the node runs as the composer reads it (selfcompose.Env.Running), with every file given by digest +// named as this node last wrote it, where the two differ (issue 924, novox 2026-10-29 01:16). What runs of a module the +// agent has not applied since its start can come from the kept declaration, older than the compositions applied since: +// its export (mesh-vault.kept) then named a digest the controller's sweep had removed, while the file on disk was the +// newer export the node wrote. A node composing its own carries such a file as it runs (mesh-sdk frombus keptOf), so it +// names what is on disk: **only when the node's record says it wrote exactly that content there** (store Applied.Wrote, +// Target), never a file someone else changed. A file that cannot be read, or that the record does not vouch for, is +// left as the composition names it. +func runsAsWritten(running []delivery.Composed, statePath string) []delivery.Composed { + if len(running) == 0 || statePath == "" { + return running + } + known, err := store.Load(statePath) + if err != nil { + return running + } + out := make([]delivery.Composed, len(running)) + for i, c := range running { + out[i] = c + var changed []json.RawMessage + for j, raw := range c.Resources { + if fixed, ok := asWritten(raw, known); ok { + if changed == nil { + changed = append([]json.RawMessage(nil), c.Resources...) + } + changed[j] = fixed + } + } + if changed != nil { + out[i].Resources = changed + } + } + return out +} + +// asWritten is the file resource raw named by the digest and size of what this node last wrote at its path, when raw is a +// file given by another digest and the file there is what the record says the node wrote. +func asWritten(raw json.RawMessage, known store.State) (json.RawMessage, bool) { + var fields map[string]json.RawMessage + if json.Unmarshal(raw, &fields) != nil { + return nil, false + } + var kind, id, path, digest string + _ = json.Unmarshal(fields["type"], &kind) + _ = json.Unmarshal(fields["id"], &id) + _ = json.Unmarshal(fields["path"], &path) + _ = json.Unmarshal(fields["digest"], &digest) + if kind != string(declaration.TypeFile) || digest == "" || path == "" { + return nil, false + } + rec, ok := known.Find(id) + if !ok || rec.Wrote == "" || rec.Target != path { + return nil, false + } + body, err := os.ReadFile(path) + if err != nil { + return nil, false + } + sum := sha256.Sum256(body) + if hex.EncodeToString(sum[:]) != rec.Wrote { + return nil, false // changed since the node wrote it: not the node's to vouch for + } + if now := declaration.DigestOf(body); now == digest { + return nil, false + } else { + fields["digest"], _ = json.Marshal(now) + } + fields["size"], _ = json.Marshal(len(body)) + fixed, err := json.Marshal(fields) + if err != nil { + return nil, false + } + return fixed, true +} diff --git a/cmd/mesh-host/trust.go b/cmd/mesh-host/trust.go new file mode 100644 index 0000000..2eadbc6 --- /dev/null +++ b/cmd/mesh-host/trust.go @@ -0,0 +1,317 @@ +package main + +import ( + "context" + "crypto/ed25519" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" + + "github.com/novox/mesh-host/internal/identity" + "github.com/novox/mesh-host/internal/link" +) + +// **What this engine trusts a mesh signature from** (novox/hq ADR 0550 §2; design 47's L3 item d). Two signers: the +// controller's key, which the node was enrolled with (its membership's signer), and the vault's key, read from the vault's +// state `signers`, which only the vault may write: the bus grants are its anchor, as the wants' and the placement's (the +// lead's design limit, 2026-10-28). The controller's key is trusted until its lease is handed away, heard on the lease key +// (buscompose.LeaseHandedAway) and never on a clock; from then the vault's alone. What it trusts now the node states in its +// record (NodeRecord.Trusts), so the gate before the lease write reads every node's from the bus. +type meshTrust struct { + mu sync.Mutex + controller ed25519.PublicKey + handedAway bool + vault ed25519.PublicKey + authority string + // changed is told when what it trusts changed, so the record is written again. + changed func() +} + +// newMeshTrust trusts the controller's key alone until the vault's is read. +func newMeshTrust(controller ed25519.PublicKey) *meshTrust { + t := &meshTrust{} + if len(controller) == ed25519.PublicKeySize { + t.controller = controller + } + return t +} + +// onChange names who is told when what it trusts changed. +func (t *meshTrust) onChange(f func()) { + t.mu.Lock() + t.changed = f + t.mu.Unlock() +} + +// Controller is the controller's key while the controller holds its lease, and nil once the lease is handed away. +func (t *meshTrust) Controller() ed25519.PublicKey { + if t == nil { + return nil + } + t.mu.Lock() + defer t.mu.Unlock() + if t.handedAway { + return nil + } + return t.controller +} + +// HandedAway says the controller's lease was handed away, as heard on the lease key. +func (t *meshTrust) HandedAway() bool { + if t == nil { + return false + } + t.mu.Lock() + defer t.mu.Unlock() + return t.handedAway +} + +// Signers are the keys a mesh signature is verified against now: the controller's while it holds the lease, and the +// vault's once read. +func (t *meshTrust) Signers() []delivery.Signer { + t.mu.Lock() + defer t.mu.Unlock() + var out []delivery.Signer + if t.controller != nil && !t.handedAway { + out = append(out, delivery.Signer{Name: delivery.SignerController, Key: t.controller}) + } + if t.vault != nil { + out = append(out, delivery.Signer{Name: delivery.SignerVault, Key: t.vault}) + } + return out +} + +// Verify is the signer that signed msg as kind, or why none it trusts did. +func (t *meshTrust) Verify(kind string, msg, sig []byte) (delivery.Signer, error) { + context, err := delivery.SignContext(kind) + if err != nil { + return delivery.Signer{}, err + } + return delivery.VerifyBy(t.Signers(), context, msg, sig) +} + +// VaultAuthority is the vault's authority certificate as read from `signers`, or "". +func (t *meshTrust) VaultAuthority() string { + if t == nil { + return "" + } + t.mu.Lock() + defer t.mu.Unlock() + return t.authority +} + +// Trusts is what the record says it trusts: each signer's key and the vault's authority, by fingerprint. +func (t *meshTrust) Trusts() []string { + if t == nil { + return nil + } + authorities := map[string]string{} + if a := t.VaultAuthority(); a != "" { + authorities[delivery.SignerVault] = a + } + return delivery.Trusts(t.Signers(), authorities) +} + +// change applies f under the lock and tells who follows when Trusts differs after it. +func (t *meshTrust) change(f func()) { + before := strings.Join(t.Trusts(), "\n") + t.mu.Lock() + f() + changed := t.changed + t.mu.Unlock() + if strings.Join(t.Trusts(), "\n") != before && changed != nil { + changed() + } +} + +// heardSigner takes one key of the vault's `signers`: its key or its authority, refused (and said) when it does not +// check; a deleted key is no longer trusted. +func (t *meshTrust) heardSigner(key string, value []byte, deleted bool, say link.Announce) { + switch key { + case delivery.SignersKeyKey: + var pub ed25519.PublicKey + if !deleted && len(value) > 0 { + k, err := delivery.DecodeSignerKey(value) + if err == nil { + pub, err = k.PublicKey() + } + if err != nil { + say("the vault's signing key in " + delivery.SignersRead + " is not trusted: " + err.Error()) + return + } + } + t.change(func() { t.vault = pub }) + case delivery.SignersAuthorityKey: + pem := "" + if !deleted && len(value) > 0 { + a, err := delivery.DecodeSignerAuthority(value) + if err != nil { + say("the vault's authority in " + delivery.SignersRead + " is not trusted: " + err.Error()) + return + } + pem = a.Certificate + } + t.change(func() { t.authority = pem }) + } +} + +// heardLease takes the controller's lease key as read: once handed away to a holder that is no controller, the +// controller's key is trusted no more. A lease that cannot be read changes nothing. +func (t *meshTrust) heardLease(raw []byte, found bool, readErr error) { + if readErr != nil { + return + } + handed, _ := buscompose.LeaseHandedAway(raw, found, nil) + t.change(func() { t.handedAway = handed }) +} + +// trustFollowAgain bounds the wait before a watch that could not be made is tried again on the same link: the vault may +// not serve `sign` yet, and until it does this node is not granted the reads. +var trustFollowAgain = 2 * time.Minute + +// followTrust follows the vault's `signers` and the controller's lease on every link, as watches (never a poll), for as +// long as ctx lasts. A watch that cannot be made is said once per reason and tried again. +func followTrust(ctx context.Context, session *link.Session, trust *meshTrust, say link.Announce) { + said := "" + sayOnce := func(line string) { + if line != said { + said = line + if line != "" { + say(line) + } + } + } + for ctx.Err() == nil { + linked, cancel, err := session.WhenLinked(ctx) + if err != nil { + return + } + watching, stop := context.WithCancel(linked) + signers, serr := session.KVWatch(watching, delivery.SignersBucket, ">") + lease, lerr := session.KVWatch(watching, buscompose.LeaseBucket, buscompose.ControllerLeaseKey) + if serr != nil || lerr != nil { + stop() + sayOnce(fmt.Sprintf("what this node trusts a mesh signature from is not followed yet (the vault's signers: %v; "+ + "the controller's lease: %v): it trusts what it last read, and looks again in %s", serr, lerr, trustFollowAgain)) + signers, lease = nil, nil + } else { + sayOnce("") + } + for signers != nil || lease != nil { + select { + case u, ok := <-signers: + if !ok { + signers = nil + continue + } + if !u.Caught { + trust.heardSigner(u.Key, u.Value, u.Deleted, say) + } + case u, ok := <-lease: + if !ok { + lease = nil + continue + } + if !u.Caught { + trust.heardLease(u.Value, !u.Deleted && len(u.Value) > 0, nil) + } + } + } + stop() + select { + case <-linked.Done(): + case <-time.After(trustFollowAgain): + } + cancel() + } +} + +// placedAllHave says whether every module the seat mesh-placement places on node has a have of its own, read from the +// bus: then the kept declaration is read no more (keptRetired). A node the seat does not place, or whose placement or haves +// cannot be read, keeps reading it. +func placedAllHave(ctx context.Context, session *link.Session, node string) (bool, string) { + key, err := delivery.PlacementKey(node) + if err != nil { + return false, err.Error() + } + h, err := session.KVHistory(ctx, delivery.PlacementBucket, key) + if err != nil || len(h) == 0 || h[len(h)-1].Deleted { + return false, "its placement is not read" + } + p, err := delivery.DecodePlacement(h[len(h)-1].Value) + if err != nil { + return false, err.Error() + } + return allHave(p, func(module string) (bool, error) { + _, ok, err := session.ReadHaveErr(ctx, module) + return ok, err + }) +} + +// allHave says whether has finds a have for every module p places, and why not. +func allHave(p delivery.Placement, has func(module string) (bool, error)) (bool, string) { + if len(p.Modules) == 0 { + return false, "the seat mesh-placement places no module here" + } + for _, m := range p.Modules { + ok, err := has(m.Module) + if err != nil { + return false, "the have of " + m.Module + " is not read: " + err.Error() + } + if !ok { + return false, m.Module + " has no have yet" + } + } + return true, fmt.Sprintf("each of the %d modules placed here has a have of its own", len(p.Modules)) +} + +// certifyThisNode asks the vault's `sign` for this node's own certificate (kind node-certificate, ADR 0550 §2), checks +// it is for this node's internal name and serving key and signed by the authority the answer names, which must be the +// vault's this node trusts, keeps it where the record reads it, and writes the record again (written). +func certifyThisNode(ctx context.Context, state, node string, trust *meshTrust, + ask func(ctx context.Context, verb string, args any) (json.RawMessage, error), written func()) error { + serving, err := identity.LoadServingKey(identity.ServingKeyPath(state)) + if err != nil { + return err + } + key, err := base64.StdEncoding.DecodeString(serving.Public) + if err != nil || len(key) != ed25519.PublicKeySize { + return fmt.Errorf("this node's serving key is not an ed25519 key") + } + raw, err := ask(ctx, delivery.SignVerb, map[string]any{"kind": delivery.KindNodeCertificate, "node": node}) + if err != nil { + return err + } + var a delivery.SignAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return fmt.Errorf("the vault's answer is not a signing: %w", err) + } + trusted := trust.VaultAuthority() + if trusted == "" { + return fmt.Errorf("this node has read no authority of the vault's to check the certificate against") + } + if err := delivery.CheckNodeCertificate(a.Certificate, []string{trusted}, overlay.InternalName(node), + ed25519.PublicKey(key), time.Now()); err != nil { + return err + } + path := identity.CertificatePath(state) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + if err := os.WriteFile(path, []byte(a.Certificate), 0o644); err != nil { + return err + } + if written != nil { + written() + } + return nil +} diff --git a/cmd/mesh-host/units_test.go b/cmd/mesh-host/units_test.go index 22ce465..bbbcf32 100644 --- a/cmd/mesh-host/units_test.go +++ b/cmd/mesh-host/units_test.go @@ -1,6 +1,9 @@ package main import ( + "errors" + "os" + "path/filepath" "strings" "testing" "time" @@ -9,6 +12,7 @@ import ( "github.com/novox/mesh-host/internal/accounts" "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/liveness" + "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/units" ) @@ -103,6 +107,61 @@ func TestTheStatementSaysAnAccountJudgedForRoot(t *testing.T) { if got["claude-code.agent"] != "never" || got["openrazer.account"] != "" { t.Fatalf("root said: %v", got) } + // 774 (ADR 0563 §6): the node seat's root-free passes only on the engine's own positive verdict at most 15 minutes old + // with execute not served here; a stale, missing or negative verdict, or execute served, is no pass and says why. + t.Run("this machine's root-free", func(t *testing.T) { + agent := accounts.Verdict{Account: accounts.Account{Module: "claude-code", ID: "claude-code.agent", Name: "agent", Root: true}, + State: accounts.Healthy, Since: at} + free := &accounts.Statement{At: at, Accounts: []accounts.Verdict{agent}} + for _, row := range []struct { + name string + st *accounts.Statement + after time.Duration + served bool + free bool + why string + }{ + {name: "a positive verdict", st: free, after: 14 * time.Minute, free: true, why: "the account agent cannot become root"}, + {name: "a stale verdict", st: free, after: 16 * time.Minute, why: "more than 15 minutes ago"}, + {name: "no verdict", st: nil, why: "judged no account yet"}, + {name: "a way to root", st: as, why: "in the group docker, which grants root"}, + {name: "no agent account", st: &accounts.Statement{At: at}, why: "agents run as the operator's account"}, + {name: "execute served", st: free, served: true, why: "the login shell runs any command"}, + } { + v := judgeRootFree("ace", row.st, at.Add(row.after), row.served, "the bus hears execute answered here") + if v.Free != row.free || !strings.Contains(v.Why, row.why) || v.Machine != "ace" { + t.Errorf("%s: %+v", row.name, v) + } + } + // execute is served while the holder's setting file says anything but withhold, or the bus hears it. + dir := t.TempDir() + statePath, conf := filepath.Join(dir, "state.json"), filepath.Join(dir, "tools.conf") + if err := store.Save(statePath, store.State{Resources: []store.Applied{{ID: "zsh.tools-conf", Type: "file", Target: conf}}}); err != nil { + t.Fatal(err) + } + quiet := func() (bool, error) { return false, nil } + for _, row := range []struct { + name, setting string + heard func() (bool, error) + served bool + }{ + {name: "withheld and not heard", setting: "# the mesh's\nexecute=withhold\n", heard: quiet}, + {name: "served", setting: "execute=serve\n", heard: quiet, served: true}, + {name: "withheld and heard", setting: "execute=withhold\n", heard: func() (bool, error) { return true, nil }, served: true}, + {name: "the bus not asked", setting: "execute=withhold\n", heard: func() (bool, error) { return false, errors.New("no link") }, served: true}, + {name: "no setting file", heard: quiet, served: true}, + } { + _ = os.Remove(conf) + if row.setting != "" { + if err := os.WriteFile(conf, []byte(row.setting), 0o644); err != nil { + t.Fatal(err) + } + } + if served, why := executeServedHere(statePath, row.heard); served != row.served { + t.Errorf("%s: served %v (%s)", row.name, served, why) + } + } + }) } // A verdict waiting for the setuid search says how far it got, in the reason the root-free verb repeats and as diff --git a/cmd/mesh-host/whatruns_test.go b/cmd/mesh-host/whatruns_test.go index 2ffd7c6..1d4601e 100644 --- a/cmd/mesh-host/whatruns_test.go +++ b/cmd/mesh-host/whatruns_test.go @@ -4,6 +4,7 @@ import ( "crypto/ed25519" "encoding/json" "errors" + "path/filepath" "strings" "testing" @@ -11,6 +12,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-host/internal/agent" + "github.com/novox/mesh-host/internal/store" ) func aFileOf(module, id string) json.RawMessage { @@ -59,6 +61,41 @@ func TestDeclaredNowReadsWhatRunsWhenTheAgentIsRunning(t *testing.T) { if err != nil || d.Provisioner != "from-agent" { t.Fatalf("declaredNow read %+v (%v), not what runs", d, err) } + + // L3 s9 (ADR 0550 §2): a node whose placed modules all have a have reads the kept declaration no more, though the + // file is there and readable and the controller's key is dropped; until then it reads it. + state := filepath.Join(t.TempDir(), "state", "state.json") + if err := store.SaveDeclared(store.DeclaredPath(state), store.Declared{Declaration: []byte(`{"declaration":1,` + + `"resources":[{"id":"zsh.rc","type":"file","module":"zsh","path":"/etc/zshrc","content":"x"}]}`)}); err != nil { + t.Fatal(err) + } + if d, err := keptDeclaration(state, nil); err != nil || len(d.Resources) != 1 { + t.Fatalf("before the read is retired: %+v, %v", d, err) + } + placed := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "zsh"}, {Module: "lemurs"}}} + for _, row := range []struct { + name string + has map[string]bool + want bool + }{ + {"every placed module has a have", map[string]bool{"zsh": true, "lemurs": true}, true}, + {"one has none yet", map[string]bool{"zsh": true}, false}, + } { + if got, why := allHave(placed, func(m string) (bool, error) { return row.has[m], nil }); got != row.want { + t.Errorf("%s: %v (%s)", row.name, got, why) + } + } + if got, _ := allHave(delivery.Placement{Node: "n1"}, nil); got { + t.Error("a node with nothing placed retires the read") + } + keptRetired.Store(true) + defer keptRetired.Store(false) + if _, err := keptDeclaration(state, nil); !errors.Is(err, errKeptRetired) { + t.Errorf("after the read is retired: %v", err) + } + if by, filled := keptOnce(options{state: state}, nil)(); by != nil || filled != nil { + t.Errorf("after the read is retired the kept declaration was split: %v %v", by, filled) + } } // 1945/AC17 (unit): a resource the node's own composition shares with a running module's composition is declared once, diff --git a/go.mod b/go.mod index fe6f90a..3cc34da 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index 9dde7f0..a2b8e01 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/internal/accounts/setuid_cache.go b/internal/accounts/setuid_cache.go index 777bbf5..eadd8a0 100644 --- a/internal/accounts/setuid_cache.go +++ b/internal/accounts/setuid_cache.go @@ -240,6 +240,17 @@ func (c *SetuidCache) SearchAgain() { c.forgetProgress() } +// Running says whether a search walks now (novox/hq ADR 0563 §2: the node seat starts no second while one does). +// Nil c runs none. +func (c *SetuidCache) Running() bool { + if c == nil { + return false + } + c.mu.Lock() + defer c.mu.Unlock() + return c.running +} + // SearchNow throws away the last complete search, kept or in memory, and any walk under way or kept, and has // the next look start a full search at once, past AgainAfter and any back-off (novox/hq issue 361): the // operator changed by hand what the last one found, which no apply says. Until the new one completes the diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 5bf7222..7b6494c 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -109,7 +109,9 @@ type Env struct { Kept func() map[string][]json.RawMessage // KeptFilled is the paths the kept declaration names as filled by the provisioner, by the module whose file is at // each (the node's own under NodeModule). A module that still runs from the kept declaration keeps its filled files - // filled: without them the filling refused each one as "not named as filled" (issue 1945). Nil names none. + // filled: without them the filling refused each one as "not named as filled" (issue 1945). Nil names none. A module + // whose have names a composition the bucket keeps runs from that composition, never from the kept declaration + // (issue 924): the kept declaration is older than every composition applied since the switch. KeptFilled func() map[string][]string // Trusted is the public keys a composition may be signed by: the controller's and the operator's (ADR 0539 §3). A // composition unsigned, signed by another key, or changed after it was signed is refused and nothing of it is @@ -131,6 +133,12 @@ type Env struct { // them. The node-engine's filling tells the provisioner's client then to fill again what it refused (mesh-host // internal/filling Knock; ace, 2026-10-25 21:17). Nil tells nothing. Settled func(module string) + // RunsChanged is told, outside the agent's lock, that what runs (Everything) changed other than by an apply or a + // removal: once at the start, when the haves and the compositions they name are read, and each time a module is + // taken as running from its have or the kept declaration. Like Settled, the filling then tells the provisioner's + // client to fill again what it refused (issue 924, the second meeting: ace, g14 and shanks on 2026-10-29 at 00:09 + // UTC refused power's three new drop-ins from an engine start on, as no module settled after it). Nil tells nothing. + RunsChanged func(why string) // Backoff is how long to wait before the try-th retry (1, 2, …) of an apply that failed transiently. Nil retries a // transient failure only on RetryTransient or a change of either key. Backoff func(try int) time.Duration @@ -143,6 +151,27 @@ type Env struct { // move of a module of policy record that recreates a container or moves its data waits for a person's yes in its // want (ADR 0547 §3). Nil judges no move. Policy func(ctx context.Context, module string) string + // HoldsData says whether a module's manifest declares data (catalogue.Manifest.DataItems), and whether its manifest + // could be read: what a composition's Data is, for what runs of a module from the kept declaration, which carries no + // Data of its own. A composition says Data, so without it a data module from the kept declaration is judged to move + // its data against its first composition after a restart (mongodb and mssql on novox, issue 924). Nil reads none. + HoldsData func(ctx context.Context, module string) (data, known bool) + // Applied keeps every composition the agent applied, by module (the node's own under NodeModule), beside the engine's + // state on the machine: read at the start before anything is watched, so what runs is known before the node composes + // its own for the first time, whatever the bucket and the kept declaration still hold (issue 774: on 2026-10-29 the + // bus node, its kept declaration retired and its own composition in no bucket, composed its own as if its folded + // modules ran nothing and removed the bus server and four held modules). Nil keeps none. + Applied AppliedStore +} + +// AppliedStore is where the engine keeps the compositions it applied (Env.Applied): each module's last, as its bytes. +type AppliedStore interface { + // Load is every composition kept, by module; a store that cannot be read says so and loads none. + Load() (map[string][]byte, error) + // Keep keeps the composition the module was applied from, in place of the one before. + Keep(module string, raw []byte) error + // Drop forgets the module's, once it is removed. + Drop(module string) error } // transientFor is how many transient retries say the phase applying before the have says failed: past them the bus has @@ -160,6 +189,13 @@ type Agent struct { // applied is the composition each module was last applied from: a revision a want named from the key's history is // not the key's latest value, and what it holds is still held out of the others' applies. applied map[string]compNow + // had is, for each module whose have names a composition the bucket keeps (its latest value or a revision), that + // composition, read once at the start (loadHad): what runs of a module the agent has not taken yet, ahead of the kept + // declaration (issue 924). + had map[string]compNow + // keptData is, for each module of the kept declaration whose manifest could be read, whether it declares data + // (Env.HoldsData): the Data of what runs of it from the kept declaration. + keptData map[string]bool // attempt is the last apply of a module that failed or is held, with the revisions it was made from. attempt map[string]attempt checked map[string]bool @@ -218,7 +254,7 @@ func New(env Env) *Agent { if env.Say == nil { env.Say = func(string) {} } - return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{}, + return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{}, had: map[string]compNow{}, keptData: map[string]bool{}, attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, notComposed: map[string]notComposedMark{}, retry: make(chan []string, 8), reconcile: make(chan []string, 64), tries: map[string]int{}, again: make(chan againAsk, 64), transient: make(chan struct{}, 1), stopped: make(chan struct{})} @@ -228,6 +264,9 @@ func New(env Env) *Agent { // before any module's, then each change as it comes. func (a *Agent) Run(ctx context.Context) error { defer close(a.stopped) + // What this engine last applied, before any composition is watched: the node composing its own reads what runs + // (Everything) from its first pass on, and that pass comes before the haves are read below (issue 774). + a.loadApplied(ctx) wants, err := a.env.Wants.Watch(ctx, "want."+a.env.Node+".>") if err != nil { return fmt.Errorf("the agent cannot watch its wants: %w", err) @@ -236,7 +275,7 @@ func (a *Agent) Run(ctx context.Context) error { if err != nil { return fmt.Errorf("the agent cannot watch its compositions: %w", err) } - var wantsCaught, compsCaught bool + var wantsCaught, compsCaught, hadRead bool dirty := map[string]bool{} for { select { @@ -303,7 +342,17 @@ func (a *Agent) Run(ctx context.Context) error { } // Nothing is applied until what was there at the start is all in: a module's composition may arrive after its // want, and the node's own composition comes before any module's. - if !(wantsCaught && compsCaught) || len(dirty) == 0 { + if !(wantsCaught && compsCaught) { + continue + } + if !hadRead { + // Before the first settle: what runs of each module with a have, from the composition it names, so the + // filling judges by it at once and not by the kept declaration until some module settles (issue 924). + hadRead = true + a.loadHad(ctx) + a.runsChanged("the haves were read at the start") + } + if len(dirty) == 0 { continue } a.settle(ctx, dirty) @@ -726,7 +775,12 @@ func (a *Agent) revisionNamed(ctx context.Context, module, fp, digest string) (c if err != nil { continue } - dig, err := delivery.VerifyComposed(c, trusted) + var dig string + if a.env.SelfComposed { + dig, err = delivery.ComposedDigestOf(c) // the node's own: unsigned (ADR 0547 §2) + } else { + dig, err = delivery.VerifyComposed(c, trusted) + } if err != nil || dig != digest { continue } @@ -755,22 +809,33 @@ func (a *Agent) others(except string) []delivery.Composed { out = append(out, c.C) } else if c, ok := a.applied[m]; ok && c.Dig == h.Composed { out = append(out, c.C) + } else if c, ok := a.had[m]; ok && c.Dig == h.Composed { + out = append(out, c.C) // a revision its have named, replaced since under the same key } } + hadNames := make([]string, 0, len(a.had)) + for m := range a.had { + if _, held := a.running[m]; !held && m != except { + hadNames = append(hadNames, m) + } + } + sort.Strings(hadNames) + for _, m := range hadNames { + out = append(out, a.had[m].C) + } if a.env.Kept != nil { kept := a.env.Kept() names := make([]string, 0, len(kept)) for m := range kept { - if _, held := a.running[m]; !held && m != except { + _, held := a.running[m] + _, had := a.had[m] + if !held && !had && m != except { names = append(names, m) } } sort.Strings(names) for _, m := range names { - c := delivery.Composed{Node: a.env.Node, Module: m, Resources: kept[m]} - if a.env.KeptFilled != nil { - c.Filled = a.env.KeptFilled()[m] - } + c, _ := a.keptComposed(m, kept[m]) if m == NodeModule { c.Module = "" } @@ -780,6 +845,188 @@ func (a *Agent) others(except string) []delivery.Composed { return out } +// loadHad reads, once at the start, the have of every module with a want or in the kept declaration and, where it names +// a composition, that composition: the bucket's value at its key when its digest is the have's, else the revision the +// bucket keeps with that digest. A have naming none, or one the bucket no longer keeps, leaves the module to the kept +// declaration, said once (issue 924). +func (a *Agent) loadHad(ctx context.Context) { + a.mu.Lock() + seen := map[string]bool{} + for m, w := range a.wants { + if m != NodeModule && w.Present && !w.Want.Removed { + seen[m] = true + } + } + if a.env.Kept != nil { + for m := range a.env.Kept() { + if m != NodeModule { + seen[m] = true + } + } + } + a.mu.Unlock() + names := make([]string, 0, len(seen)) + for m := range seen { + names = append(names, m) + } + sort.Strings(names) + a.readKeptData(ctx, names...) + for _, m := range names { + if ctx.Err() != nil { + return + } + h, ok := a.env.Haves.Read(ctx, m) + if !ok || h.Composed == "" || !haveRuns(h) { + continue + } + a.mu.Lock() + if c, kept := a.had[m]; kept && c.Dig == h.Composed { + a.mu.Unlock() + continue // what this engine applied and kept itself (loadApplied) + } + c, found := a.compositionFor(m, h.FP) + a.mu.Unlock() + if !found || c.Err != nil || c.Dig != h.Composed { + c, found = a.revisionNamed(ctx, m, h.FP, h.Composed) + } + a.mu.Lock() + if found { + a.had[m] = c + } else if _, applied := a.had[m]; applied { + // Kept as this engine applied it (loadApplied), never composed as nothing. + } else if _, kept := a.keptHas(m); kept { + a.sayOnce("had:"+m, fmt.Sprintf("%s's have names the composition %s, which the bucket no longer keeps: it runs "+ + "from the kept declaration until it is applied or taken", m, short(h.Composed))) + } + a.mu.Unlock() + } +} + +// loadApplied takes, once at the start, what this engine last applied of each module as what runs of it, ahead of the +// bucket and the kept declaration (Env.Applied, issue 774): it is what the machine holds, whatever the bus still keeps. +// One whose have names another composition runs as applied until loadHad reads that one from the bucket; one whose +// have cannot be read, or that has none, is kept as applied, so a module is never composed as running nothing. +func (a *Agent) loadApplied(ctx context.Context) { + if a.env.Applied == nil { + return + } + kept, err := a.env.Applied.Load() + if err != nil { + a.env.Say(fmt.Sprintf("the compositions this engine applied cannot be read, so what runs is read from the haves alone: %v", err)) + } + // At an engine's first start nothing is kept yet: the node's own runs as the kept declaration has it, which the store + // keeps now, so the node never composes its own as if its folded modules ran nothing once the declaration retires + // (issue 774, review of the re-land, round 1). Taken whatever digest the node's have names: the composition that names + // is in no bucket, and the declaration is the one record of what runs on disk. + seeded := false + if _, own := kept[NodeModule]; !own && err == nil && a.env.Kept != nil { + if rs, ok := a.env.Kept()[NodeModule]; ok && len(rs) > 0 { + c, _ := a.keptComposed(NodeModule, rs) + c.Module = "" + raw, eerr := delivery.EncodeComposed(c) + if eerr == nil { + eerr = a.env.Applied.Keep(NodeModule, raw) + } + if eerr != nil { + a.env.Say(fmt.Sprintf("the node's own as the kept declaration has it cannot be kept: %v", eerr)) + } else { + if kept == nil { + kept = map[string][]byte{} + } + kept[NodeModule], seeded = raw, true + a.env.Say("the node's own runs as the kept declaration has it, kept now as what this engine runs") + } + } + } + names := make([]string, 0, len(kept)) + for m := range kept { + names = append(names, m) + } + sort.Strings(names) + for _, m := range names { + if ctx.Err() != nil { + return + } + c, err := delivery.DecodeComposed(kept[m]) + var dig string + if err == nil { + dig, err = delivery.ComposedDigestOf(c) + } + if err != nil { + a.env.Say(fmt.Sprintf("the composition this engine kept of %s cannot be read, so what runs of it is read from its have: %v", m, err)) + continue + } + h, found, unread := a.readHave(ctx, m) + if seeded && m == NodeModule { + found, unread = true, false // what the kept declaration has runs, whatever the have names + } else if found && h.Composed != "" && h.Composed != dig && haveRuns(h) { + // The composition the have names replaces this once loadHad finds it in the bucket; until then, and when + // the bucket keeps it no more, what this engine applied runs (issue 774, review of the re-land, round 1). + a.env.Say(fmt.Sprintf("%s's have names the composition %s, not %s this engine applied last: it runs as applied "+ + "until the one its have names is read", m, short(h.Composed), short(dig))) + } + if m == NodeModule { + c.Module, c.FP = "", "" + } + a.mu.Lock() + a.had[m] = compNow{Raw: kept[m], C: c, Dig: dig} + a.mu.Unlock() + switch { + case unread: + a.env.Say(fmt.Sprintf("%s's have cannot be read: it runs as this engine applied it, %s", m, short(dig))) + case !found: + a.env.Say(fmt.Sprintf("%s has no have: it runs as this engine applied it, %s", m, short(dig))) + } + } +} + +// keepApplied keeps the composition a module was applied from (Env.Applied); one that cannot be kept is said, and the +// apply stands. +func (a *Agent) keepApplied(module string, comp compNow) { + if a.env.Applied == nil { + return + } + raw := comp.Raw + if len(raw) == 0 { + var err error + if raw, err = delivery.EncodeComposed(comp.C); err != nil { + a.env.Say(fmt.Sprintf("the composition %s was applied from cannot be kept: %v", module, err)) + return + } + } + if err := a.env.Applied.Keep(module, raw); err != nil { + a.env.Say(fmt.Sprintf("the composition %s was applied from cannot be kept: %v", module, err)) + } +} + +// haveRuns says a have names what runs, in the phases baseline takes as running: running (or empty), held for a failed +// backup, or a composer's mark over what still runs. A have backing up, applying, checking, failed or waiting names a +// composition not applied, or not yet (issue 924, review of 031e106). +func haveRuns(h Have) bool { + switch delivery.Phase(h.Phase) { + case "", delivery.PhaseRunning, delivery.PhaseHeld: + return true + } + return composerMark(h) +} + +// keptHas says whether the kept declaration has resources for the module. Called holding a.mu or not: Kept takes no lock +// of the agent's. +func (a *Agent) keptHas(module string) ([]json.RawMessage, bool) { + if a.env.Kept == nil { + return nil, false + } + rs, ok := a.env.Kept()[module] + return rs, ok +} + +// runsChanged tells RunsChanged, outside the agent's lock. +func (a *Agent) runsChanged(why string) { + if a.env.RunsChanged != nil { + a.env.RunsChanged(why) + } +} + // unreadableModules are the keys of the module compositions in the node's bucket that cannot be read or are refused, of a // module the assigner has not removed: for each module only the composition its want names, else its newest, since a // stale one nobody applies (written before a rule the SDK now holds) must not hold the node's own back. Called holding @@ -901,7 +1148,8 @@ func (a *Agent) heldOut(except string) []delivery.Composed { } } if len(rs) > 0 { - out = append(out, delivery.Composed{Node: a.env.Node, Module: m, Resources: rs}) + // Held out, never judged: its Data from its manifest where known, as every other composition says it. + out = append(out, delivery.Composed{Node: a.env.Node, Module: m, Resources: rs, Data: a.keptData[m]}) } } return out @@ -916,6 +1164,34 @@ func (a *Agent) Everything() []delivery.Composed { return a.others("\x00") } +// Placed is the modules a want still places on the node (present and not removed), held ones included, but except: +// what no apply but the module's own may remove (apply.Scope.Placed, issue 774). Called without a.mu held. +func (a *Agent) Placed(except string) map[string]bool { + a.mu.Lock() + defer a.mu.Unlock() + out := map[string]bool{} + for m, w := range a.wants { + if m != NodeModule && m != except && w.Present && !w.Want.Removed { + out[m] = true + } + } + return out +} + +// RemovedByWant is the modules whose want says removed (apply.Scope.RemovedByWant, issue 774): in the node's own apply, +// only their resources may go. Called without a.mu held. +func (a *Agent) RemovedByWant() map[string]bool { + a.mu.Lock() + defer a.mu.Unlock() + out := map[string]bool{} + for m, w := range a.wants { + if m != NodeModule && w.Present && w.Want.Removed { + out[m] = true + } + } + return out +} + // HoldsNode says whether the agent runs the node's own composition, so the node's parts are its and not the kept // declaration's. func (a *Agent) HoldsNode() bool { @@ -1048,6 +1324,7 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning} } a.mu.Unlock() + a.runsChanged(module + " was taken as held") return true } if h.Composed != "" { @@ -1068,6 +1345,7 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com a.mu.Lock() a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning} a.mu.Unlock() + a.runsChanged(module + " was taken from its have") // Said again, so the session holds the module and its key carries the health this engine judges, // not the one an older engine left there (xorg's unknown on g14 and shanks, 2026-10-25). a.writeHave(ctx, Have{Module: module, FP: h.FP, Composed: h.Composed, @@ -1096,6 +1374,7 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com a.mu.Lock() a.running[module] = delivery.Have{FP: w.Want.FP, Composed: w.Want.Composed, Phase: delivery.PhaseRunning} a.mu.Unlock() + a.runsChanged(module + " was taken from the kept declaration") a.writeHave(ctx, Have{Module: module, FP: w.Want.FP, Composed: w.Want.Composed, Phase: string(delivery.PhaseRunning)}) return true } @@ -1184,6 +1463,7 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo a.running[module] = delivery.Have{FP: fp, Composed: dig, Phase: delivery.PhaseRunning} a.applied[module] = comp a.mu.Unlock() + a.keepApplied(module, comp) if a.env.Settled != nil { a.env.Settled(module) } @@ -1213,8 +1493,14 @@ func (a *Agent) remove(ctx context.Context, module string, w wantNow) { return } delete(a.attempt, module) + delete(a.had, module) a.running[module] = delivery.Have{Phase: delivery.PhaseRemoved} a.mu.Unlock() + if a.env.Applied != nil { + if err := a.env.Applied.Drop(module); err != nil { + a.env.Say(fmt.Sprintf("the composition %s was applied from cannot be forgotten now it is removed: %v", module, err)) + } + } if a.env.Settled != nil { a.env.Settled(module) } @@ -1286,6 +1572,7 @@ func (a *Agent) waitsForYes(ctx context.Context, module string, w wantNow, comp if module == NodeModule || a.env.Policy == nil || a.env.Policy(ctx, module) != catalogue.PolicyRecord || w.Want.Yes == comp.Dig { return false } + a.readKeptData(ctx, module) // the manifest may read now though it did not at the start a.mu.Lock() r, ran := a.running[module] if ran && r.Composed == comp.Dig && r.Phase != delivery.PhaseRemoved { @@ -1331,14 +1618,61 @@ func (a *Agent) waitsForYes(ctx context.Context, module string, w wantNow, comp // runningComposition is what the module runs now: the composition last applied, else its resources in the declaration // the node kept from before the switch. Called with a.mu held. +// +// The composition its have named at the start comes before the kept declaration (issue 924). From the kept declaration, +// what runs carries Data as its manifest declares it; a module whose manifest could not be read cannot be judged against +// it (false), and that is said, rather than taking it to hold no data and judging a false move of its data. func (a *Agent) runningComposition(module string) (delivery.Composed, bool) { if c, ok := a.applied[module]; ok && c.Err == nil { return c.C, true } - if a.env.Kept != nil { - if rs, ok := a.env.Kept()[module]; ok { - return delivery.Composed{Node: a.env.Node, Module: module, Resources: rs}, true + if c, ok := a.had[module]; ok { + return c.C, true + } + if rs, ok := a.keptHas(module); ok { + c, known := a.keptComposed(module, rs) + if !known { + a.sayOnce("kept-data:"+module, fmt.Sprintf("%s runs from the kept declaration, and whether it holds data cannot be "+ + "read from its manifest: a move of it cannot be judged", module)) } + return c, known } return delivery.Composed{}, false } + +// keptComposed is what runs of a module from the kept declaration: its resources there, the paths it names as filled, +// and Data as the module's manifest declares it; false when the manifest could not be read, so Data is not known. Called +// with a.mu held. +func (a *Agent) keptComposed(module string, rs []json.RawMessage) (delivery.Composed, bool) { + c := delivery.Composed{Node: a.env.Node, Module: module, Resources: rs} + if a.env.KeptFilled != nil { + c.Filled = a.env.KeptFilled()[module] + } + data, known := a.keptData[module] + c.Data = data + return c, known || module == NodeModule +} + +// readKeptData reads, outside the agent's lock, whether each module of the kept declaration not known yet declares data +// (Env.HoldsData), and keeps the answers it could read. +func (a *Agent) readKeptData(ctx context.Context, modules ...string) { + if a.env.HoldsData == nil { + return + } + for _, m := range modules { + if m == NodeModule { + continue + } + a.mu.Lock() + _, done := a.keptData[m] + a.mu.Unlock() + if _, kept := a.keptHas(m); done || !kept { + continue + } + if data, known := a.env.HoldsData(ctx, m); known { + a.mu.Lock() + a.keptData[m] = data + a.mu.Unlock() + } + } +} diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 7c256bb..9ec0741 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -769,6 +769,162 @@ func TestEverythingIsWhatTheAgentRunsAndWhatIsKeptForTheRest(t *testing.T) { if a.HoldsNode() { t.Fatal("the agent holds no node composition") } + + // Issue 774 (novox, 2026-10-29 07:54 UTC): the kept declaration retired and the node's own composition in no bucket, a + // fresh engine's first own composition was composed from what runs with the node's own missing, so its folded modules + // (the bus server among them) were composed as nothing and removed. What this engine applied is kept on the machine + // and is what runs from before the first composition is watched; an apply keeps, a removal forgets. + t.Run("what this engine applied is what runs before the node composes its own", func(t *testing.T) { + b := newBed(t) + b.env.SelfComposed, b.env.Trusted, b.kept = true, nil, nil + store := &memApplied{kept: map[string][]byte{}} + b.env.Applied = store + // A folded module's resource in the node's own composition: its id names the module, as the controller folds it. + natsConf, _ := json.Marshal(map[string]any{"id": "nats.conf", "type": "file", "path": "/tmp/r6/nats/conf", "content": "listen 4222"}) + own := delivery.Composed{Node: node, Resources: []json.RawMessage{natsConf}} + ownRaw, err := delivery.EncodeComposed(own) + if err != nil { + t.Fatal(err) + } + ownDig, _ := delivery.ComposedDigestOf(own) + web := delivery.Composed{Node: node, Module: "web", FP: fpOf("a"), Resources: []json.RawMessage{file("web", "x", "1")}} + webRaw, _ := delivery.EncodeComposed(web) + webDig, _ := delivery.ComposedDigestOf(web) + store.kept[NodeModule], store.kept["web"] = ownRaw, webRaw + b.haves.Write(context.Background(), Have{Module: NodeModule, Composed: ownDig}) + b.haves.Write(context.Background(), Have{Module: "web", FP: fpOf("a"), Composed: webDig, Phase: string(delivery.PhaseRunning)}) + // The composer's first pass reads what runs when its watch begins, before the agent reads any have. + firstCh := make(chan []delivery.Composed, 1) + comps := b.comps + var ag *Agent + b.env.Composed = watchFunc(func(ctx context.Context, p string) (<-chan Update, error) { + firstCh <- ag.Everything() + return comps.Watch(ctx, p) + }) + ag = New(b.env) + b.ag = ag + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); _ = ag.Run(ctx) }() + defer func() { cancel(); <-done }() + var first []delivery.Composed + select { + case first = <-firstCh: + case <-time.After(5 * time.Second): + t.Fatal("the composer was not watched") + } + var ranOwn, ranWeb bool + for _, c := range first { + ranOwn = ranOwn || (c.Module == "" && len(c.Resources) == 1 && strings.Contains(string(c.Resources[0]), "4222")) + ranWeb = ranWeb || c.Module == "web" + } + if !ranOwn || !ranWeb { + t.Fatalf("what runs at the node's first own composition: own %v, web %v (%d compositions); want both as this engine applied them", + ranOwn, ranWeb, len(first)) + } + // An apply keeps the composition it applied; a removal forgets it. + web2 := delivery.Composed{Node: node, Module: "web", FP: fpOf("b"), Resources: []json.RawMessage{file("web", "x", "2")}} + ck, _ := delivery.ComposedKey(node, "web", fpOf("b")) + raw2, _ := delivery.EncodeComposed(web2) + comps.put(ck, raw2) + wk, _ := delivery.WantKey(node, "web") + wraw, _ := delivery.EncodeWant(delivery.Want{FP: fpOf("b"), Self: true}) + b.wants.put(wk, wraw) + eventually(t, "the applied composition was not kept", func() bool { return string(store.get("web")) == string(raw2) }) + rraw, _ := delivery.EncodeWant(delivery.Want{Removed: true}) + b.wants.put(wk, rraw) + eventually(t, "the removed module's composition was not forgotten", func() bool { return store.get("web") == nil }) + }) + + // Issue 774, review of the re-land, round 1: at the re-landed engine's first start nothing is kept yet, and the node's + // have names a composition the node composed itself, in no bucket. The node's own runs as the kept declaration has it + // and is kept then, so the next start, the declaration retired, still runs nats. + t.Run("at the first start nothing is kept: the node's own runs as the kept declaration has it", func(t *testing.T) { + natsConf, _ := json.Marshal(map[string]any{"id": "nats.conf", "type": "file", "path": "/tmp/r6/nats/conf", "content": "listen 4222"}) + store := &memApplied{kept: map[string][]byte{}} + h := &haves{} + h.Write(context.Background(), Have{Module: NodeModule, Composed: "sha256:" + strings.Repeat("c", 64)}) + firstPass := func(kept map[string][]json.RawMessage) []delivery.Composed { + b := newBed(t) + b.env.SelfComposed, b.env.Trusted, b.kept, b.env.Applied, b.env.Haves = true, nil, kept, store, h + firstCh := make(chan []delivery.Composed, 1) + comps := b.comps + var ag *Agent + b.env.Composed = watchFunc(func(ctx context.Context, p string) (<-chan Update, error) { + firstCh <- ag.Everything() + return comps.Watch(ctx, p) + }) + ag = New(b.env) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); _ = ag.Run(ctx) }() + defer func() { cancel(); <-done }() + select { + case got := <-firstCh: + return got + case <-time.After(5 * time.Second): + t.Fatal("the composer was not watched") + } + return nil + } + runsNats := func(cs []delivery.Composed) bool { + for _, c := range cs { + if c.Module == "" && len(c.Resources) == 1 && strings.Contains(string(c.Resources[0]), "4222") { + return true + } + } + return false + } + if got := firstPass(map[string][]json.RawMessage{NodeModule: {natsConf}}); !runsNats(got) { + t.Fatalf("the first start does not run nats in the node's own (%d compositions)", len(got)) + } + if store.get(NodeModule) == nil { + t.Fatal("the first start kept no node's own composition") + } + if got := firstPass(nil); !runsNats(got) { + t.Fatalf("the next start, the kept declaration retired, does not run nats in the node's own (%d compositions)", len(got)) + } + }) +} + +// watchFunc is a Watcher from a function. +type watchFunc func(ctx context.Context, pattern string) (<-chan Update, error) + +func (f watchFunc) Watch(ctx context.Context, pattern string) (<-chan Update, error) { + return f(ctx, pattern) +} + +// memApplied is an AppliedStore in memory. +type memApplied struct { + mu sync.Mutex + kept map[string][]byte +} + +func (m *memApplied) Load() (map[string][]byte, error) { + m.mu.Lock() + defer m.mu.Unlock() + out := map[string][]byte{} + for k, v := range m.kept { + out[k] = v + } + return out, nil +} +func (m *memApplied) Keep(module string, raw []byte) error { + m.mu.Lock() + defer m.mu.Unlock() + m.kept[module] = raw + return nil +} +func (m *memApplied) Drop(module string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.kept, module) + return nil +} +func (m *memApplied) get(module string) []byte { + m.mu.Lock() + defer m.mu.Unlock() + return m.kept[module] } // A failed apply is applied again after a restart even when the kept declaration holds the composition: once this @@ -1026,18 +1182,102 @@ func TestTheRevisionAWantNamesIsAppliedFromHistory(t *testing.T) { } } -// 1945/AC18 (unit): a module that still runs from the kept declaration carries the kept declaration's filled paths of -// its files, so the filling writes them ("not named as filled" for claude-code and screen-lock, 2026-10-25). -func TestAModuleRunningFromTheKeptDeclarationKeepsItsFilledPaths(t *testing.T) { - b := newBed(t) - b.kept = map[string][]json.RawMessage{"claude-code": {file("claude-code", "settings", "")}} - b.env.KeptFilled = func() map[string][]string { - return map[string][]string{"claude-code": {"/tmp/r6/claude-code/settings"}} +// 1945/AC18 and issue 924 (unit): what runs of a module the agent has not taken yet carries the filled paths the +// filling judges by. From the kept declaration while the module has no have naming a composition ("not named as filled" +// for claude-code and screen-lock, 2026-10-25); from the composition its have names once it has one, the kept +// declaration being older, read at the start before any module settles and told to the filling then, so a file the +// newer composition fills is accepted on the first fill (power's three drop-ins on ace, g14 and shanks, 2026-10-29). +// Only a have in a phase baseline takes as running names what runs; and what runs carries Data, from the composition +// or, from the kept declaration, from the module's manifest, so a data module is not judged to move its data against +// its first composition after a restart (mongodb and mssql on novox). +func TestWhatRunsBeforeTheAgentTakesAModuleNamesItsFilledPaths(t *testing.T) { + const old, added = "/tmp/r6/power/settings", "/tmp/r6/power/bound" + filledFile := func(id string) json.RawMessage { + return json.RawMessage(`{"id":"power.` + id + `","type":"file","module":"power","path":"/tmp/r6/power/` + id + + `","mode":"0644","owner-at-runtime":"owned"}`) } - a := New(b.env) - got := a.Everything() - if len(got) != 1 || got[0].Module != "claude-code" || strings.Join(got[0].Filled, ",") != "/tmp/r6/claude-code/settings" { - t.Fatalf("what runs from the kept declaration: %+v", got) + fp := fpOf("a") + ck, _ := delivery.ComposedKey(node, "power", fp) + applied := delivery.Composed{Node: node, Module: "power", FP: fp, Filled: []string{old, added}, Data: true, + Resources: []json.RawMessage{filledFile("settings"), filledFile("bound")}} + _, appliedRaw, appliedDig := signedBy(t, applied, ck, controllerKey) + newer := applied + newer.Resources = append([]json.RawMessage{}, applied.Resources...) + newer.Resources = append(newer.Resources, file("power", "later", "1")) + _, newerRaw, _ := signedBy(t, newer, ck, controllerKey) + for _, tc := range []struct { + name string + // have is the module's have; bucket the value its composition key holds now, history the revisions kept. + have *Have + bucket []byte + history []Update + filled, told string + // data is what runs' Data; holdsData the manifest's answer (nil: none read). + data bool + holdsData func(context.Context, string) (bool, bool) + }{ + {name: "no have: the kept declaration's", filled: old}, + {name: "a have naming the bucket's value", have: &Have{Module: "power", FP: fp, Composed: appliedDig}, + bucket: appliedRaw, filled: old + "," + added, told: "the haves were read at the start", data: true}, + {name: "a have applying names nothing that runs yet", have: &Have{Module: "power", FP: fp, Composed: appliedDig, + Phase: string(delivery.PhaseApplying)}, bucket: appliedRaw, filled: old, told: "the haves were read at the start"}, + {name: "a data module from the kept declaration says Data as its manifest does", filled: old, + told: "the haves were read at the start", data: true, + holdsData: func(context.Context, string) (bool, bool) { return true, true }}, + {name: "a have naming a revision replaced since", have: &Have{Module: "power", FP: fp, Composed: appliedDig}, + bucket: newerRaw, history: []Update{{Key: ck, Value: appliedRaw, Revision: 3}, {Key: ck, Value: newerRaw, Revision: 7}}, + filled: old + "," + added, told: "the haves were read at the start", data: true}, + {name: "a have naming a composition the bucket no longer keeps", have: &Have{Module: "power", FP: fp, Composed: appliedDig}, + bucket: newerRaw, filled: old, told: "the haves were read at the start"}, + } { + t.Run(tc.name, func(t *testing.T) { + b := newBed(t) + b.kept = map[string][]json.RawMessage{"power": {filledFile("settings")}} + b.env.KeptFilled = func() map[string][]string { return map[string][]string{"power": {old}} } + if tc.have != nil { + b.haves.stored = map[string]Have{"power": *tc.have} + } + if tc.bucket != nil { + b.comps.before(ck, tc.bucket) + } + b.env.Revisions = func(_ context.Context, key string) ([]Update, error) { return tc.history, nil } + b.env.HoldsData = tc.holdsData + var told []string + var mu sync.Mutex + b.env.RunsChanged = func(why string) { mu.Lock(); told = append(told, why); mu.Unlock() } + filledNow := func() string { + for _, c := range b.ag.Everything() { + if c.Module == "power" { + if c.Data != tc.data { + t.Errorf("what runs says Data %v, want %v", c.Data, tc.data) + } + return strings.Join(c.Filled, ",") + } + } + return "" + } + if tc.told == "" { // no want, no have: nothing to start, and the kept declaration is what runs + a := New(b.env) + b.ag = a + if got := filledNow(); got != tc.filled { + t.Fatalf("filled %q, want %q", got, tc.filled) + } + return + } + stop := b.run() + defer stop() + eventually(t, "the filling was not told what runs at the start", func() bool { + mu.Lock() + defer mu.Unlock() + return len(told) > 0 && told[0] == tc.told + }) + if got := filledNow(); got != tc.filled { + t.Fatalf("filled %q, want %q", got, tc.filled) + } + if got := b.appliedList(); len(got) != 0 { + t.Fatalf("applied %v: nothing has a want", got) + } + }) } } diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 1c1023c..8eb83d8 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -312,6 +312,38 @@ type Scope struct { Module string // Others are the resources of every other module the node has a composition for. Others []declaration.Resource + // Placed are the modules a want still places on the node (present, not removed), held ones included: no apply but + // the module's own removes a resource of one of them (issue 774: on 2026-10-29 the node's own apply on the bus node, + // composed without what its folded modules ran, removed the bus server and four held modules). Nil guards nothing. + Placed map[string]bool + // RemovedByWant are the modules whose want says removed. In the node's own apply (Module NodeModule) with this set + // non-nil, a resource of any other module is removed only when that module is here: a module the node runs folded + // in its own composition has no want, so Placed does not name it (issue 774, review of the re-land, round 1: nats on + // the bus node has none, and its nats.conf was guessed as nats's and removed). Nil guards by Placed alone. + RemovedByWant map[string]bool +} + +// placedOwner is the module a want still places on the node that the record belongs to, other than the scope's own: +// the module recorded with it, else the one its id names (a resource the node's own composition carried for a folded +// module is recorded as the node's, its id naming the module). Empty when there is none. +func (sc *Scope) placedOwner(a store.Applied) string { + if sc == nil || (len(sc.Placed) == 0 && sc.RemovedByWant == nil) { + return "" + } + m := a.Module + if m == "" || m == store.NodeModule { + m, _ = declaration.ModuleGuess(a.ID) + } + if m == "" || m == sc.Module || m == store.NodeModule { + return "" + } + if sc.Placed[m] { + return m + } + if sc.Module == store.NodeModule && sc.RemovedByWant != nil && !sc.RemovedByWant[m] { + return m // a module the node's own composition carried, which no want removed + } + return "" } // owns reports whether a record belongs to the scope's module. @@ -441,6 +473,31 @@ func applyScoped( } } + // **A module a want still places here is never removed by another's apply** (issue 774): its resources go only with + // its own apply or its want's removal. One that would is refused whole, before anything is changed, naming them: a + // composition that leaves them out lost what runs, and removing what runs is the worst a node can do with that. + if scope != nil { + var refused []string + for _, orphan := range known.Orphans(declared, origin) { + if !scope.owns(orphan) { + continue + } + if _, kept := heldByOthers[orphan.ID]; kept { + continue + } + if m := scope.placedOwner(orphan); m != "" { + refused = append(refused, orphan.ID+" of "+m) + } + } + if len(refused) > 0 { + sort.Strings(refused) + err := fmt.Errorf("the apply of %s would remove %s, of modules a want still places here; a module's resources "+ + "go only with its own apply or its want's removal, so nothing was applied", scope.Module, strings.Join(refused, ", ")) + log(" refused: " + err.Error()) + return report, known, &Error{Resource: "the resources of modules placed here", Err: err, Done: report} + } + } + removeOrphan := func(orphan store.Applied) error { var action, detail string var err error diff --git a/internal/apply/filled.go b/internal/apply/filled.go index 5d4d2ca..6bd9b7a 100644 --- a/internal/apply/filled.go +++ b/internal/apply/filled.go @@ -45,6 +45,30 @@ func WriteFilled( log func(string), keep Keep, windows *Windows, +) (report Report, _ store.State, _ error) { + return WriteFilledWith(ctx, sys, d, known, path, content, run, log, keep, windows, Filled{}) +} + +// Filled is what the node-engine did to a filled file's content before its write. +type Filled struct { + // Secret says the node-engine filled a value into the content, a provision's `${secret:}` (novox/hq ADR 0570 + // clause 6): the file is then written 0600 when the declaration gives no mode, as a file carrying a secret is. + Secret bool +} + +// WriteFilledWith is WriteFilled for content the node-engine filled before the write (Filled). +func WriteFilledWith( + ctx context.Context, + sys system.System, + d *declaration.Declaration, + known store.State, + path string, + content []byte, + run Runner, + log func(string), + keep Keep, + windows *Windows, + filled Filled, ) (report Report, _ store.State, _ error) { if log == nil { log = func(string) {} @@ -65,7 +89,7 @@ func WriteFilled( return report, known, &Error{Resource: path, Err: fmt.Errorf("%d resources declare %s, and a filled file "+ "is written for exactly one, so nothing was written", len(declared), path)} } - file := filledAs(declared[0], content) + file := filledAs(declared[0], content, filled.Secret) if held, isHeld := known.HeldAt(file.ID); isHeld { return report, known, &Error{Resource: file.ID, Err: fmt.Errorf("%s is held as found for a module not "+ "yet taken (%s), so nothing was written", path, held.Why)} @@ -163,10 +187,10 @@ func WriteFilled( } // filledAs is the declared file with the filled content in place of the declaration's. -func filledAs(declared *declaration.File, content []byte) *declaration.File { +func filledAs(declared *declaration.File, content []byte, secret bool) *declaration.File { f := *declared f.Content, f.Sealed, f.VaultSealed, f.Bytes, f.Secrets = string(content), "", nil, "", nil - if f.Mode == "" && (declared.Secret() || len(declared.Secrets) > 0) { + if f.Mode == "" && (secret || declared.Secret() || len(declared.Secrets) > 0) { // Written as the declaration would have written it: a file carrying a credential is not world-readable. f.Mode = "0600" } diff --git a/internal/apply/module_scope_test.go b/internal/apply/module_scope_test.go index 67b494a..51e42a5 100644 --- a/internal/apply/module_scope_test.go +++ b/internal/apply/module_scope_test.go @@ -3,6 +3,7 @@ package apply import ( "context" "path/filepath" + "strings" "testing" "git.novox.be/novox/mesh-sdk/go/declaration" @@ -116,32 +117,78 @@ func TestARecordWithoutAnOwnerBelongsToTheModuleItsIdNames(t *testing.T) { // leaves it as it is, whatever its type, until the module's own apply takes it (issue 1945, live on 2026-10-25 at 02:32: // shanks and g14 lost systemd-resolved's guard, its resolver files and sudo's operator file, ace qbittorrent's server, // because the node's own apply removed every record no longer in it, the others holding the very same resources or not). +// +// And a resource the node's own composition carried for a module a want still places here (a folded module of the old +// form, held) is never removed by the node's own apply when a composition leaves it out: the apply is refused whole and +// says why (issue 774, live on 2026-10-29 at 07:54 UTC on the bus node: the node's own composition, composed without what +// it ran once the kept declaration was read no more, removed the bus server, the proxy and three held modules). Only its +// want's removal lets it go. func TestAResourceThatMovedToItsModulesCompositionIsLeftByTheNodesOwnApply(t *testing.T) { - dir := t.TempDir() - fact := filepath.Join(dir, "suffix") - gone := filepath.Join(dir, "gone.conf") - own := declare(t, `{"id":"systemd-resolved.fact-suffix","type":"file","path":"`+fact+`","content":"mesh\n"}, - {"id":"old.gone","type":"file","path":"`+gone+`","content":"g\n"}`) - _, state, err := ApplyModule(context.Background(), archHost(t), own, Scope{Module: store.NodeModule}, - state0(), store.OriginDeclared, noServices, nil, nil, nil, nil) - if err != nil { - t.Fatal(err) - } - // systemd-resolved now has a composition of its own, not applied yet; old.gone is no module's any more. - moved := declare(t, `{"id":"systemd-resolved.fact-suffix","type":"file","path":"`+fact+`","content":"mesh\n","module":"systemd-resolved"}`) - report, state, err := ApplyModule(context.Background(), archHost(t), &declaration.Declaration{}, - Scope{Module: store.NodeModule, Others: moved.Resources}, state, store.OriginDeclared, noServices, nil, nil, nil, nil) - if err != nil { - t.Fatal(err) - } - if !exists(fact) { - t.Fatalf("the moved resource was removed: %+v", outcomeOf(report, "systemd-resolved.fact-suffix")) - } - if exists(gone) { - t.Fatal("a resource no composition holds any more was kept") - } - if _, still := state.Find("systemd-resolved.fact-suffix"); !still { - t.Fatal("the moved resource's record was dropped, so its module's apply would not know it") + for _, tc := range []struct { + name string + id string // the resource the node's own composition carried, and no longer does + moved bool // its module's own composition now declares it (Others) + placed bool // a want still places its module here + refused bool // the node's own apply is refused whole + stays bool // the resource is still on the machine and recorded + // wants names the modules whose want says removed (Scope.RemovedByWant); nil leaves the guard to Placed alone. + wants []string + }{ + {name: "moved to its module's own composition", id: "systemd-resolved.fact-suffix", moved: true, placed: true, stays: true}, + {name: "a held folded module, in no composition", id: "nats.server-conf", placed: true, refused: true, stays: true}, + {name: "its module's want removed", id: "nats.server-conf"}, + // Issue 774, review of the re-land, round 1: a module folded in the node's own has no want, so Placed does not name + // it; in the node's own apply only a module a want removed loses its resources. + {name: "a folded module with no want", id: "nats.server-conf", wants: []string{}, refused: true, stays: true}, + {name: "a folded module a want removed", id: "nats.server-conf", wants: []string{"nats", "old"}}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + fact := filepath.Join(dir, "carried") + gone := filepath.Join(dir, "gone.conf") + own := declare(t, `{"id":"`+tc.id+`","type":"file","path":"`+fact+`","content":"mesh\n"}, + {"id":"old.gone","type":"file","path":"`+gone+`","content":"g\n"}`) + _, state, err := ApplyModule(context.Background(), archHost(t), own, Scope{Module: store.NodeModule}, + state0(), store.OriginDeclared, noServices, nil, nil, nil, nil) + if err != nil { + t.Fatal(err) + } + module, _ := declaration.ModuleGuess(tc.id) + scope := Scope{Module: store.NodeModule} + if tc.moved { + moved := declare(t, `{"id":"`+tc.id+`","type":"file","path":"`+fact+`","content":"mesh\n","module":"`+module+`"}`) + scope.Others = moved.Resources + } + if tc.placed { + scope.Placed = map[string]bool{module: true} + } + if tc.wants != nil { + scope.RemovedByWant = map[string]bool{} + for _, m := range tc.wants { + scope.RemovedByWant[m] = true + } + } + report, after, err := ApplyModule(context.Background(), archHost(t), &declaration.Declaration{}, scope, state, + store.OriginDeclared, noServices, nil, nil, nil, nil) + if tc.refused != (err != nil) { + t.Fatalf("refused %v, want %v: %v", err != nil, tc.refused, err) + } + if tc.refused { + if !strings.Contains(err.Error(), tc.id+" of "+module) { + t.Fatalf("the refusal does not name what it kept: %v", err) + } + after = state + } + if exists(fact) != tc.stays { + t.Fatalf("the carried resource exists %v, want %v: %+v", exists(fact), tc.stays, outcomeOf(report, tc.id)) + } + if _, still := after.Find(tc.id); still != tc.stays { + t.Fatalf("the carried resource's record kept %v, want %v", still, tc.stays) + } + if exists(gone) == !tc.refused { + t.Fatalf("a resource no composition holds any more exists %v, with the apply refused %v", exists(gone), tc.refused) + } + }) } } diff --git a/internal/apply/objectkeep.go b/internal/apply/objectkeep.go new file mode 100644 index 0000000..d5be5df --- /dev/null +++ b/internal/apply/objectkeep.go @@ -0,0 +1,111 @@ +package apply + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "git.novox.be/novox/mesh-sdk/go/declaration" +) + +// ObjectKeep is the node-engine's own keep of the content of files given by digest (novox/hq ADR 0486 §1, issue 924): one +// file per object, named as the object is, beside the node's state. **What a node once had it never asks the bus for +// again**, so neither the controller's sweep of the bucket (an object is removed a day after no composition of the +// controller names it) nor a bus that does not answer can refuse a composition the node already ran. The node's account +// may only read its bucket (ADR 0486 §2), so the keep is the node's, not the bucket's: nothing is written back. +// +// An object unused for KeptObjectsFor is let go when another is kept, so the keep holds what the node applied lately and +// not every export it ever ran. Every failure to keep is silent and costs only a fetch: the keep is a copy. +type ObjectKeep struct { + Dir string + // Now is the clock; nil is time.Now. + Now func() time.Time +} + +// KeptObjectsFor is how long an object of the keep is held after it was last used. +const KeptObjectsFor = 14 * 24 * time.Hour + +// KeepObjectsIn is the keep at dir; an empty dir keeps nothing. +func KeepObjectsIn(dir string) *ObjectKeep { + if dir == "" { + return nil + } + return &ObjectKeep{Dir: dir} +} + +func (k *ObjectKeep) now() time.Time { + if k.Now != nil { + return k.Now() + } + return time.Now() +} + +// get is the kept content of the object name when it is what f names, and marks it used. +func (k *ObjectKeep) get(name string, f *declaration.File) ([]byte, bool) { + if k == nil { + return nil, false + } + path := filepath.Join(k.Dir, name) + body, err := os.ReadFile(path) + if err != nil || int64(len(body)) != f.Size || f.Verify(body) != nil { + return nil, false + } + now := k.now() + _ = os.Chtimes(path, now, now) + return body, true +} + +// put keeps body under name, unless it is kept already (then it is marked used), and lets go of what was unused too long. +func (k *ObjectKeep) put(name string, body []byte) { + if k == nil { + return + } + path := filepath.Join(k.Dir, name) + now := k.now() + if info, err := os.Stat(path); err == nil && info.Size() == int64(len(body)) { + _ = os.Chtimes(path, now, now) + return + } + if err := os.MkdirAll(k.Dir, 0o700); err != nil { + return + } + tmp, err := os.CreateTemp(k.Dir, ".put-*") + if err != nil { + return + } + _, werr := tmp.Write(body) + cerr := tmp.Close() + if werr != nil || cerr != nil || os.Chmod(tmp.Name(), 0o600) != nil || os.Rename(tmp.Name(), path) != nil { + _ = os.Remove(tmp.Name()) + return + } + _ = os.Chtimes(path, now, now) + k.prune(now) +} + +// prune lets go of every object unused for KeptObjectsFor. +func (k *ObjectKeep) prune(now time.Time) { + entries, err := os.ReadDir(k.Dir) + if err != nil { + return + } + for _, e := range entries { + info, err := e.Info() + if err != nil || !info.Mode().IsRegular() { + continue + } + if now.Sub(info.ModTime()) >= KeptObjectsFor { + _ = os.Remove(filepath.Join(k.Dir, e.Name())) + } + } +} + +// never says, in a failed fetch's error before its cause (a transient one's detail keeps the tail), that the node's own +// keep did not hold the object either. +func (k *ObjectKeep) never() string { + if k == nil { + return "" + } + return fmt.Sprintf(" (this node's own keep of objects, %s, never held it)", k.Dir) +} diff --git a/internal/apply/references.go b/internal/apply/references.go index ad6bed5..64ce30c 100644 --- a/internal/apply/references.go +++ b/internal/apply/references.go @@ -27,13 +27,23 @@ type FetchObject func(ctx context.Context, name string, limit int64) ([]byte, er // unchanged declaration, and a node that is asleep and has no bus, apply as they always did. One object is fetched // once however many files name its digest. func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetch FetchObject) error { + return ResolveFileReferencesKept(ctx, d, fetch, nil) +} + +// ResolveFileReferencesKept is ResolveFileReferences with the node's own keep of objects (ObjectKeep): every content had +// (a file here that verifies, or an object of the bus) is kept, and a kept one is used before the bus is asked. **A node +// depends on its bucket only for content it never had** (issue 924, 2026-10-29: novox's own composition named the +// export mesh-vault runs by a digest whose object the controller's sweep had removed, so the vault's apply failed while +// it ran; the bucket's objects are kept only while a composition of the controller names them, and a node composing its +// own names objects the controller no longer knows of). A nil keep keeps nothing. +func ResolveFileReferencesKept(ctx context.Context, d *declaration.Declaration, fetch FetchObject, keep *ObjectKeep) error { fetched := map[string][]byte{} for i, r := range d.Resources { f, ok := r.(*declaration.File) if !ok || !f.IsReference() { continue } - body, err := referencedContent(ctx, f, fetch, fetched) + body, err := referencedContent(ctx, f, fetch, fetched, keep) if err != nil { return err } @@ -50,31 +60,37 @@ func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetc } func referencedContent(ctx context.Context, f *declaration.File, fetch FetchObject, - fetched map[string][]byte) ([]byte, error) { + fetched map[string][]byte, keep *ObjectKeep) ([]byte, error) { if body, ok := fetched[f.Digest]; ok { return body, nil } - if body, ok := alreadyThere(f); ok { - fetched[f.Digest] = body - return body, nil - } name, err := declaration.ObjectName(f.Digest) if err != nil { return nil, fmt.Errorf("file %s (%s): %w", f.ID, f.Path, err) } + if body, ok := alreadyThere(f); ok { + fetched[f.Digest] = body + keep.put(name, body) + return body, nil + } + if body, ok := keep.get(name, f); ok { + fetched[f.Digest] = body + return body, nil + } if fetch == nil { return nil, fmt.Errorf("file %s (%s) is given by digest %s and is not on this machine, and this node-engine "+ "has no link to the bus to fetch it from", f.ID, f.Path, f.Digest) } body, err := fetch(ctx, name, f.Size) if err != nil { - return nil, fmt.Errorf("file %s (%s): fetching the object %s of this node's own object bucket: %w", - f.ID, f.Path, name, err) + return nil, fmt.Errorf("file %s (%s): fetching the object %s of this node's own object bucket%s: %w", + f.ID, f.Path, name, keep.never(), err) } if err := f.Verify(body); err != nil { return nil, fmt.Errorf("%s (%s): %w; nothing was applied", f.ID, f.Path, err) } fetched[f.Digest] = body + keep.put(name, body) return body, nil } diff --git a/internal/link/access_act_test.go b/internal/link/access_act_test.go index 0bd38f6..82bf596 100644 --- a/internal/link/access_act_test.go +++ b/internal/link/access_act_test.go @@ -6,6 +6,10 @@ import ( "strings" "testing" "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asker" + "git.novox.be/novox/mesh-sdk/go/asks" ) // The take act's ask (novox/hq ADR 0325 §8): a hand-over's fields and the act's own in one signed object, under a @@ -68,6 +72,112 @@ func TestTheTakeActRunsOnlyOnAWarrant(t *testing.T) { if a.Said != "started" || len(given) != 1 || given[0].Warrant != "w-1" || given[0].Path != "/srv/media/movies" { t.Fatalf("a warranted ask did not reach the act whole: %+v %+v", a, given) } + // 774 (ADR 0563 §3): a hand-over asked through the node seat is the engine's own ask, performed only on the + // operator's warrant to it: never on the call, not on a warrant for another digest, once on a warrant heard twice, + // and not when the disk changed since it was asked. + t.Run("the node seat's hand-over acts only on the operator's warrant", func(t *testing.T) { + for _, row := range []struct { + name string + warrant func(asked asks.Ask, w asks.Warrant) asks.Warrant + diskAfter string + twice bool + done int + state string + }{ + {name: "no warrant", done: 0, state: HandAsked}, + {name: "the operator's yes", warrant: yes, done: 1, state: HandDone}, + {name: "the yes heard twice", warrant: yes, twice: true, done: 1, state: HandDone}, + {name: "a warrant for another digest", warrant: func(a asks.Ask, w asks.Warrant) asks.Warrant { + w = yes(a, w) + w.AskDigest = "sha256:another" + return w + }, done: 0, state: HandAsked}, + {name: "a no", warrant: func(a asks.Ask, w asks.Warrant) asks.Warrant { + w = yes(a, w) + w.Option, w.Label, w.Level = "no", "No, leave it", asks.Acknowledge + return w + }, done: 0, state: HandRefused}, + {name: "the disk changed", warrant: yes, diskAfter: "0:0 700 9:2", done: 0, state: HandRefused}, + {name: "expired", warrant: func(a asks.Ask, w asks.Warrant) asks.Warrant { + return asks.Warrant{Ask: a.ID, Asker: w.Asker, Outcome: asks.OutcomeExpired, At: w.At} + }, done: 0, state: HandRefused}, + } { + t.Run(row.name, func(t *testing.T) { + h, asked, done := anEngine(t) + disk := "1000:1000 755 9:1" + h.JudgeHandOver = func(path string) (Judged, error) { + return Judged{Subject: path, Path: path, Sets: "owner jo, group jo, mode 0750", Disk: disk, + Words: "A directory used as found."}, nil + } + if _, err := h.Call(EngineCall{Verb: ActHandOver, Caller: "shanks.node-tools", + Args: map[string]any{"path": "/srv/x", "why": "a test"}}); err != nil { + t.Fatal(err) + } + if len(*asked) != 1 || *done != 0 { + t.Fatalf("the call asked %d and performed %d", len(*asked), *done) + } + // The act is one key, and the open ask one key made by a create (ADR 0563 §3). + if acts, _ := h.State.Keys("act."); len(acts) != 1 || acts[0] != "act."+(*asked)[0].ID { + t.Fatalf("the act is kept as %v", acts) + } + if open, _ := h.State.Keys("open."); len(open) != 1 || !strings.HasPrefix(open[0], "open.hand-over.") { + t.Fatalf("the open ask is kept as %v", open) + } + if _, err := h.Call(EngineCall{Verb: ActHandOver, Caller: "x", Args: map[string]any{"path": "/srv/x"}}); err == nil { + t.Fatal("a second ask of the same path was asked while one is open") + } + if row.diskAfter != "" { + disk = row.diskAfter + } + if row.warrant != nil { + w := row.warrant((*asked)[0], asks.Warrant{Asker: asks.EngineAsker("n1"), At: time.Now()}) + h.Heard(w) + if row.twice { + h.Heard(w) + } + } + got, _ := h.Acts(0) + acts := got.(map[string]any)["acts"].([]HandAct) + if *done != row.done || len(acts) != 1 || acts[0].State != row.state { + t.Fatalf("performed %d (want %d), acts %+v (want %s)", *done, row.done, acts, row.state) + } + if open, _ := h.State.Keys("open."); (len(open) == 1) != (row.state == HandAsked) { + t.Fatalf("an ask %s leaves its open key as %v", row.state, open) + } + if row.state == HandDone && (acts[0].By != "operator" || acts[0].Via == "" || acts[0].Ask == "") { + t.Fatalf("the act does not record the warrant: %+v", acts[0]) + } + }) + } + }) +} + +// yes is the operator's yes to an ask, as the router writes it. +func yes(a asks.Ask, w asks.Warrant) asks.Warrant { + w.Ask, w.Outcome, w.Option, w.Label, w.Level = a.ID, asks.OutcomeChosen, "yes", "Yes", asks.Approve + w.By = &asks.Person{Who: asks.Operator, Kind: "telegram", Verified: "user id verified"} + w.AskDigest, w.Proofs = a.Digest(), []string{"P1"} + return w +} + +// anEngine is a node seat with its own state in a temporary folder, a router that keeps what it was asked, and a +// hand-over that counts what it performed. +func anEngine(t *testing.T) (*HandActs, *[]asks.Ask, *int) { + t.Helper() + var asked []asks.Ask + done := 0 + state := &MemState{} + h := &HandActs{Node: "n1", State: state, + HandOver: func(path, by string) (string, error) { done++; return path + " handed over by " + by, nil }} + h.Asker = &asker.Client{Module: asks.EngineAsker("n1"), State: state, + Publish: func(subject string, body any, id string) error { + if a, ok := body.(asks.Ask); ok && subject == asks.AskSubject(asks.EngineAsker("n1")) { + asked = append(asked, a) + } + return nil + }, + Record: func(string) (*stdio.StateEntry, error) { return nil, nil }} + return h, &asked, &done } // The controller names an access by its declared resource id exactly, which may hold a `/`: it reaches the act, diff --git a/internal/link/engineseat.go b/internal/link/engineseat.go new file mode 100644 index 0000000..258e3f5 --- /dev/null +++ b/internal/link/engineseat.go @@ -0,0 +1,382 @@ +package link + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/micro" +) + +// The node seat node-engine, served on the engine's own link (novox/hq ADR 0563 §1). +// +// Each verb is answered on this node's subject alone, `mesh.seat.node-engine.tool..`, and on the same +// address naming its caller, `mesh.seat.node-engine.call...`, whose last token the bus lets each user +// publish with its own name alone (novox/hq issue 365): the caller is that token, and a call on the subject naming none is +// from an unproven caller, recorded so. The body is the call's arguments; the answer is {result, error, node}, as every +// tool's (mesh-tools node-tools). One service, `node-engine`, answers discovery with each verb's seat, tool and node. + +// EngineSeat is the seat. +const EngineSeat = catalogue.EngineSeat + +// EngineToolSubject is where a verb of this node's seat is asked naming no caller. +func EngineToolSubject(verb, node string) string { + return "mesh.seat." + EngineSeat + ".tool." + verb + "." + node +} + +// EngineCallPattern is where a verb of this node's seat is asked naming its caller, any caller. +func EngineCallPattern(verb, node string) string { + return "mesh.seat." + EngineSeat + ".call." + verb + "." + node + ".*" +} + +// EngineEventSubject is where the seat says one of its events about this node. +func EngineEventSubject(event, node string) string { + return "mesh.seat." + EngineSeat + ".event." + event + "." + node +} + +// EngineCall is one call of the seat: its verb, its caller as the bus proved it, and its arguments. +type EngineCall struct { + Verb string + Caller string + Args map[string]any +} + +// EngineFunc answers one call: its result, or why it is refused. +type EngineFunc func(EngineCall) (any, error) + +// CallerOfSubject is the caller a call's subject names for this node's verb: the last token of a caller-named subject, +// each `~` a dot, or stdio.Unproven on the subject that names none or on any other. +func CallerOfSubject(subject, verb, node string) string { + token, ok := strings.CutPrefix(subject, "mesh.seat."+EngineSeat+".call."+verb+"."+node+".") + if !ok || token == "" || strings.ContainsAny(token, ".*> ") { + return stdio.Unproven + } + return strings.ReplaceAll(token, "~", ".") +} + +// MaxEngineCall bounds a call's arguments: a path, an access and a few words. +const MaxEngineCall = 16 << 10 + +// AnswerEngineCall is the envelope one call is answered with: the verb's result, or its refusal in words. +func AnswerEngineCall(handle EngineFunc, verb, node, subject string, body []byte) []byte { + envelope := map[string]any{"node": node} + say := func() []byte { + raw, err := json.Marshal(envelope) + if err != nil { + raw, _ = json.Marshal(map[string]any{"node": node, "error": "the answer could not be written: " + err.Error()}) + } + return raw + } + args := map[string]any{} + switch { + case handle == nil: + envelope["error"] = "this engine serves no verb of " + EngineSeat + return say() + case len(body) > MaxEngineCall: + envelope["error"] = fmt.Sprintf("the call is %d bytes, more than a call of %s takes", len(body), EngineSeat) + return say() + case len(strings.TrimSpace(string(body))) > 0: + if err := json.Unmarshal(body, &args); err != nil { + envelope["error"] = "the call's arguments are not an object: " + err.Error() + return say() + } + } + result, err := handle(EngineCall{Verb: verb, Caller: CallerOfSubject(subject, verb, node), Args: args}) + if err != nil { + envelope["error"] = err.Error() + } else { + envelope["result"] = result + } + return say() +} + +// EngineSeats is a link that serves this node's seat. Optional, as Asked is. +type EngineSeats interface { + ServeEngineSeat(EngineFunc) +} + +// ServeEngineSeat answers every verb of the seat on this node's subjects, as the service node-engine, which says each +// verb's seat, tool and node to discovery (novox/hq ADR 0197). A bus whose user list is older than the grant refuses the +// subscriptions, said by the client; a caller then hears nobody answer. +func (l *natsLink) ServeEngineSeat(handle EngineFunc) { + if handle == nil || l.conn == nil { + return + } + svc, err := micro.AddService(l.conn, micro.Config{Name: EngineSeat, Version: "1.0.0", + Description: "the node seat node-engine of " + l.node + " (novox/hq ADR 0563)", QueueGroupDisabled: true}) + if err != nil { + return + } + for _, v := range catalogue.EngineVerbs() { + verb := v.Name + answer := micro.HandlerFunc(func(r micro.Request) { + _ = r.Respond(AnswerEngineCall(handle, verb, l.node, r.Subject(), r.Data())) + }) + meta := map[string]string{"seat": EngineSeat, "tool": verb, "node": l.node} + _ = svc.AddEndpoint(verb, answer, micro.WithEndpointSubject(EngineToolSubject(verb, l.node)), + micro.WithEndpointMetadata(meta)) + _ = svc.AddEndpoint(verb+"-called", answer, micro.WithEndpointSubject(EngineCallPattern(verb, l.node)), + micro.WithEndpointMetadata(meta)) + } + l.engine = svc +} + +// SayEngine says one of the seat's events about this node, on the events stream. +func (l *natsLink) SayEngine(event string, body any) error { + raw, err := json.Marshal(body) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err = l.js.Publish(EngineEventSubject(event, l.node), raw, nats.Context(ctx)) + return err +} + +// The engine asks the operator under its own asker name (asks.EngineAsker) on the router's seat (ADR 0259 §3, as ADR +// 0563 §3 changes it): its ask and cancel on the seat's work stream, its warrants heard on its own decided subject, and +// its asks' state read from the router's records under its own name. + +// RouterRecords is the router's bucket of asks, which each asker reads under its own name: the state `asks` of the +// operator-channel's holder, the messenger (mesh-catalog modules/messenger). +const RouterRecords = "messenger_asks" + +// OperatorAsks is a link that asks the operator as this node's engine. Optional. +type OperatorAsks interface { + PublishAsk(subject string, body any, id string) error + RouterRecord(key string) (*stdio.StateEntry, error) + HearWarrants(asker string, heard func(asks.Warrant)) error +} + +// PublishAsk publishes an ask or a cancel on the router's seat, once per id within the stream's duplicate window. +func (l *natsLink) PublishAsk(subject string, body any, id string) error { + raw, err := json.Marshal(body) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _, err = l.js.Publish(subject, raw, nats.MsgId(id), nats.Context(ctx)) + return err +} + +// RouterRecord reads one key of the router's records, `.`; nil when it holds none. +func (l *natsLink) RouterRecord(key string) (*stdio.StateEntry, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + msg, err := l.conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+RouterRecords+".$KV."+RouterRecords+"."+key, nil) + if err != nil { + return nil, err + } + if status := msg.Header.Get("Status"); status == "404" { + return nil, nil + } else if status != "" { + return nil, fmt.Errorf("the router's record %s could not be read: %s %s", key, status, msg.Header.Get("Description")) + } + return &stdio.StateEntry{Key: key, Value: msg.Data}, nil +} + +// HearWarrants hears the warrants for asker on its decided subject, each handed to heard as it arrives. +func (l *natsLink) HearWarrants(asker string, heard func(asks.Warrant)) error { + sub, err := l.conn.Subscribe(asks.DecidedSubject(asker), func(msg *nats.Msg) { + var w asks.Warrant + if json.Unmarshal(msg.Data, &w) == nil { + heard(w) + } + }) + if err != nil { + return err + } + l.warrants = sub + return nil +} + +// errNoAsking is a link that cannot ask the operator. +var errNoAsking = errors.New("the link open now cannot ask the operator") + +// discoveryQuiet is how long the engine waits after the last answer to the bus's discovery, and discoveryMost the +// longest it waits in all. +const ( + discoveryQuiet = 300 * time.Millisecond + discoveryMost = 2 * time.Second +) + +// HeardServed asks the bus's discovery (novox/hq ADR 0197) whether a seat's verb is answered on this machine: an +// endpoint naming the seat, the verb as its tool and this node. +func (l *natsLink) HeardServed(seat, verb string) (bool, error) { + inbox := l.conn.NewRespInbox() + sub, err := l.conn.SubscribeSync(inbox) + if err != nil { + return false, err + } + defer func() { _ = sub.Unsubscribe() }() + if err := l.conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil { + return false, fmt.Errorf("asking the bus who serves what: %w", err) + } + heard := false + deadline := time.Now().Add(discoveryMost) + for time.Now().Before(deadline) { + msg, err := sub.NextMsg(discoveryQuiet) + if err != nil { + break + } + var info micro.Info + if json.Unmarshal(msg.Data, &info) != nil { + continue + } + for _, e := range info.Endpoints { + node := e.Metadata["node"] + if node == "" { + node = info.ID + } + if e.Metadata["seat"] == seat && e.Metadata["tool"] == verb && node == l.node { + heard = true + } + } + } + return heard, nil +} + +// HeardServed is the link open now asking the bus's discovery whether a seat's verb is answered on this machine. +func (q *Session) HeardServed(seat, verb string) (bool, error) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + asker, ok := bus.(interface { + HeardServed(string, string) (bool, error) + }) + if bus == nil || !ok { + return false, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked) + } + return asker.HeardServed(seat, verb) +} + +// ---- the engine's acts in the bus's own storage (ADR 0563 §2, §3) ------------------------------------------------- + +// actsStore is this node's keys of ActsBucket over the link open now: bound, never created (the grants part creates +// it, as the node-engine's declared state). +func (l *natsLink) actsStore() (nats.KeyValue, error) { + kv, err := l.js.KeyValue(ActsBucket) + if err != nil { + return nil, fmt.Errorf("the node-engine's state %s cannot be bound (it is created from the node-engine's "+ + "manifest; asked again once it is): %w", ActsBucket, err) + } + return kv, nil +} + +// BusState is the engine's ActStore: its own node's keys of ActsBucket, `.`, over whichever link the session +// has open; with none open every read and write is refused, and nothing is kept anywhere else. +type BusState struct { + Session *Session + Node string +} + +func (b BusState) kv() (nats.KeyValue, error) { + b.Session.init() + b.Session.mu.Lock() + bus := b.Session.bus + b.Session.mu.Unlock() + l, ok := bus.(interface{ actsStore() (nats.KeyValue, error) }) + if bus == nil || !ok { + return nil, fmt.Errorf("%w: this machine has no link to the bus open now, where its acts are kept", ErrNotAsked) + } + return l.actsStore() +} + +func (b BusState) key(k string) string { return b.Node + "." + k } + +// changed is a write that lost its compare-and-set, as stdio.ErrChanged. +func changed(err error) error { + var api *nats.APIError + if errors.Is(err, nats.ErrKeyExists) || (errors.As(err, &api) && api.ErrorCode == nats.JSErrCodeStreamWrongLastSequence) { + return stdio.ErrChanged + } + return err +} + +func (b BusState) Get(key string) (*stdio.StateEntry, error) { + kv, err := b.kv() + if err != nil { + return nil, err + } + e, err := kv.Get(b.key(key)) + if errors.Is(err, nats.ErrKeyNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return &stdio.StateEntry{Key: key, Value: e.Value(), Revision: e.Revision()}, nil +} + +func (b BusState) Create(key string, value any) (uint64, error) { + kv, err := b.kv() + if err != nil { + return 0, err + } + raw, err := json.Marshal(value) + if err != nil { + return 0, err + } + rev, err := kv.Create(b.key(key), raw) + return rev, changed(err) +} + +func (b BusState) Update(key string, value any, revision uint64) (uint64, error) { + kv, err := b.kv() + if err != nil { + return 0, err + } + raw, err := json.Marshal(value) + if err != nil { + return 0, err + } + rev, err := kv.Update(b.key(key), raw, revision) + return rev, changed(err) +} + +func (b BusState) Delete(key string) error { + kv, err := b.kv() + if err != nil { + return err + } + return kv.Delete(b.key(key)) +} + +// Keys are this node's keys under prefix, without the node: a watch of `.>` alone, the one filter the node's +// grant names, its values left out. +func (b BusState) Keys(prefix string) ([]string, error) { + kv, err := b.kv() + if err != nil { + return nil, err + } + w, err := kv.Watch(b.Node+".>", nats.MetaOnly(), nats.IgnoreDeletes()) + if err != nil { + return nil, err + } + defer func() { _ = w.Stop() }() + var out []string + for { + select { + case e := <-w.Updates(): + if e == nil { + sort.Strings(out) + return out, nil + } + if k := strings.TrimPrefix(e.Key(), b.Node+"."); strings.HasPrefix(k, prefix) { + out = append(out, k) + } + case <-time.After(10 * time.Second): + return nil, errors.New("the node-engine's acts could not be listed within 10s") + } + } +} diff --git a/internal/link/handacts.go b/internal/link/handacts.go new file mode 100644 index 0000000..a411516 --- /dev/null +++ b/internal/link/handacts.go @@ -0,0 +1,769 @@ +package link + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "strings" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/asker" + "git.novox.be/novox/mesh-sdk/go/asks" +) + +// The node seat node-engine's acts (novox/hq ADR 0563). +// +// A hand-over and an access act are the engine's own asks, performed only on the warrant (§3): a call performs nothing. +// The engine judges every value as it does for the controller's signed ask, then refuses in words or asks the operator +// under its own asker name (asks.EngineAsker) at the level approve, binding the digest of the verb, the node, every +// argument and the owner, group and mode it will set. A warrant is claimed once by compare-and-set in the engine's own +// state (asker.Client.Take), held to the ask it answers, and to the act bound and the disk as judged, judged again now; +// only then is the act performed and recorded with the warrant. A decline, an expiry or a changed disk says refused and +// changes nothing. One ask is open per path or access. +// +// The read-only verbs need no warrant (§2): judge-directories changes nothing; setuid-search is bounded here, one at a +// time and at most one started in an hour. Every act, with who asked as the runtime proved it, is kept in the engine's own +// declared state acts (ActsBucket, created by the grants part), under its own node's keys alone, read by the verb acts. + +// The acts of the seat, as its verbs name them. +const ( + ActHandOver = "hand-over" + ActAccess = "access-act" + ActJudgeDirectories = "judge-directories" + ActSetuidSearch = "setuid-search" +) + +// The states of an act. +const ( + HandAsked = "asked" + HandDone = "done" + HandRefused = "refused" + HandAnswered = "answered" +) + +// SearchEvery is how often a setuid search may be started on a node (ADR 0563 §2): one in any hour. +const SearchEvery = time.Hour + +// askMargin is how long past an ask's expiry the router is given to say it ended, before the engine closes it itself. +const askMargin = 5 * time.Minute + +// ActsBucket is the node-engine's declared state acts (its manifest's state, ADR 0563 §2, §3), the bucket the grants +// part creates (mesh-sdk buscompose EngineActsBucket): keys `.act.`, `.open..` and +// `.ask.`, each written by this node's engine alone. +const ActsBucket = "mesh-host_acts" + +// Judged is what the engine judged of one act before it asks: what it is about, what it will set (the owner, group and +// mode, in words), and what the disk holds now (Disk), compared again when the warrant arrives. +type Judged struct { + // Subject is the path of a hand-over, or the access of an access act. + Subject string + // Path is the directory the act changes, as the engine states it. + Path string + // Sets is the owner, group and mode the act will set, in words; bound in the ask's digest. + Sets string + // Disk is what the engine read on disk when it judged: owner, group, mode and the file it is. + Disk string + // Words is what the operator is shown of it, without a path (ADR 0234 §6): the path is in the whole words alone. + Words string +} + +// HandAct is one act asked through the seat, as acts reads it. +type HandAct struct { + ID string `json:"id"` + Act string `json:"act"` + Subject string `json:"subject,omitempty"` + Caller string `json:"caller"` + Why string `json:"why,omitempty"` + At time.Time `json:"at"` + State string `json:"state"` + Said string `json:"said,omitempty"` + Ended time.Time `json:"ended,omitempty"` + // Ask is the operator's ask of a hand-over or an access act; Sets and Disk what was judged when it was asked. + Ask string `json:"ask,omitempty"` + Sets string `json:"sets,omitempty"` + Disk string `json:"disk,omitempty"` + Path string `json:"path,omitempty"` + // Expires is when the ask ends unanswered. + Expires time.Time `json:"expires,omitempty"` + // By, Via, Proofs are the operator's warrant, recorded with the act (ADR 0563 §3). + By string `json:"by,omitempty"` + Via string `json:"via,omitempty"` + Proofs []string `json:"proofs,omitempty"` + // Auto says the engine asked it itself, on finding a library's root holding data outside its group (§4). + Auto bool `json:"auto,omitempty"` +} + +// RootVerdict is the engine's answer to root-free (ADR 0563 §6), in the controller's words. +type RootVerdict struct { + Machine string `json:"machine"` + Free bool `json:"free"` + Why string `json:"why"` + Judged time.Time `json:"judged"` +} + +// HandActs is the engine's side of the seat. Every func is the engine's own judgement or act; nil refuses. +type HandActs struct { + Node string + // State is where the acts and the asks are kept: this node's keys of ActsBucket (BusState), each key one act. + State ActStore + // Asker asks the operator as asks.EngineAsker(Node), with State as its store. + Asker *asker.Client + // JudgeHandOver judges a hand-over's path, refusing in words; HandOver records it on the warrant. + JudgeHandOver func(path string) (Judged, error) + HandOver HandOverFunc + // JudgeAccess judges an access act on an access, refusing in words; AccessAct runs it on the warrant. + JudgeAccess func(act, access string) (Judged, error) + AccessAct AccessActFunc + // JudgeDirectories, SetuidSearch and Searching are the read-only verbs' and whether a search runs now. + JudgeDirectories JudgeDirectoriesFunc + SetuidSearch SetuidSearchFunc + Searching func() bool + // RootFree is this machine's root-free verdict now. + RootFree func() RootVerdict + // Say says one of the seat's events, asked, done or refused; nil says nothing. + Say func(event string, body any) + Now func() time.Time + // Nonce keeps two asks of one second apart; nil is random. + Nonce func() string + + mu sync.Mutex +} + +func (h *HandActs) now() time.Time { + if h.Now != nil { + return h.Now() + } + return time.Now() +} + +func (h *HandActs) nonce() string { + if h.Nonce != nil { + return h.Nonce() + } + var b [2]byte + _, _ = rand.Read(b[:]) + return hex.EncodeToString(b[:]) +} + +func (h *HandActs) say(event string, a HandAct) { + if h.Say != nil { + h.Say(event, map[string]any{"act": a.Act, "subject": a.Subject, "caller": a.Caller, "ask": a.Ask, + "state": a.State, "said": a.Said, "node": h.Node}) + } +} + +// Call is the seat's one entry: a verb, the caller as the runtime proved it, and its arguments. +func (h *HandActs) Call(c EngineCall) (any, error) { + arg := func(k string) string { s, _ := c.Args[k].(string); return strings.TrimSpace(s) } + switch c.Verb { + case ActHandOver: + return h.AskHandOver(c.Caller, arg("path"), arg("why"), false) + case ActAccess: + return h.AskAccess(c.Caller, arg("act"), arg("access"), arg("why"), false) + case ActJudgeDirectories: + return h.Judge(c.Caller) + case ActSetuidSearch: + return h.Search(c.Caller, arg("why")) + case "root-free": + if h.RootFree == nil { + return nil, errors.New("this engine judges no machine root-free") + } + return map[string]any{"machines": []RootVerdict{h.RootFree()}}, nil + case "acts": + if id := arg("id"); id != "" { + return h.Act(id) + } + limit, _ := strconv.Atoi(arg("limit")) + if f, ok := c.Args["limit"].(float64); ok { + limit = int(f) + } + return h.Acts(limit) + } + return nil, fmt.Errorf("the node seat %s serves no verb %q", EngineSeat, c.Verb) +} + +// ---- the record -------------------------------------------------------------------------------------------------- + +// ActStore is the engine's own state, this node's keys alone (BusState prefixes them with the node): each act one key, +// written by compare-and-set on its revision, an open ask one key made by a create. +type ActStore interface { + Get(key string) (*stdio.StateEntry, error) + Create(key string, value any) (uint64, error) + Update(key string, value any, revision uint64) (uint64, error) + Delete(key string) error + // Keys are the keys under prefix. + Keys(prefix string) ([]string, error) +} + +func actKey(id string) string { return "act." + id } + +// openKey is the one key an open ask of act on subject holds: made by a create, so a second ask of it is refused. +func openKey(act, subject string) string { + sum := sha256.Sum256([]byte(subject)) + return "open." + act + "." + hex.EncodeToString(sum[:12]) +} + +// read is every act kept, oldest first. +func (h *HandActs) read() ([]HandAct, error) { + keys, err := h.State.Keys("act.") + if err != nil { + return nil, fmt.Errorf("the engine's acts cannot be listed: %w", err) + } + var out []HandAct + for _, k := range keys { + a, _, err := h.readOne(k) + if err != nil { + return nil, err + } + if a != nil { + out = append(out, *a) + } + } + sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) }) + return out, nil +} + +func (h *HandActs) readOne(key string) (*HandAct, uint64, error) { + e, err := h.State.Get(key) + if err != nil || e == nil { + return nil, 0, err + } + var a HandAct + if err := json.Unmarshal(e.Value, &a); err != nil { + return nil, 0, fmt.Errorf("the engine's act %s cannot be read: %w", key, err) + } + return &a, e.Revision, nil +} + +// judgedKey keeps the node's judgements and refused searches, the last keptJudged of them: any caller may ask them, so +// they roll in one key and never add one (ADR 0563 §2: a wrong caller gains nothing, and fills no bucket). +const ( + judgedKey = "judged" + keptJudged = 50 +) + +// roll adds a to the rolling key by compare-and-set, the oldest falling out. +func (h *HandActs) roll(a HandAct) error { + for range 5 { + e, err := h.State.Get(judgedKey) + if err != nil { + return err + } + var kept []HandAct + if e != nil { + if err := json.Unmarshal(e.Value, &kept); err != nil { + kept = nil // an unreadable roll is started again, never a reason to refuse + } + } + kept = append(kept, a) + if len(kept) > keptJudged { + kept = kept[len(kept)-keptJudged:] + } + if e == nil { + _, err = h.State.Create(judgedKey, kept) + } else { + _, err = h.State.Update(judgedKey, kept, e.Revision) + } + if !errors.Is(err, stdio.ErrChanged) { + return err + } + } + return errors.New("the engine's judgements kept changing under its write") +} + +func (h *HandActs) rolled() ([]HandAct, error) { + e, err := h.State.Get(judgedKey) + if err != nil || e == nil { + return nil, err + } + var kept []HandAct + if err := json.Unmarshal(e.Value, &kept); err != nil { + return nil, fmt.Errorf("the engine's judgements cannot be read: %w", err) + } + return kept, nil +} + +func (h *HandActs) record(a HandAct) error { + _, err := h.State.Create(actKey(a.ID), a) + return err +} + +// end marks the open act of an ask ended, once, by compare-and-set on its revision, and frees its open key: false when +// it was not open. +func (h *HandActs) end(ask string, set func(*HandAct)) (HandAct, bool, error) { + for range 5 { + a, rev, err := h.readOne(actKey(ask)) + if err != nil || a == nil || a.State != HandAsked { + return HandAct{}, false, err + } + set(a) + a.Ended = h.now().UTC() + if _, err := h.State.Update(actKey(ask), *a, rev); errors.Is(err, stdio.ErrChanged) { + continue + } else if err != nil { + return HandAct{}, false, err + } + _ = h.State.Delete(openKey(a.Act, a.Subject)) + return *a, true, nil + } + return HandAct{}, false, errors.New("the engine's act " + ask + " kept changing under its write") +} + +// Act is the verb acts with an id: that one act. +func (h *HandActs) Act(id string) (any, error) { + a, _, err := h.readOne(actKey(id)) + if err != nil { + return nil, err + } + if a == nil { + return nil, fmt.Errorf("this engine keeps no act %q", id) + } + return map[string]any{"node": h.Node, "acts": []HandAct{*a}}, nil +} + +// Acts is the verb acts: every act kept, newest first, at most limit when given. +func (h *HandActs) Acts(limit int) (any, error) { + acts, err := h.read() + if err != nil { + return nil, err + } + judged, err := h.rolled() + if err != nil { + return nil, err + } + acts = append(acts, judged...) + sort.SliceStable(acts, func(i, j int) bool { return acts[i].At.Before(acts[j].At) }) + out := make([]HandAct, 0, len(acts)) + for i := len(acts) - 1; i >= 0; i-- { + out = append(out, acts[i]) + } + if limit > 0 && len(out) > limit { + out = out[:limit] + } + return map[string]any{"node": h.Node, "acts": out}, nil +} + +// ---- the read-only verbs (§2) ------------------------------------------------------------------------------------- + +// Judge is judge-directories: the engine's judgement, read-only, recorded with its caller. +func (h *HandActs) Judge(caller string) (any, error) { + if h.JudgeDirectories == nil { + return nil, errors.New("this engine judges no directory") + } + said, err := h.JudgeDirectories(caller) + a := HandAct{ID: h.id(ActJudgeDirectories), Act: ActJudgeDirectories, Caller: caller, At: h.now().UTC(), State: HandAnswered} + if err != nil { + a.State, a.Said = HandRefused, err.Error() + } + if rerr := h.roll(a); rerr != nil && err == nil { + return nil, rerr + } + if err != nil { + return nil, err + } + return map[string]any{"node": h.Node, "said": said}, nil +} + +// Search is setuid-search: refused while one runs and when one was started within the hour, naming when the next may +// start; otherwise the last search is thrown away and a full one starts, recorded with its caller. +func (h *HandActs) Search(caller, why string) (any, error) { + if h.SetuidSearch == nil { + return nil, errors.New("this engine keeps no search for setuid programs") + } + h.mu.Lock() + defer h.mu.Unlock() + now := h.now().UTC() + if h.Searching != nil && h.Searching() { + return nil, errors.New("a search for setuid programs runs on this machine now; no second is started while it does") + } + // The last start is a key of its own, claimed by compare-and-set before the search starts: no number of other acts, + // nor the record of acts, moves it (ADR 0563 §2). + if err := h.claimSearch(now, caller); err != nil { + return nil, err + } + said, err := h.SetuidSearch(caller) + a := HandAct{ID: h.id(ActSetuidSearch), Act: ActSetuidSearch, Caller: caller, Why: why, At: now, State: HandDone, Said: said} + if err != nil { + a.State, a.Said = HandRefused, err.Error() + } + // A search started is bounded to one an hour and kept as an act of its own; a refused one, which any caller may + // cause, rolls with the judgements. + record := h.record + if err != nil { + record = h.roll + } + if rerr := record(a); rerr != nil { + return nil, rerr + } + if err != nil { + return nil, err + } + return map[string]any{"node": h.Node, "said": said}, nil +} + +// searchKey keeps the last search's start on this node, and libKey each library's own take ask: keys of their own, which +// no other act's key moves, kept for the bucket's age (ninety days, the node-engine's manifest). +const searchKey = "search.last" + +func libKey(access string) string { + sum := sha256.Sum256([]byte(access)) + return "lib." + hex.EncodeToString(sum[:12]) + ".asked" +} + +// started is the last search's start as kept, and who asked it. +type started struct { + At time.Time `json:"at"` + By string `json:"by"` +} + +// claimSearch refuses a search within SearchEvery of the last start, naming when the next may start, and otherwise +// claims this one by compare-and-set: a second claim at the same moment loses. +func (h *HandActs) claimSearch(now time.Time, caller string) error { + e, err := h.State.Get(searchKey) + if err != nil { + return err + } + var rev uint64 + if e != nil { + var last started + if err := json.Unmarshal(e.Value, &last); err != nil { + return fmt.Errorf("the last search's start cannot be read, so none is started: %w", err) + } + if now.Sub(last.At) < SearchEvery { + return fmt.Errorf("a search was started at %s by %s, and one is started in any hour: the next may start "+ + "at %s", last.At.Format(time.RFC3339), last.By, last.At.Add(SearchEvery).Format(time.RFC3339)) + } + rev = e.Revision + } + claim := started{At: now, By: caller} + if rev == 0 { + _, err = h.State.Create(searchKey, claim) + } else { + _, err = h.State.Update(searchKey, claim, rev) + } + if errors.Is(err, stdio.ErrChanged) { + return errors.New("another search was started at this moment; no second is started") + } + return err +} + +// ---- the asks (§3, §4) -------------------------------------------------------------------------------------------- + +func (h *HandActs) id(act string) string { + return act + "-" + h.now().UTC().Format("20060102t150405") + "-" + h.nonce() +} + +// bound is the act an ask's yes binds: the verb, the node, every argument and what it will set (ADR 0563 §3). +func (h *HandActs) bound(a HandAct) asks.Act { + act := asks.Act{"verb": a.Act, "node": h.Node, "arg.subject": a.Subject, "arg.why": a.Why, "sets": a.Sets} + if a.Path != "" { + act["path"] = a.Path + } + return act +} + +// AskHandOver is hand-over: judged, refused in words or asked of the operator. auto is the engine asking by itself. +func (h *HandActs) AskHandOver(caller, path, why string, auto bool) (any, error) { + if h.JudgeHandOver == nil || h.HandOver == nil { + return nil, errors.New("this engine takes no hand-over") + } + j, err := h.JudgeHandOver(path) + if err != nil { + return nil, err + } + return h.ask(ActHandOver, caller, why, auto, j) +} + +// AskAccess is access-act: the act take on an access, judged on the library's home, refused or asked of the operator. +func (h *HandActs) AskAccess(caller, act, access, why string, auto bool) (any, error) { + if h.JudgeAccess == nil || h.AccessAct == nil { + return nil, errors.New("this engine runs no act on an accessed folder") + } + if act != ActTake { + return nil, fmt.Errorf("the act asked is %q, and the one act on an access is %q", act, ActTake) + } + j, err := h.JudgeAccess(act, access) + if err != nil { + return nil, err + } + return h.ask(ActAccess, caller, why, auto, j) +} + +// AskedFor says whether an act on subject was ever asked by the engine itself: a library's take is asked by itself once +// (ADR 0563 §4), and again only by hand. +func (h *HandActs) AskedFor(act, subject string) bool { + if act == ActAccess { + e, err := h.State.Get(libKey(subject)) + return err != nil || e != nil // unread is never a reason to ask again + } + acts, err := h.read() + if err != nil { + return true + } + for _, a := range acts { + if a.Act == act && a.Subject == subject && (a.Auto || a.State == HandAsked) { + return true + } + } + return false +} + +func (h *HandActs) ask(act, caller, why string, auto bool, j Judged) (any, error) { + if h.Asker == nil { + return nil, errors.New("this engine reaches no operator to ask: nothing was asked and nothing changed") + } + h.mu.Lock() + defer h.mu.Unlock() + now := h.now().UTC() + a := HandAct{ID: h.id(act), Act: act, Subject: j.Subject, Path: j.Path, Caller: caller, Why: why, At: now, + State: HandAsked, Sets: j.Sets, Disk: j.Disk, Expires: now.Add(asks.ApproveLasts - time.Minute), Auto: auto} + a.Ask = a.ID + // One ask open per path or access: its open key made by a create, which a second ask cannot make. + if _, err := h.State.Create(openKey(act, j.Subject), map[string]string{"ask": a.Ask}); errors.Is(err, stdio.ErrChanged) { + held := "another" + if e, _ := h.State.Get(openKey(act, j.Subject)); e != nil { + var o struct { + Ask string `json:"ask"` + } + if json.Unmarshal(e.Value, &o) == nil && o.Ask != "" { + held = o.Ask + } + } + return nil, fmt.Errorf("%s of %s is asked already: the ask %s waits for the operator's answer", act, j.Subject, held) + } else if err != nil { + return nil, err + } + if err := h.record(a); err != nil { + _ = h.State.Delete(openKey(act, j.Subject)) + return nil, err + } + bind, err := asks.ActDigest(h.bound(a)) + if err != nil { + return nil, err + } + headline, does, stays := "Hand a directory on "+h.Node+" to the mesh?", "The mesh gives it "+j.Sets+".", + "The directory stays as it was found." + if act == ActAccess { + headline, does, stays = "Keep the library "+strings.TrimPrefix(j.Subject, "library:")+" on "+h.Node+"?", + "The library's group, group write and setgid are set on everything under its root.", + "The library waits and nothing under its root changes." + } + if len([]rune(headline)) > asks.HeadlineLength { + headline = string([]rune(headline)[:asks.HeadlineLength]) + } + asked := asks.Ask{ID: a.Ask, Headline: headline, Who: asks.Operator, + Explanation: j.Words + " Your yes sets " + j.Sets + "; nothing changes until you answer.", + Whole: j.Words + " (" + j.Path + ") Your yes sets " + j.Sets + "; nothing changes until you answer.", + Details: "Asked by " + caller + " on " + h.Node + orWhy(why), + Options: []asks.Option{{ID: "yes", Label: "Yes", Does: does, Level: asks.Approve, Binds: bind}, + {ID: "no", Label: "No, leave it", Does: stays, Level: asks.Acknowledge}}, + Expires: a.Expires, OnExpiry: stays, About: EngineSeat + "-" + h.Node + "-" + act} + if err := h.Asker.Ask(asked); err != nil { + _, _, _ = h.end(a.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, "the operator could not be asked: "+err.Error() }) + return nil, fmt.Errorf("the operator could not be asked, and nothing changed: %w", err) + } + // Once asked, a library's own take ask is kept's own take ask: once, in a key of its own (ADR 0563 §4). + if auto && act == ActAccess { + _, _ = h.State.Create(libKey(j.Subject), map[string]any{"ask": a.Ask, "at": now}) + } + h.say(HandAsked, a) + return map[string]any{"asked": a.Ask, "node": h.Node, "says": "the operator is asked on their channel; only " + + "their yes changes anything. acts shows where it stands."}, nil +} + +func orWhy(why string) string { + if why == "" { + return "" + } + return ": " + why +} + +// Warranted is the act on the operator's answer, handed once by asker.Client.Take after it held the warrant to the +// ask: a no is recorded refused; a yes is held to the act the option bound and the disk as judged, judged again now, +// and only then performed and recorded done with the warrant. +func (h *HandActs) Warranted(o asks.Option, w asks.Warrant) error { + a, _, err := h.readOne(actKey(w.Ask)) + if err != nil { + return err + } + if a == nil || a.State != HandAsked { + return fmt.Errorf("the warrant answers the ask %s, which is not open here: nothing is done", w.Ask) + } + refuse := func(why string) error { + ended, ok, err := h.end(w.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, why; warrantOn(x, w) }) + if err == nil && ok { + h.say(HandRefused, ended) + } + return err + } + if o.ID != "yes" { + return refuse("the operator said no") + } + var j Judged + switch a.Act { + case ActHandOver: + j, err = h.JudgeHandOver(a.Subject) + case ActAccess: + j, err = h.JudgeAccess(ActTake, a.Subject) + default: + err = fmt.Errorf("the act %s is not performed on a warrant", a.Act) + } + if err != nil { + return refuse("judged again on the operator's yes, it is refused: " + err.Error()) + } + now := *a + now.Sets, now.Path = j.Sets, j.Path + if err := o.Performs(h.bound(now)); err != nil { + return refuse("what the yes bound is not what would be done now: " + err.Error()) + } + if j.Disk != a.Disk { + return refuse("the disk changed since the operator was asked (" + a.Disk + ", now " + j.Disk + "): ask again") + } + // Recorded with the way it came (ADR 0563 §5): the operator's yes to this engine's ask, not a terminal's line. + by := operatorOf(w) + ", to " + EngineSeat + "'s ask " + w.Ask + var said string + switch a.Act { + case ActHandOver: + said, err = h.HandOver(a.Subject, by) + case ActAccess: + said, err = h.AccessAct(AccessAct{Act: ActTake, Access: a.Subject, Path: j.Path, Warrant: w.Ask, By: by}) + } + if err != nil { + return refuse(err.Error()) + } + ended, ok, err := h.end(w.Ask, func(x *HandAct) { x.State, x.Said = HandDone, said; warrantOn(x, w) }) + if err == nil && ok { + h.say(HandDone, ended) + } + return err +} + +func warrantOn(a *HandAct, w asks.Warrant) { + a.Proofs = w.Proofs + if w.By != nil { + a.By, a.Via = w.By.Who, w.By.Kind + if w.By.Verified != "" { + a.Via += " (" + w.By.Verified + ")" + } + } +} + +func operatorOf(w asks.Warrant) string { + if w.By == nil { + return "the operator" + } + return "the " + w.By.Who + " via " + w.By.Kind +} + +// Heard is a warrant heard on the engine's decided subject: taken once (asker.Client.Take), and an ask that ended with +// no choice recorded refused. +func (h *HandActs) Heard(w asks.Warrant) { + if h.Asker == nil { + return + } + // A warrant the ask kept here does not hold to (another digest, another option, too late) ends nothing: it is no + // answer to the ask, which stays open for the operator's. + if handled, _ := h.Asker.Take(w, h.Warranted); handled == asker.Recorded { + why := "the ask ended " + string(w.Outcome) + if w.Words != "" { + why += ": " + w.Words + } + if ended, ok, _ := h.end(w.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, why; warrantOn(x, w) }); ok { + h.say(HandRefused, ended) + } + } +} + +// Reconcile catches up what the engine missed while it was away: a warrant the router holds for an open ask is taken, +// and an ask past its expiry and a margin is closed refused, expired. +func (h *HandActs) Reconcile() { + if h.Asker == nil { + return + } + acts, err := h.read() + if err != nil { + return + } + for _, a := range acts { + if a.State != HandAsked || a.Ask == "" { + continue + } + if _, w, err := h.Asker.Where(a.Ask); err == nil && w != nil { + h.Heard(*w) + continue + } + if !a.Expires.IsZero() && h.now().After(a.Expires.Add(askMargin)) { + if ended, ok, _ := h.end(a.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, "the ask expired unanswered" }); ok { + h.say(HandRefused, ended) + } + } + } +} + +// MemState is an ActStore in memory: an engine's acts while no bus is reached in a test. +type MemState struct { + mu sync.Mutex + keys map[string]stdio.StateEntry + top uint64 +} + +func (m *MemState) Get(key string) (*stdio.StateEntry, error) { + m.mu.Lock() + defer m.mu.Unlock() + if e, ok := m.keys[key]; ok { + return &e, nil + } + return nil, nil +} + +func (m *MemState) Create(key string, value any) (uint64, error) { return m.put(key, value, 0, true) } + +func (m *MemState) Update(key string, value any, revision uint64) (uint64, error) { + return m.put(key, value, revision, false) +} + +func (m *MemState) put(key string, value any, revision uint64, create bool) (uint64, error) { + raw, err := json.Marshal(value) + if err != nil { + return 0, err + } + m.mu.Lock() + defer m.mu.Unlock() + was, kept := m.keys[key] + if (create && kept) || (!create && (!kept || was.Revision != revision)) { + return 0, stdio.ErrChanged + } + if m.keys == nil { + m.keys = map[string]stdio.StateEntry{} + } + m.top++ + m.keys[key] = stdio.StateEntry{Key: key, Value: raw, Revision: m.top} + return m.top, nil +} + +func (m *MemState) Delete(key string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.keys, key) + return nil +} + +func (m *MemState) Keys(prefix string) ([]string, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []string + for k := range m.keys { + if strings.HasPrefix(k, prefix) { + out = append(out, k) + } + } + sort.Strings(out) + return out, nil +} diff --git a/internal/link/handover.go b/internal/link/handover.go index 1c5188b..e6103f0 100644 --- a/internal/link/handover.go +++ b/internal/link/handover.go @@ -76,9 +76,13 @@ const MaxHandOverBy = 200 // HandOverCheck is what an ask is held to: the mesh's key, this node's name, the clock, and the nonces taken. type HandOverCheck struct { Signer ed25519.PublicKey - Node string - Now func() time.Time - Nonces *Nonces + // SignerNow, when set, is the controller's key as the engine trusts it at this ask, read per ask in place of Signer: + // nil once the controller's lease is handed away, which refuses every ask signed by its key (novox/hq ADR 0563 §7, + // ADR 0550 §2). + SignerNow func() ed25519.PublicKey + Node string + Now func() time.Time + Nonces *Nonces } // Nonces are the nonces of asks taken, each kept until its ask expires: an ask is taken once. Kept in memory — @@ -157,8 +161,14 @@ func believedBytesWithin(check HandOverCheck, context string, withPath bool, bod if err := json.Unmarshal(body, &signed); err != nil { return HandOverAsk{}, nil, "the ask is not a signed ask: " + err.Error() } - if len(check.Signer) != ed25519.PublicKeySize || - !ed25519.Verify(check.Signer, append([]byte(context), signed.Ask...), signed.Signature) { + signer := check.Signer + if check.SignerNow != nil { + if signer = check.SignerNow(); signer == nil { + return HandOverAsk{}, nil, "the controller's lease is handed away, and no ask signed by its key is taken" + } + } + if len(signer) != ed25519.PublicKeySize || + !ed25519.Verify(signer, append([]byte(context), signed.Ask...), signed.Signature) { return HandOverAsk{}, nil, "the ask is not signed by the mesh" } var ask HandOverAsk diff --git a/internal/link/handover_test.go b/internal/link/handover_test.go index 3442a33..a19d009 100644 --- a/internal/link/handover_test.go +++ b/internal/link/handover_test.go @@ -9,6 +9,9 @@ import ( "strings" "testing" "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "github.com/nats-io/nats.go" ) // The hand-over's ask and answer hold these field names; the controller's side holds the same list @@ -145,6 +148,21 @@ func TestAHandOverAskIsBelievedOnlySignedForThisNodeUnexpiredAndOnce(t *testing. if strings.Join(given, "; ") != "/srv/notes by jo through mesh-cli on anchor; /refused by jo through mesh-cli on anchor" { t.Fatalf("the handler was given %v", given) } + // ADR 0563 §7, ADR 0550 §2: after the lease write the engine answers no ask signed by the controller's key. The key + // is read per ask from what the engine trusts now; nil (the lease handed away) refuses, and nothing is handed over. + trusted := m.Signer + following := HandOverCheck{Signer: m.Signer, SignerNow: func() ed25519.PublicKey { return trusted }, Node: m.Node, + Now: func() time.Time { return now }, Nonces: &Nonces{}} + before := len(given) + trusted = nil + late := signedAsk(t, key, with(func(a *HandOverAsk) { a.Expires = now.Add(time.Minute); a.Nonce = strings.Repeat("cd", 16) })) + if a := AnswerHandOver(following, handle, late); !strings.Contains(a.Refused, "lease is handed away") || len(given) != before { + t.Fatalf("after the lease write an ask signed by the controller was answered %+v", a) + } + trusted = m.Signer + if a := AnswerHandOver(following, handle, late); a.Refused != "" || len(given) != before+1 { + t.Fatalf("while the controller holds the lease its ask was answered %+v", a) + } // A nonce is forgotten once its ask has expired, so the set does not grow for ever. now = now.Add(HandOverGood + time.Second) check.Nonces.take("ff", now.Add(time.Second), now) @@ -174,17 +192,23 @@ func TestAHandOverIsAnsweredOnlyWhenItCarriesAReply(t *testing.T) { // handingLink is a quiet link that hears hand-overs, keeping the handler it was given. type handingLink struct { *quietLink - given chan HandOverFunc + given chan HandOverFunc + signer chan func() ed25519.PublicKey } func (l handingLink) HearHandOvers(h HandOverFunc) { l.given <- h } +// FollowSigner keeps what the link is told to verify the controller's asks with. +func (l handingLink) FollowSigner(f func() ed25519.PublicKey) { l.signer <- f } + // **The link is given the queue's hand-over** when it is served: what the controller asks reaches the engine's // judgement, and nothing else. func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) { m, _ := aMember(t) - l := handingLink{quietLink: newQuietLink(), given: make(chan HandOverFunc, 1)} + l := handingLink{quietLink: newQuietLink(), given: make(chan HandOverFunc, 1), signer: make(chan func() ed25519.PublicKey, 1)} q := &Session{Membership: m, Timeout: time.Second} + // The link verifies the controller's asks with the key the engine trusts now (ADR 0563 §7). + q.ControllerKey = func() ed25519.PublicKey { return nil } q.HandOver = func(path, by string) (string, error) { return "the queue's: " + path + " by " + by, nil } ctx, stop := context.WithCancel(context.Background()) served := make(chan error, 1) @@ -197,8 +221,40 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) { case <-time.After(5 * time.Second): t.Fatal("the link was never given the queue's hand-over") } + select { + case f := <-l.signer: + if f() != nil { + t.Fatal("the link was given another key than the queue's") + } + case <-time.After(5 * time.Second): + t.Fatal("the link was never given the key the engine trusts now") + } stop() <-served + // 774 (ADR 0563 §1): a call of the node seat reaches its verb with its caller as the subject proves it, unproven on + // the subject that names none; a verb the seat does not serve is refused, and the answer is a tool's envelope. + t.Run("a call of the node seat reaches its verb with its proven caller", func(t *testing.T) { + h := &HandActs{Node: "n1", State: &MemState{}, + RootFree: func() RootVerdict { return RootVerdict{Machine: "n1", Free: true, Why: "judged"} }, + JudgeDirectories: func(by string) (string, error) { return "judged for " + by, nil }} + for _, row := range []struct{ verb, subject, want string }{ + {"judge-directories", "mesh.seat.node-engine.call.judge-directories.n1.shanks~node-tools", `"said":"judged for shanks.node-tools"`}, + {"judge-directories", EngineToolSubject("judge-directories", "n1"), `"said":"judged for unproven"`}, + {"judge-directories", "mesh.seat.node-engine.call.judge-directories.n2.shanks~node-tools", `"said":"judged for unproven"`}, + {"root-free", "mesh.seat.node-engine.call.root-free.n1.ace~messenger", `"free":true`}, + {"unserved", "mesh.seat.node-engine.call.unserved.n1.x", `"error":"the node seat node-engine serves no verb \"unserved\""`}, + } { + if got := string(AnswerEngineCall(h.Call, row.verb, "n1", row.subject, []byte(`{}`))); !strings.Contains(got, row.want) || + !strings.Contains(got, `"node":"n1"`) { + t.Errorf("%s on %s: %s", row.verb, row.subject, got) + } + } + acts, _ := h.Acts(0) + if got := acts.(map[string]any)["acts"].([]HandAct); len(got) != 3 || got[0].Caller != "unproven" || + got[2].Caller != "shanks.node-tools" || got[2].Act != ActJudgeDirectories { + t.Errorf("the judgements are recorded as %+v", got) + } + }) } // **Against a real server**: a signed request on the node's own subject is answered; one on another node's is @@ -237,4 +293,63 @@ func TestNatsAHandOverIsAskedOfTheNode(t *testing.T) { if len(handled) != 1 { t.Fatalf("handled %d asks; want only the one with a reply", len(handled)) } + // 774 (ADR 0563 §2, §3): the engine's acts are its node's keys of its declared state on the bus, each written by + // compare-and-set: a second create and a stale revision lose, and another node's keys are not this node's. + t.Run("the acts are kept in the bus's own storage under this node's keys", func(t *testing.T) { + _ = js.DeleteKeyValue(ActsBucket) + if _, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: ActsBucket}); err != nil { + t.Fatal(err) + } + q := &Session{Membership: m, Timeout: time.Second} + q.bus = l + mine, theirs := BusState{Session: q, Node: m.Node}, BusState{Session: q, Node: "another"} + rev, err := mine.Create("act.a-1", map[string]string{"state": "asked"}) + if err != nil { + t.Fatal(err) + } + if _, err := mine.Create("act.a-1", map[string]string{}); !errors.Is(err, stdio.ErrChanged) { + t.Errorf("a second create: %v", err) + } + if _, err := mine.Update("act.a-1", map[string]string{"state": "done"}, rev+7); !errors.Is(err, stdio.ErrChanged) { + t.Errorf("an update at a stale revision: %v", err) + } + if _, err := mine.Update("act.a-1", map[string]string{"state": "done"}, rev); err != nil { + t.Errorf("an update at its revision: %v", err) + } + _, _ = theirs.Create("act.b-1", map[string]string{}) + if keys, err := mine.Keys("act."); err != nil || len(keys) != 1 || keys[0] != "act.a-1" { + t.Errorf("this node's acts are %v, %v", keys, err) + } + if e, _ := js.KeyValue(ActsBucket); e != nil { + if got, err := e.Get(m.Node + ".act.a-1"); err != nil || !strings.Contains(string(got.Value()), "done") { + t.Errorf("the bucket holds %v, %v", got, err) + } + } + if err := mine.Delete("act.a-1"); err != nil { + t.Fatal(err) + } + if e, err := mine.Get("act.a-1"); err != nil || e != nil { + t.Errorf("a deleted act reads %v, %v", e, err) + } + }) + // 774 (ADR 0563 §1): the node seat is served on this node's subjects alone, naming its caller, and said to discovery. + t.Run("the node seat is served on this node and discovered", func(t *testing.T) { + l.ServeEngineSeat(func(c EngineCall) (any, error) { return c.Verb + " for " + c.Caller, nil }) + reply, err := conn.Request("mesh.seat.node-engine.call.acts."+m.Node+".shanks~node-tools", []byte(`{}`), 5*time.Second) + if err != nil || !strings.Contains(string(reply.Data), `"result":"acts for shanks.node-tools"`) { + t.Fatalf("the call answered %v, %v", reply, err) + } + if _, err := conn.Request(EngineToolSubject("acts", "another"), []byte(`{}`), 300*time.Millisecond); err == nil { + t.Fatal("another node's verb was answered here") + } + for seat, want := range map[string]bool{EngineSeat: true, "node-login-shell": false} { + verb := "acts" + if !want { + verb = "execute" + } + if heard, err := l.HeardServed(seat, verb); err != nil || heard != want { + t.Errorf("discovery hears %s.%s here: %v, %v", seat, verb, heard, err) + } + } + }) } diff --git a/internal/link/have_test.go b/internal/link/have_test.go index 4280811..86f5a95 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -274,7 +274,8 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) { // 1126 (ADR 0555 §3): with its keys, the node key carries the node's record: the profile's present capabilities and // networks, the outward links, the health's contract and the public keys; a report that changes none of it writes // nothing, a changed fact writes the key once, and a report without a profile or health leaves them standing. - keys := RecordKeys{TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert"} + keys := RecordKeys{TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert", + Trusts: []string{"controller key sha256:aa", "vault key sha256:bb"}} // and what it trusts (L3 s6, ADR 0550 §2) q.Keys = func() RecordKeys { return keys } rep.Profile = map[string]any{"capabilities": []any{map[string]any{"name": "systemd", "present": true}, map[string]any{"name": "docker", "present": false}, map[string]any{"name": "apt", "present": true}}, @@ -286,7 +287,7 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) { t.Fatalf("the node key holds no record: %+v, %v", got, err) } wantRecord := delivery.NodeRecord{V: delivery.Version, Node: "n1", TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", - Certificate: "cert", Capabilities: []string{"apt", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true, + Certificate: "cert", Trusts: keys.Trusts, Capabilities: []string{"apt", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true, Networks: []catalogue.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, Contract: NodeKeyContract} if !delivery.SameNodeRecord(*got.Record, wantRecord) { t.Errorf("record = %+v, want %+v", *got.Record, wantRecord) diff --git a/internal/link/health_test.go b/internal/link/health_test.go index db7935a..449f0cb 100644 --- a/internal/link/health_test.go +++ b/internal/link/health_test.go @@ -3,9 +3,12 @@ package link import ( "context" "encoding/json" + "slices" "strings" "testing" "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" ) // A health statement is said on the link open now, as the machine's own word on its own subject, and @@ -50,6 +53,101 @@ func TestAHealthStatementIsSaidOnTheLinkOpenNow(t *testing.T) { if HealthSubject("anchor") != "mesh.control.anchor.health" { t.Fatalf("the subject %s is not inside the host's own grant", HealthSubject("anchor")) } + + // Issue 774: the node's own health report in its key of mesh-host_health, for the watchman's six alerts: written on + // a change only, the streaks and the time of the look not being one, and read first after a restart. + t.Run("the node's health report is written on a change only", func(t *testing.T) { + since := time.Date(2026, 10, 29, 20, 0, 0, 0, time.UTC) + look := func(streak int, writer string) Health { + return Health{Contract: 7, At: time.Now(), Network: &NetworkHealth{State: StateUnhealthy, Since: since, Parts: []NetworkPart{ + {Part: "resolv-conf", State: StateUnhealthy, Writer: writer, Owner: "dnsmasq", Since: since, Streak: streak}}}, + Units: &UnitsHealth{State: "degraded", Failed: []FailedUnit{{Unit: "cups.service", Scope: "system", Since: since}}}, + Resources: []ResourceHealth{ + {Module: "brother", Resource: "brother.scanner", Kind: KindAccount, Account: "jochen", State: StateUnhealthy, + Reason: "relogin needed: the session began before scanner", Since: since}, + {Module: "photos", Resource: "photos.library", Kind: KindDirectory, Target: "/srv/photos", State: StateUnhealthy, + Reason: ReasonUsedAsFound + " owned by root", Since: since}, + {Module: "letta", Resource: "letta.server", Kind: "container", State: StateHealthy, Since: since}, + {Module: "claude-code", Resource: "claude-code.agent", Kind: KindAccount, Account: "agent", Root: "never", + State: StateHealthy, Since: since}}} + } + peers := []PeerSeen{{Key: "B=", Seen: true}, {Key: "A=", Addresses: []string{"10.10.0.2/32"}}} + r := NodeHealthOf("anchor", look(1, "NetworkManager"), peers, Machines{}) + if r.Network.Parts[0].Writer != "NetworkManager" || r.Network.Parts[0].Streak != 0 || r.Peers[0].Key != "A=" || + len(r.Units.Failed) != 1 || len(r.Relogin) != 1 || r.Relogin[0].Account != "jochen" || + len(r.Found) != 1 || r.Found[0].Path != "/srv/photos" || + len(r.Root) != 1 || r.Root[0].Account != "agent" || r.Root[0].State != StateHealthy || r.Contract != 7 { + t.Fatalf("the report is %+v", r) + } + // The machines an unhealthy part points at, as the controller holds it there; none for the machine's own. + machines := Machines{ByAddress: map[string]string{"10.10.0.1": "novox", "10.10.0.4": "ace", "10.10.0.9": "anchor"}, Hub: "novox"} + for _, tc := range []struct { + name string + toward []string + state string + want []string + }{ + {"the tunnel to the hub", []string{"hub"}, StateUnhealthy, []string{"novox"}}, + {"two mesh resolvers", []string{"10.10.0.1", "10.10.0.4"}, StateUnhealthy, []string{"novox", "ace"}}, + {"a resolver a VPN client wrote in is the machine's own", []string{"10.10.0.1", "192.0.2.53"}, StateUnhealthy, nil}, + {"its own resolver is its own", []string{"10.10.0.9"}, StateUnhealthy, nil}, + {"a healthy part points at nothing", []string{"hub"}, StateHealthy, nil}, + } { + h := Health{Network: &NetworkHealth{State: tc.state, Parts: []NetworkPart{{Part: "names", State: tc.state, Toward: tc.toward}}}} + got := NodeHealthOf("anchor", h, nil, machines).Network.Parts[0].Targets + if !slices.Equal(got, tc.want) { + t.Errorf("%s: targets %v, want %v", tc.name, got, tc.want) + } + } + store := &memHealth{} + var last *NodeHealth + clock := func() time.Time { return since } + for i, step := range []struct { + h NodeHealth + fresh bool // a restart: nothing written in this run yet + wrote bool + }{ + {h: r, wrote: true}, + {h: NodeHealthOf("anchor", look(2, "NetworkManager"), peers, Machines{})}, + {h: NodeHealthOf("anchor", look(3, "NetworkManager"), peers, Machines{}), fresh: true}, + {h: NodeHealthOf("anchor", look(3, "openvpn"), peers, Machines{}), wrote: true}, + {h: NodeHealthOf("anchor", look(3, "openvpn"), []PeerSeen{{Key: "A=", Seen: true}}, Machines{}), wrote: true}, + } { + if step.fresh { + last = nil + } + wrote, err := writeNodeHealth(t.Context(), store, &last, step.h, clock) + if err != nil || wrote != step.wrote { + t.Fatalf("step %d: wrote %v (want %v), %v", i, wrote, step.wrote, err) + } + } + got, err := delivery.DecodeNodeHealth(store.values["anchor"]) + if err != nil || got.Node != "anchor" || got.Network.Parts[0].Writer != "openvpn" || !got.Peers[0].Seen || got.At.IsZero() { + t.Fatalf("the key holds %+v, %v", got, err) + } + // The key's bytes are mesh-sdk's layout: decoded and encoded again by it, the same bytes. + if again, err := delivery.EncodeNodeHealth(got); err != nil || string(again) != string(store.values["anchor"]) { + t.Fatalf("the key's bytes are not the sdk's layout: %s\n%s (%v)", store.values["anchor"], again, err) + } + if store.puts != 3 { + t.Fatalf("%d writes, want 3: one per change", store.puts) + } + }) +} + +// memHealth is a HealthStore in memory. +type memHealth struct { + values map[string][]byte + puts int +} + +func (m *memHealth) Get(_ context.Context, key string) ([]byte, error) { return m.values[key], nil } +func (m *memHealth) Put(_ context.Context, key string, value []byte) error { + if m.values == nil { + m.values = map[string][]byte{} + } + m.values[key], m.puts = value, m.puts+1 + return nil } // A declared tool check reads the node tools' reply envelope: healthy only when the tool says so (novox/hq diff --git a/internal/link/hearing_nats.go b/internal/link/hearing_nats.go index da68fa2..9bd0cde 100644 --- a/internal/link/hearing_nats.go +++ b/internal/link/hearing_nats.go @@ -6,9 +6,11 @@ import ( "errors" "fmt" "strings" + "sync" "time" "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/micro" ) // The host's link on the bus: one connection, and what is asked of this node over it. The node hears no declaration @@ -35,11 +37,14 @@ func natsURL(address string) string { // natsLink is this node's connection as a JetStream subscription. type natsLink struct { - conn *nats.Conn - js nats.JetStreamContext - asking *nats.Subscription - asked chan struct{} - handing *nats.Subscription + // mu guards signerNow (FollowSigner). + mu sync.Mutex + signerNow func() ed25519.PublicKey + conn *nats.Conn + js nats.JetStreamContext + asking *nats.Subscription + asked chan struct{} + handing *nats.Subscription // searching hears the controller's terminal asking for a fresh setuid search (novox/hq issue 361). searching *nats.Subscription // judging hears the controller asking for a read-only judgement of this node's directories (issue 358). @@ -48,7 +53,10 @@ type natsLink struct { acting *nats.Subscription // moving hears the controller asking to move a value sealed to this machine into the vault (issue 676). moving *nats.Subscription - node string + // engine serves the node seat node-engine, and warrants hears the operator's answers to its asks (ADR 0563). + engine micro.Service + warrants *nats.Subscription + node string // signer is the mesh's key, which a hand-over ask is verified with as a declaration is (issue 356). signer ed25519.PublicKey lost chan error @@ -141,6 +149,21 @@ func (l *natsLink) hearAsks() { } } +// FollowSigner has every signed ask heard on this link verified with the key the engine trusts at that ask, the +// controller's while it holds its lease and none from its hand-over (novox/hq ADR 0563 §7, ADR 0550 §2). +func (l *natsLink) FollowSigner(f func() ed25519.PublicKey) { + l.mu.Lock() + l.signerNow = f + l.mu.Unlock() +} + +// check is how a signed ask is believed on this link: the static membership signer until FollowSigner names the trust. +func (l *natsLink) check() HandOverCheck { + l.mu.Lock() + defer l.mu.Unlock() + return HandOverCheck{Signer: l.signer, SignerNow: l.signerNow, Node: l.node} +} + // HearHandOvers listens for the controller asking this node to hand a directory used as found to the mesh // (novox/hq issue 356): a request on this machine's own subject, answered once on the reply it carried. The // ask is verified with the mesh's key before anything is read out of it (AnswerHandOver), because a responder @@ -151,7 +174,7 @@ func (l *natsLink) HearHandOvers(handle HandOverFunc) { if handle == nil { return } - check := HandOverCheck{Signer: l.signer, Node: l.node} + check := l.check() handing, err := l.conn.Subscribe(AskHandOverSubject(l.node), func(msg *nats.Msg) { if body, answer := HandOverReply(check, handle, msg.Reply, msg.Data); answer { _ = msg.Respond(body) @@ -170,7 +193,7 @@ func (l *natsLink) HearSetuidSearches(handle SetuidSearchFunc) { if handle == nil { return } - check := HandOverCheck{Signer: l.signer, Node: l.node} + check := l.check() searching, err := l.conn.Subscribe(AskSetuidSearchSubject(l.node), func(msg *nats.Msg) { if body, answer := SetuidSearchReply(check, handle, msg.Reply, msg.Data); answer { _ = msg.Respond(body) @@ -189,7 +212,7 @@ func (l *natsLink) HearDirectoryJudgements(handle JudgeDirectoriesFunc) { if handle == nil { return } - check := HandOverCheck{Signer: l.signer, Node: l.node} + check := l.check() judging, err := l.conn.Subscribe(AskJudgeDirectoriesSubject(l.node), func(msg *nats.Msg) { if body, answer := JudgeDirectoriesReply(check, handle, msg.Reply, msg.Data); answer { _ = msg.Respond(body) @@ -207,7 +230,7 @@ func (l *natsLink) HearAccessActs(handle AccessActFunc) { if handle == nil { return } - check := HandOverCheck{Signer: l.signer, Node: l.node} + check := l.check() acting, err := l.conn.Subscribe(AskAccessActSubject(l.node), func(msg *nats.Msg) { if body, answer := AccessActReply(check, handle, msg.Reply, msg.Data); answer { _ = msg.Respond(body) @@ -225,7 +248,7 @@ func (l *natsLink) HearMoveSecrets(handle MoveSecretFunc) { if handle == nil { return } - check := HandOverCheck{Signer: l.signer, Node: l.node} + check := l.check() moving, err := l.conn.Subscribe(AskMoveSecretSubject(l.node), func(msg *nats.Msg) { if body, answer := MoveSecretReply(check, handle, msg.Reply, msg.Data); answer { _ = msg.Respond(body) @@ -258,6 +281,12 @@ func (l *natsLink) Close() { if l.judging != nil { _ = l.judging.Unsubscribe() } + if l.engine != nil { + _ = l.engine.Stop() + } + if l.warrants != nil { + _ = l.warrants.Unsubscribe() + } if l.conn != nil { l.conn.Close() } @@ -301,10 +330,12 @@ func (l *natsLink) Ask(ctx context.Context, subject string, body []byte) ([]byte // What the queue asks of a live link beside Bus, each by a type assertion that fails quietly: said here // so a link that drops one does not build. var ( - _ Bus = (*natsLink)(nil) - _ HealthBus = (*natsLink)(nil) - _ HaveBus = (*natsLink)(nil) - _ ToolBus = (*natsLink)(nil) - _ Asker = (*natsLink)(nil) - _ Asked = (*natsLink)(nil) + _ Bus = (*natsLink)(nil) + _ HealthBus = (*natsLink)(nil) + _ HaveBus = (*natsLink)(nil) + _ ToolBus = (*natsLink)(nil) + _ Asker = (*natsLink)(nil) + _ Asked = (*natsLink)(nil) + _ EngineSeats = (*natsLink)(nil) + _ OperatorAsks = (*natsLink)(nil) ) diff --git a/internal/link/messages.go b/internal/link/messages.go index 2582562..7a64191 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -313,58 +313,14 @@ const KindRunner = "runner" // ReasonUsedAsFound starts the reason of a directory used as found. const ReasonUsedAsFound = "used as found:" -// UnitsHealth is the machine's service managers in one statement (issue 315). -type UnitsHealth struct { - // State is running, degraded — a unit failed, the modules' or the machine's — or unknown when no - // manager could be read. - State string `json:"state"` - // Failed is every unit failed on two looks in a row that no module places: the machine's own. - Failed []FailedUnit `json:"failed"` - // Unread names a manager that could not be read, with why. - Unread []string `json:"unread,omitempty"` - // Away names an account's own manager not running, so not read: nothing in it is failed (issue 764). - Away []string `json:"away,omitempty"` -} - -// FailedUnit is one failed unit no module places. -type FailedUnit struct { - Unit string `json:"unit"` - // Scope is system, or user: an account's own manager. - Scope string `json:"scope"` - // Load is loaded, not-found — a unit whose file is gone and whose failure its manager still holds… - Load string `json:"load,omitempty"` - // Result is how it failed: exit-code, timeout, start-limit-hit… - Result string `json:"result,omitempty"` - // Resource is the mesh's own resource that names it, when the mesh placed it in its own right. - Resource string `json:"resource,omitempty"` - Since time.Time `json:"since"` -} - -// NetworkHealth is the machine's networking in one statement (ADR 0241): the worst of its parts, since -// when, and each part. -type NetworkHealth struct { - State string `json:"state"` - Since time.Time `json:"since"` - Parts []NetworkPart `json:"parts"` -} - -// NetworkPart is one part of a machine's networking, as its engine judged it on its second look. -type NetworkPart struct { - // Part is resolv-conf, names, tunnel, bus or route. - Part string `json:"part"` - State string `json:"state"` - // Reason is why it is not healthy, in words with no address, path or domain; Said the detail. - Reason string `json:"reason,omitempty"` - Said string `json:"said,omitempty"` - // Writer is the program that rewrote the resolver file, when it can be named; Owner the module whose - // file it is — the uplink's holder. - Writer string `json:"writer,omitempty"` - Owner string `json:"owner,omitempty"` - // Toward is what the failure points at: "hub", or each resolver's address that failed. - Toward []string `json:"toward,omitempty"` - Since time.Time `json:"since"` - Streak int `json:"streak,omitempty"` -} +// The machine's networking and its service managers in a statement are the node's health report's parts (mesh-sdk +// delivery, issue 774): one layout for the message, the report and the watchman that reads it. +type ( + NetworkHealth = delivery.NetworkHealth + NetworkPart = delivery.NetworkPart + UnitsHealth = delivery.UnitsHealth + FailedUnit = delivery.FailedUnit +) // The states a long-running resource is said in (ADR 0240 §4). const ( diff --git a/internal/link/nodehealth.go b/internal/link/nodehealth.go new file mode 100644 index 0000000..b938dbb --- /dev/null +++ b/internal/link/nodehealth.go @@ -0,0 +1,208 @@ +package link + +import ( + "context" + "errors" + "fmt" + "slices" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/declaration" + "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go" +) + +// The node-engine's own health report in the bus's own storage (issue 774, the operator's alert table, 2026-10-29): one +// key per node, ``, of the engine's declared state `health` (HealthBucket, created by the grants part from the +// engine's manifest as its state `acts` is), written by this node's engine alone and only when what it says changed. +// The watchman (seat mesh-watchdogs) reads it for six alerts the controller raised from the passing health message: +// +// - machine-network: Network, the machine's route, tunnel, name servers, resolver file and bus, each a part; +// - network-rewritten: the resolver file's part, unhealthy with Writer, the program that rewrote it, and Owner, the +// module whose file it is; +// - network-unreachable: Peers, each peer of the mesh's tunnel and whether this machine saw it within five minutes, +// read across every node's report; +// - machine-units: Units, the failed system units no module owns; +// - relogin-needed: Relogin, each account whose running session lacks a group it was given; +// - directory-used-as-found: Found, each directory a module declares that the engine uses as found. +// + +// The report's layout is mesh-sdk's (delivery.NodeHealth): the bucket, its version and its parts. +const ( + HealthBucket = delivery.NodeHealthBucket + NodeHealthVersion = delivery.NodeHealthVersion +) + +type ( + NodeHealth = delivery.NodeHealth + PeerSeen = delivery.PeerSeen + NodeAccount = delivery.NodeAccount + NodeDirectory = delivery.NodeDirectory +) + +// ReasonReloginNeeded starts an account's reason when its session needs a new login (accounts.ReasonRelogin). +const ReasonReloginNeeded = "relogin needed" + +// Machines names the mesh's machines for a report's targets: each node by its address on the private network, and the +// hub's node. The zero value names none, and every part is then the machine's own. +type Machines struct { + ByAddress map[string]string + Hub string +} + +// targets is the machines a part points at, as the controller holds it (mesh-controller machine_network.go): the hub's +// node for TowardHub, each resolver's node by its address; none when it points at nothing, or at this machine or an +// address no other mesh machine has, as one of them makes the part the machine's own. +func (m Machines) targets(node string, toward []string) []string { + if len(toward) == 0 { + return nil + } + var out []string + for _, t := range toward { + x := m.ByAddress[t] + if t == TowardHub { + x = m.Hub + } + if x == "" || x == node { + return nil + } + if !slices.Contains(out, x) { + out = append(out, x) + } + } + return out +} + +// TowardHub is what a part failing toward the hub names in Toward (network.TowardHub). +const TowardHub = "hub" + +// NodeHealthOf is the report a health statement and the tunnel's peers make: the statement's network without its +// streaks, each unhealthy part naming the machines it points at (machines), its machine units, and each account and +// directory of its resources that the six alerts read. +func NodeHealthOf(node string, h Health, peers []PeerSeen, machines Machines) NodeHealth { + out := NodeHealth{V: NodeHealthVersion, Node: node, Units: h.Units, Contract: h.Contract} + if len(peers) > 0 { + out.Peers = append([]PeerSeen(nil), peers...) + sort.Slice(out.Peers, func(i, k int) bool { return out.Peers[i].Key < out.Peers[k].Key }) + } + if h.Network != nil { + n := NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: make([]NetworkPart, 0, len(h.Network.Parts))} + for _, p := range h.Network.Parts { + p.Streak = 0 // a look's count moves every look; the report says a part's state, not how often it was seen + p.Targets = nil + if p.State == StateUnhealthy { + p.Targets = machines.targets(node, p.Toward) + } + n.Parts = append(n.Parts, p) + } + out.Network = &n + } + for _, r := range h.Resources { + // Each alert reads its own: an account may need a new login and be declared never to become root at once. + if r.Kind == KindAccount && r.State == StateUnhealthy && strings.HasPrefix(r.Reason, ReasonReloginNeeded) { + out.Relogin = append(out.Relogin, NodeAccount{Account: r.Account, Module: r.Module, Resource: r.Resource, + Reason: r.Reason, Since: r.Since}) + } + if r.Kind == KindAccount && r.Root == declaration.RootNever { + // The account declared never to become root without a person, judged (ADR 0266): what DA and D-root read. + out.Root = append(out.Root, delivery.RootVerdict{Account: r.Account, Module: r.Module, Resource: r.Resource, + State: r.State, Reason: r.Reason, Since: r.Since}) + } + if r.Kind == KindDirectory && strings.HasPrefix(r.Reason, ReasonUsedAsFound) { + out.Found = append(out.Found, NodeDirectory{Module: r.Module, Resource: r.Resource, Path: r.Target, + Reason: r.Reason, Since: r.Since}) + } + } + sort.Slice(out.Relogin, func(i, k int) bool { + return out.Relogin[i].Account+out.Relogin[i].Module < out.Relogin[k].Account+out.Relogin[k].Module + }) + sort.Slice(out.Found, func(i, k int) bool { return out.Found[i].Path < out.Found[k].Path }) + sort.Slice(out.Root, func(i, k int) bool { + return out.Root[i].Module+out.Root[i].Resource < out.Root[k].Module+out.Root[k].Resource + }) + return out +} + +// HealthStore is where a node's report is kept: its own key of HealthBucket. +type HealthStore interface { + // Get is the key's value, nil when it has none. + Get(ctx context.Context, key string) ([]byte, error) + Put(ctx context.Context, key string, value []byte) error +} + +// healthStore binds HealthBucket over the link open now: bound, never created. +func (l *natsLink) healthStore() (HealthStore, error) { + kv, err := l.js.KeyValue(HealthBucket) + if err != nil { + return nil, fmt.Errorf("the node-engine's state %s cannot be bound (it is created from the node-engine's "+ + "manifest; written again once it is): %w", HealthBucket, err) + } + return natsHealth{kv}, nil +} + +type natsHealth struct{ kv nats.KeyValue } + +func (n natsHealth) Get(_ context.Context, key string) ([]byte, error) { + e, err := n.kv.Get(key) + if errors.Is(err, nats.ErrKeyNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return e.Value(), nil +} + +func (n natsHealth) Put(_ context.Context, key string, value []byte) error { + _, err := n.kv.Put(key, value) + return err +} + +// WriteNodeHealth writes this node's report to its key when it says something other than the key holds, and says +// whether it wrote. The key is read before the first write of a run, so a restart that finds the same report writes +// nothing; a store that cannot be bound or written is an error, and the next look tries again. +func (q *Session) WriteNodeHealth(ctx context.Context, h NodeHealth) (bool, error) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + l, ok := bus.(interface{ healthStore() (HealthStore, error) }) + if bus == nil || !ok { + return false, ErrNoLink + } + store, err := l.healthStore() + if err != nil { + return false, err + } + return writeNodeHealth(ctx, store, &q.healthWritten, h, time.Now) +} + +// writeNodeHealth is WriteNodeHealth over a store: last is the report last written or read, nil before the first. +func writeNodeHealth(ctx context.Context, store HealthStore, last **NodeHealth, h NodeHealth, now func() time.Time) (bool, error) { + if *last == nil { + raw, err := store.Get(ctx, h.Node) + if err != nil { + return false, fmt.Errorf("the health report of %s cannot be read: %w", h.Node, err) + } + if raw != nil { + if was, err := delivery.DecodeNodeHealth(raw); err == nil { + *last = &was + } + } + } + if *last != nil && delivery.SameNodeHealth(**last, h) { + return false, nil + } + h.At = now().UTC() + raw, err := delivery.EncodeNodeHealth(h) + if err != nil { + return false, err + } + if err := store.Put(ctx, h.Node, raw); err != nil { + return false, fmt.Errorf("the health report of %s cannot be written: %w", h.Node, err) + } + *last = &h + return true, nil +} diff --git a/internal/link/record.go b/internal/link/record.go index 1dc28a6..f176bab 100644 --- a/internal/link/record.go +++ b/internal/link/record.go @@ -23,6 +23,8 @@ type RecordKeys struct { Certificate string // Authority is the mesh's own certificate as this machine last applied it (public). Authority string + // Trusts is what this machine trusts a mesh signature from now (delivery.Trusts, ADR 0550 §2). + Trusts []string } // recordOf is the node's record from one facts report and its keys; nil for a report that carries no profile (one that @@ -45,7 +47,7 @@ func recordOf(node string, r Report, keys RecordKeys) *delivery.NodeRecord { return nil } rec := &delivery.NodeRecord{V: delivery.Version, Node: node, TunnelKey: keys.TunnelKey, SealingKey: keys.SealingKey, - ServingKey: keys.ServingKey, Certificate: keys.Certificate, Authority: keys.Authority} + ServingKey: keys.ServingKey, Certificate: keys.Certificate, Authority: keys.Authority, Trusts: keys.Trusts} for _, c := range p.Capabilities { if c.Present && c.Name != "" { rec.Capabilities = append(rec.Capabilities, c.Name) diff --git a/internal/link/run.go b/internal/link/run.go index 3d17258..eac39ef 100644 --- a/internal/link/run.go +++ b/internal/link/run.go @@ -7,6 +7,8 @@ import ( "errors" "fmt" "time" + + meshasks "git.novox.be/novox/mesh-sdk/go/asks" ) // Membership is what a node needs to reach its mesh again, held by the caller. @@ -185,6 +187,12 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann } // And the controller asking this node to hand a directory used as found to the mesh (novox/hq issue // 356), answered on the link the question came over. + // Every signed ask is verified with the key trusted at that ask, none after the lease write (ADR 0563 §7). + if f, ok := link.(interface { + FollowSigner(func() ed25519.PublicKey) + }); ok && queue.ControllerKey != nil { + f.FollowSigner(queue.ControllerKey) + } if h, ok := link.(HandOvers); ok { h.HearHandOvers(queue.HandOver) } @@ -200,6 +208,19 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann if s, ok := link.(AccessActs); ok { s.HearAccessActs(queue.AccessAct) } + // And its node seat, node-engine, with the operator's answers to its own asks (novox/hq ADR 0563): served on this + // link, and what was answered while no link was open caught up from the router's records. + if queue.Acts != nil { + if s, ok := link.(EngineSeats); ok { + s.ServeEngineSeat(queue.Acts.Call) + } + if a, ok := link.(OperatorAsks); ok { + if err := a.HearWarrants(meshasks.EngineAsker(m.Node), queue.Acts.Heard); err != nil { + say("cannot hear the operator's answers to this engine's asks: " + err.Error()) + } + go queue.Acts.Reconcile() + } + } // And asking to move a value sealed to this machine into the vault (design 57's V3, issue 676). if s, ok := link.(MoveSecrets); ok { s.HearMoveSecrets(queue.MoveSecret) diff --git a/internal/link/session.go b/internal/link/session.go index 31f8a5a..5b2fb89 100644 --- a/internal/link/session.go +++ b/internal/link/session.go @@ -2,12 +2,14 @@ package link import ( "context" + "crypto/ed25519" "encoding/json" "errors" "fmt" "sync" "time" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/health" ) @@ -47,6 +49,12 @@ type Session struct { // MoveSecret moves one value sealed to this machine into the vault, asked by the controller during the move of // the secrets (design 57's V3, issue 676). Nil answers every such ask with a refusal. MoveSecret MoveSecretFunc + // ControllerKey is the controller's key as the engine trusts it now: nil once its lease is handed away (novox/hq + // ADR 0563 §7). Each link verifies every signed ask with it, read per ask; nil ControllerKey keeps the membership's. + ControllerKey func() ed25519.PublicKey + // Acts is the node seat node-engine this engine holds (novox/hq ADR 0563): served on every link, and its asks to the + // operator asked and heard over it. Nil serves no seat. + Acts *HandActs // Keys reads the node's public keys and its certificate for its record (ADR 0555 §3), at each facts report; nil writes // no record. Keys func() RecordKeys @@ -78,6 +86,9 @@ type Session struct { // haves is what this node-engine writes beside its facts (novox/hq ADR 0429 §4, issue 1132). haves haveState + // healthWritten is this node's health report as last written or read (WriteNodeHealth): the health loop alone + // writes it, one look at a time. + healthWritten *NodeHealth } func (q *Session) init() { @@ -337,3 +348,49 @@ func (q *Session) AskSeat(ctx context.Context, seat, verb string, args any, with } return env.Result, nil } + +// PublishAsk publishes an ask or a cancel to the operator's router over the link open now (novox/hq ADR 0563 §3). +func (q *Session) PublishAsk(subject string, body any, id string) error { + asking, err := q.operatorAsks() + if err != nil { + return err + } + return asking.PublishAsk(subject, body, id) +} + +// RouterRecord reads one key of the router's records over the link open now. +func (q *Session) RouterRecord(key string) (*stdio.StateEntry, error) { + asking, err := q.operatorAsks() + if err != nil { + return nil, err + } + return asking.RouterRecord(key) +} + +// SayEngine says one of the node seat's events over the link open now; with none open it is not said. +func (q *Session) SayEngine(event string, body any) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + if sayer, ok := bus.(interface{ SayEngine(string, any) error }); ok { + if err := sayer.SayEngine(event, body); err != nil { + q.Say("could not say the node seat's " + event + ": " + err.Error()) + } + } +} + +func (q *Session) operatorAsks() (OperatorAsks, error) { + q.init() + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + if bus == nil { + return nil, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked) + } + asking, ok := bus.(OperatorAsks) + if !ok { + return nil, errNoAsking + } + return asking, nil +} diff --git a/internal/link/setuid_search_test.go b/internal/link/setuid_search_test.go index 5e90229..8b333e7 100644 --- a/internal/link/setuid_search_test.go +++ b/internal/link/setuid_search_test.go @@ -57,4 +57,82 @@ func TestASetuidSearchAskIsBelievedOnlySignedForThisNodeAndOnce(t *testing.T) { if a := AnswerSetuidSearch(check, nil, body); a.Refused == "" { t.Fatalf("no handler: %+v", a) } + // 774 (ADR 0563 §2): the node seat's setuid-search starts none while one runs, at most one in any hour, naming when the + // next may start, and records who asked. + t.Run("the node seat's search is bounded", func(t *testing.T) { + at, running, started := now, false, 0 + h := &HandActs{Node: "n1", State: &MemState{}, Now: func() time.Time { return at }, + Searching: func() bool { return running }, + SetuidSearch: func(by string) (string, error) { started++; return "a new search starts", nil }} + for _, row := range []struct { + name string + after time.Duration + running bool + refused string + started int + }{ + {name: "one runs", running: true, refused: "runs on this machine now", started: 0}, + {name: "the first", started: 1}, + {name: "within the hour", after: 59 * time.Minute, refused: "the next may start at 2026-10-10T03:00:00Z", started: 1}, + {name: "after the hour", after: time.Hour + time.Second, started: 2}, + } { + at, running = now.Add(row.after), row.running + _, err := h.Call(EngineCall{Verb: ActSetuidSearch, Caller: "ace.node-tools"}) + if (row.refused == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), row.refused)) || started != row.started { + t.Errorf("%s: %v, %d started", row.name, err, started) + } + } + acts, _ := h.Acts(1) + if got := acts.(map[string]any)["acts"].([]HandAct); len(got) != 1 || got[0].Caller != "ace.node-tools" || got[0].State != HandDone { + t.Errorf("the last search is recorded as %+v", got) + } + // The bound is kept in a key of its own (rev-l3 round 1): two hundred judgements before a search, or the acts + // gone from the record, never let a second search start within the hour. + h.JudgeDirectories = func(string) (string, error) { return "judged", nil } + for range 200 { + if _, err := h.Call(EngineCall{Verb: ActJudgeDirectories, Caller: "ace.node-tools"}); err != nil { + t.Fatal(err) + } + } + at = now.Add(time.Hour + 2*time.Minute) + if _, err := h.Call(EngineCall{Verb: ActSetuidSearch, Caller: "ace.node-tools"}); err == nil || !strings.Contains(err.Error(), "the next may start") { + t.Errorf("a search after two hundred judgements: %v", err) + } + keys, _ := h.State.Keys("act.") + for _, k := range keys { + _ = h.State.Delete(k) + } + if _, err := h.Call(EngineCall{Verb: ActSetuidSearch, Caller: "ace.node-tools"}); err == nil || !strings.Contains(err.Error(), "the next may start") || started != 2 { + t.Errorf("a search with the acts gone: %v, %d started", err, started) + } + // rev-l3 round 2 (ADR 0563 §2, a wrong caller gains nothing): judgements and refused searches, which any caller + // may ask, keep one rolling key of the node's, so ten thousand of them write a bounded number of bytes and add no + // key to the bucket every node shares. + keysNow := func() (int, int) { + all, _ := h.State.Keys("") + size := 0 + for _, k := range all { + if e, _ := h.State.Get(k); e != nil { + size += len(e.Value) + } + } + return len(all), size + } + _, _ = h.Call(EngineCall{Verb: ActJudgeDirectories, Caller: "unproven"}) + keysBefore, _ := keysNow() + for i := range 10000 { + verb := ActJudgeDirectories + if i%10 == 0 { + verb = ActSetuidSearch + } + _, _ = h.Call(EngineCall{Verb: verb, Caller: "unproven"}) + } + if n, size := keysNow(); n != keysBefore || size > 64<<10 { + t.Errorf("ten thousand judgements and refused searches left %d keys (was %d) of %d bytes", n, keysBefore, size) + } + listed, _ := h.Acts(0) + if got := listed.(map[string]any)["acts"].([]HandAct); len(got) == 0 || got[0].Caller != "unproven" { + t.Errorf("acts does not show the latest judgement first: %d", len(got)) + } + }) } diff --git a/internal/network/network.go b/internal/network/network.go index bd65d52..f1f511a 100644 --- a/internal/network/network.go +++ b/internal/network/network.go @@ -32,6 +32,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "sort" "strconv" "strings" @@ -94,6 +95,18 @@ type Finding struct { Writer string // Toward is what the failure points at: TowardHub, or each resolver's address that failed. Toward []string + // Peers is, for the tunnel, each peer of the mesh's interface and whether it was seen (Peer). + Peers []Peer +} + +// Peer is one peer of the mesh's tunnel as this machine sees it: its public key, the addresses it is allowed, and +// whether it handshook within StaleHandshake. What the watchman reads to tell a machine nobody reaches from one whose +// tunnel alone is down (network-unreachable). Seen, not the handshake's time: the time moves every two minutes, the +// fact only when a peer comes or goes. +type Peer struct { + Key string `json:"key"` + Addresses []string `json:"addresses,omitempty"` + Seen bool `json:"seen"` } // Part is one part's state, as the statement says it. @@ -115,6 +128,8 @@ type Statement struct { Since time.Time `json:"since"` At time.Time `json:"at"` Parts []Part `json:"parts"` + // Peers are the tunnel's peers on the last look that read them, sorted by key; nil where the tunnel is not read. + Peers []Peer `json:"peers,omitempty"` } // Machine is what a look reads, replaced in tests. @@ -266,6 +281,9 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) { changed = true } st.State, st.Since = worst, j.overall.since + // The tunnel's peers as this look read them; a peer seen or lost is a change of its own (network-unreachable). + st.Peers = findings[PartTunnel].Peers + changed = changed || !samePeers(j.said.Peers, st.Peers) j.said = st return st, changed } @@ -435,14 +453,22 @@ func (j *Judge) lookTunnel(ctx context.Context, now time.Time) Finding { } } hub := "" + allowed := map[string][]string{} for _, line := range strings.Split(ips, "\n") { f := strings.Fields(line) for _, prefix := range f[min(1, len(f)):] { + allowed[f[0]] = append(allowed[f[0]], prefix) if _, bits, ok := strings.Cut(prefix, "/"); ok && bits != "32" && bits != "128" { hub = f[0] } } } + peers := make([]Peer, 0, len(handshakes)) + for key, at := range handshakes { + peers = append(peers, Peer{Key: key, Addresses: allowed[key], Seen: at > 0 && now.Sub(time.Unix(at, 0)) <= StaleHandshake}) + } + sort.Slice(peers, func(i, k int) bool { return peers[i].Key < peers[k].Key }) + found := func(f Finding) Finding { f.Peers = peers; return f } age := func(at int64) string { if at == 0 { return "never" @@ -452,23 +478,36 @@ func (j *Judge) lookTunnel(ctx context.Context, now time.Time) Finding { if hub != "" { at := handshakes[hub] if at > 0 && now.Sub(time.Unix(at, 0)) <= StaleHandshake { - return Finding{OK: true} + return found(Finding{OK: true}) } - return Finding{Reason: "the tunnel to the hub has not handshaken for over five minutes", - Said: fmt.Sprintf("%s's newest handshake with the hub was %s", Interface, age(at)), Toward: []string{TowardHub}} + return found(Finding{Reason: "the tunnel to the hub has not handshaken for over five minutes", + Said: fmt.Sprintf("%s's newest handshake with the hub was %s", Interface, age(at)), Toward: []string{TowardHub}}) } if len(handshakes) == 0 { - return Finding{OK: true} + return found(Finding{OK: true}) } var newest int64 for _, at := range handshakes { newest = max(newest, at) } if newest > 0 && now.Sub(time.Unix(newest, 0)) <= StaleHandshake { - return Finding{OK: true} + return found(Finding{OK: true}) } - return Finding{Reason: "no machine has handshaken with the hub's tunnel for over five minutes", - Said: fmt.Sprintf("%s's newest handshake with any of its %d peers was %s", Interface, len(handshakes), age(newest))} + return found(Finding{Reason: "no machine has handshaken with the hub's tunnel for over five minutes", + Said: fmt.Sprintf("%s's newest handshake with any of its %d peers was %s", Interface, len(handshakes), age(newest))}) +} + +// samePeers says two looks saw the same peers, each seen or not alike. +func samePeers(a, b []Peer) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i].Key != b[i].Key || a[i].Seen != b[i].Seen || !slices.Equal(a[i].Addresses, b[i].Addresses) { + return false + } + } + return true } func (j *Judge) lookBus() Finding { diff --git a/internal/network/network_test.go b/internal/network/network_test.go index f27caa0..67155e7 100644 --- a/internal/network/network_test.go +++ b/internal/network/network_test.go @@ -302,9 +302,27 @@ func TestOnTheHubAnyFreshPeerIsAHealthyTunnel(t *testing.T) { f.hs = "A=\t" + itoa(f.now.Add(-time.Hour).Unix()) + "\nB=\t" + itoa(f.now.Unix()) + "\nC=\t0\n" f.ips = "A=\t10.10.0.2/32\nB=\t10.10.0.3/32\nC=\t10.10.0.4/32\n" j := f.judge(t) - if st := f.look(t, j); st.State != Healthy { + st := f.look(t, j) + if st.State != Healthy { t.Fatalf("the hub with one fresh peer is %+v", st.Parts) } + // Each peer and whether it was seen within five minutes, for the watchman's network-unreachable (issue 774); a + // peer seen again is a change, a handshake renewed alone is not. + want := []Peer{{Key: "A=", Addresses: []string{"10.10.0.2/32"}}, {Key: "B=", Addresses: []string{"10.10.0.3/32"}, Seen: true}, + {Key: "C=", Addresses: []string{"10.10.0.4/32"}}} + if !samePeers(st.Peers, want) { + t.Fatalf("the peers are %+v", st.Peers) + } + f.hs = "A=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nB=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nC=\t0\n" + f.now = f.now.Add(LookEvery) + if st, changed := j.Look(t.Context()); !changed || !st.Peers[0].Seen { + t.Fatalf("a peer seen again is no change: changed %v, %+v", changed, st.Peers) + } + f.hs = "A=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nB=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nC=\t0\n" + f.now = f.now.Add(LookEvery) + if _, changed := j.Look(t.Context()); changed { + t.Fatal("handshakes renewed alone are a change") + } } func TestAnUnreadableTunnelIsSaidAndAMissingToolSkipsIt(t *testing.T) { diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index fb14265..44ca041 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -12,7 +12,9 @@ package selfcompose import ( "context" + "crypto/ed25519" "crypto/sha256" + "encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -28,6 +30,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/compose" "git.novox.be/novox/mesh-sdk/go/compose/frombus" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/overlay" "github.com/nats-io/nats.go" @@ -75,6 +78,13 @@ type Env struct { // Fetch reads an archive at a URL in the artifact store: a moved module's templates archive, whose texts its built // manifest carries (issue 1126). Read once per digest. Nil reads none, and a moved module does not compose. Fetch func(ctx context.Context, url string) ([]byte, error) + // VaultAuthority is the vault's authority as this node read it from the vault's `signers` (ADR 0550 §2), composed + // beside the controller's as the mesh's authority (frombus.SignersReader); "" or nil composes the controller's alone. + VaultAuthority func() string + // Certify asks the vault's `sign` for this node's own certificate, keeps it and writes the node's record again (ADR + // 0550 §2): asked only when a module placed here asks for a certificate and the record holds none from an authority + // the node trusts. Nil asks nothing, and such a module stands in, as before. + Certify func(ctx context.Context) error } // VaultSecretChanged is the vault's event that a secret changed: a name and its generation, never a value. @@ -99,6 +109,8 @@ type Composer struct { // sealedOwn is each own secret's sealed value as first handed, by module/name and generation: a value sealed again // is other bytes, so the one first had is kept while the generation stands, and the composition stays the same. sealedOwn map[string]ownSealed + // certified is the serving key a certificate was last asked for successfully, so it is asked once per key. + certified string said string refusal string // held is what the composer's last pass held: each module refused alone, and the node's own ("node") when it could @@ -411,6 +423,24 @@ func (c *Composer) takeModuleHave(u agent.Update) bool { } // inOwnOf is, of the modules p places, those the node runs inside its own composition (c.inOwn), sorted; c.mu held. +// Machines is, from the placements this composer holds, each node by its address on the private network, and the hub's +// node: what names the machine a failing network part points at (delivery.NetworkPart.Targets, issue 774). Empty +// before any placement is read. +func (c *Composer) Machines() (byAddress map[string]string, hub string) { + c.mu.Lock() + defer c.mu.Unlock() + byAddress = map[string]string{} + for node, p := range c.placements { + if a, _, _ := strings.Cut(p.Decided.Address, "/"); a != "" { + byAddress[a] = node + } + if p.Decided.Hub { + hub = node + } + } + return byAddress, hub +} + func (c *Composer) inOwnOf(p delivery.Placement) []string { var out []string for _, m := range p.Modules { @@ -501,6 +531,8 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { if c.env.Running != nil { running = c.env.Running() } + // Its own certificate, asked of the vault when a module here asks for one and the record holds none it trusts. + c.certifyIfWanted(ctx, r, own) scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running} missed := c.missed.Load() compositions, _, held, err := frombus.Compose(ctx, r, c.env.Mesh, scope, fps) @@ -743,9 +775,93 @@ func (c *Composer) Policy(ctx context.Context, module string) string { return policy } +// HoldsData says whether a module's manifest declares data (catalogue.Manifest.DataItems), as a composition's Data says +// it, and whether the manifest could be read: the agent's Data for what runs of a module from the kept declaration +// (agent.Env.HoldsData, issue 924). +func (c *Composer) HoldsData(ctx context.Context, module string) (data, known bool) { + c.mu.Lock() + m, ok := c.modules[module] + c.mu.Unlock() + if !ok || m.Manifest == "" { + return false, false + } + raw, err := reader{c: c}.Manifest(ctx, m.Manifest) + if err != nil { + return false, false + } + parsed, err := catalogue.ParseManifestForInputs(raw) + if err != nil { + return false, false + } + return len(parsed.DataItems()) > 0, true +} + // reader is the composer's view of the bus as frombus reads it. type reader struct{ c *Composer } +// VaultAuthority is the vault's authority the node follows (frombus.SignersReader). +func (r reader) VaultAuthority(context.Context) string { + if r.c.env.VaultAuthority == nil { + return "" + } + return r.c.env.VaultAuthority() +} + +var _ frombus.SignersReader = reader{} + +// certificateWanted says whether a module the placement p puts on this node asks for a certificate for the machine's +// internal name (its manifest's `certificate`), read from the indexer's manifests; a manifest that cannot be read is +// taken as asking none, and the composition says what it misses. +func (c *Composer) certificateWanted(ctx context.Context, r reader, p delivery.Placement) bool { + for _, m := range p.Modules { + c.mu.Lock() + digest := c.modules[m.Module].Manifest + c.mu.Unlock() + if digest == "" { + continue + } + raw, err := r.Manifest(ctx, digest) + if err != nil { + continue + } + if parsed, err := catalogue.ParseManifestForInputs(raw); err == nil && parsed.Certificate != nil { + return true + } + } + return false +} + +// certifyIfWanted asks Certify once per serving key when a module placed here asks for a certificate and this node's +// record holds none for its internal name and serving key from an authority it trusts (the one it holds, or the vault's). +func (c *Composer) certifyIfWanted(ctx context.Context, r reader, p delivery.Placement) { + if c.env.Certify == nil || !c.certificateWanted(ctx, r, p) { + return + } + c.mu.Lock() + rec, ok := c.records[c.env.Node] + asked := c.certified + c.mu.Unlock() + if !ok || rec.ServingKey == "" || asked == rec.ServingKey { + return + } + key, err := base64.StdEncoding.DecodeString(rec.ServingKey) + if err != nil || len(key) != ed25519.PublicKeySize { + return + } + trusted := []string{rec.Authority, r.VaultAuthority(ctx)} + if rec.Certificate != "" && delivery.CheckNodeCertificate(rec.Certificate, trusted, overlay.InternalName(c.env.Node), + ed25519.PublicKey(key), time.Now()) == nil { + return + } + if err := c.env.Certify(ctx); err != nil { + c.sayOnce(fmt.Sprintf("a module here asks for a certificate, and the vault's sign gave none: %v", err)) + return + } + c.mu.Lock() + c.certified = rec.ServingKey + c.mu.Unlock() +} + func (r reader) Placements(context.Context) (map[string]delivery.Placement, error) { r.c.mu.Lock() defer r.c.mu.Unlock() diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index afe1490..e6c36b4 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -3,6 +3,7 @@ package selfcompose import ( "context" "crypto/sha256" + "encoding/base64" "encoding/hex" "encoding/json" "fmt" @@ -1221,6 +1222,32 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { } } b.mu.Unlock() + + // L3 s7 (ADR 0550 §2): the node asks the vault for its own certificate only when a module placed here asks for one + // and its record holds none from an authority it trusts; once per serving key. + certBus := newFakeBus() + certAsks := 0 + certEnv := certBus.env() + certEnv.Certify = func(context.Context) error { certAsks++; return nil } + certEnv.VaultAuthority = func() string { return "" } + cc := New(certEnv) + withCert := `{"module":"web","version":"1","certificate":{"into":"/etc/web/tls.crt"}}` + cdigest := delivery.ManifestDigest([]byte(withCert)) + cc.manifests[digest] = []byte(lemurs) + cc.manifests[cdigest] = []byte(withCert) + cc.modules = map[string]delivery.Module{"lemurs": {Manifest: digest}, "web": {Manifest: cdigest}} + cc.records["n1"] = delivery.NodeRecord{Node: "n1", ServingKey: base64.StdEncoding.EncodeToString(make([]byte, 32))} + onlyLemurs := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "lemurs"}}} + cc.certifyIfWanted(context.Background(), reader{c: cc}, onlyLemurs) + if certAsks != 0 { + t.Errorf("no module asks for a certificate, and the vault was asked %d times", certAsks) + } + withWeb := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "web"}}} + cc.certifyIfWanted(context.Background(), reader{c: cc}, withWeb) + cc.certifyIfWanted(context.Background(), reader{c: cc}, withWeb) + if certAsks != 1 { + t.Errorf("a module asks for a certificate: the vault was asked %d times, want once", certAsks) + } } // patternBus is a bus whose every watch gets the updates of its bucket matching its pattern, as the bus's own watches diff --git a/module.json b/module.json index 8fd4fa6..146d6a5 100644 --- a/module.json +++ b/module.json @@ -17,6 +17,15 @@ "consumes": [ "restic.backed-up" ], + "state": [ + { + "name": "acts", + "ttl-seconds": 7776000 + }, + { + "name": "health" + } + ], "config": { "operator-signers": { "type": "string", diff --git a/packaging/operator_keys_test.go b/packaging/operator_keys_test.go index 5935047..ecf8ed1 100644 --- a/packaging/operator_keys_test.go +++ b/packaging/operator_keys_test.go @@ -68,4 +68,16 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) { if len(m.Consumes) != 1 || m.Consumes[0] != "restic.backed-up" { t.Fatalf("consumes %v", m.Consumes) } + // 774 (ADR 0563 §2, §3): the acts of the node seat and the engine's own asks are its declared state acts, the bucket + // mesh-host_acts the grants part creates and each node's engine writes under its own keys. + parsed, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatal(err) + } + // 774 (the alert table): beside it the state health, one key per node, the node's own health report, kept until + // written again (no TTL: a node that stops writing keeps its last word, which the watchman reads with its time). + if len(parsed.State) != 2 || parsed.State[0].Name != "acts" || parsed.State[0].TTLSeconds != 90*24*60*60 || + parsed.State[1].Name != "health" || parsed.State[1].TTLSeconds != 0 { + t.Fatalf("the manifest declares the state %+v", parsed.State) + } }