novox/mesh-host
18
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request 'Design 47's last milestones: L3's engine re-landed with held modules never removed, provision secrets, the node's own health report (issues 774, 1126, 1954, 1952)' (#177) from feat/774-l3-reland into main

Browse Source
This commit was merged in pull request #177.
This commit is contained in:
mesh-tools
2026-10-30 11:02:15 +00:00
parent a3c098a5c6 015334020a
commit a9edd2951c
47 changed files with 4744 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mesh-host/applied_store.go
+107
View File
@@ -0,0 +1,107 @@
package main
import (
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
)
// appliedStore keeps every composition the agent applied, one file per module (the node's own as `node.json`), in the
// directory `applied` beside the node's state (agent.Env.Applied, issue 774). It is the machine's own record of what runs:
// read at the agent's start before anything is watched, so the node never composes its own as if a module it runs ran
// nothing because the bucket no longer keeps that composition or the kept declaration is retired. Files are 0600 in a
// 0700 directory: a composition carries values sealed to this machine.
type appliedStore struct{ dir string }
func appliedStoreFor(statePath string) appliedStore {
return appliedStore{dir: filepath.Join(filepath.Dir(statePath), "applied")}
}
// appliedName is a module name the store takes as a file name: a catalogue module's or the node's own, never a path.
var appliedName = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]*$`)
func (s appliedStore) path(module string) (string, error) {
if !appliedName.MatchString(module) || strings.Contains(module, "..") {
return "", fmt.Errorf("%q is not a module name the applied store takes", module)
}
return filepath.Join(s.dir, module+".json"), nil
}
// holds says whether the store keeps the module's composition.
func (s appliedStore) holds(module string) bool {
path, err := s.path(module)
if err != nil {
return false
}
_, err = os.Stat(path)
return err == nil
}
func (s appliedStore) Load() (map[string][]byte, error) {
entries, err := os.ReadDir(s.dir)
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
if err != nil {
return nil, err
}
out := map[string][]byte{}
var errs []error
for _, e := range entries {
module, ok := strings.CutSuffix(e.Name(), ".json")
if !ok || e.IsDir() || !appliedName.MatchString(module) {
continue
}
raw, err := os.ReadFile(filepath.Join(s.dir, e.Name()))
if err != nil {
errs = append(errs, err)
continue
}
out[module] = raw
}
return out, errors.Join(errs...)
}
func (s appliedStore) Keep(module string, raw []byte) error {
path, err := s.path(module)
if err != nil {
return err
}
if was, err := os.ReadFile(path); err == nil && string(was) == string(raw) {
return nil
}
if err := os.MkdirAll(s.dir, 0o700); err != nil {
return err
}
tmp, err := os.CreateTemp(s.dir, "."+module+".*")
if err != nil {
return err
}
defer os.Remove(tmp.Name())
if _, err := tmp.Write(raw); err != nil {
tmp.Close()
return err
}
if err := tmp.Sync(); err != nil {
tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
return os.Rename(tmp.Name(), path)
}
func (s appliedStore) Drop(module string) error {
path, err := s.path(module)
if err != nil {
return err
}
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
return nil
}
cmd/mesh-host/authority_test.go
+143 -2
View File
@@ -1,16 +1,27 @@
package main
import (
"context"
"crypto/ed25519"
"crypto/x509"
"encoding/base64"
"encoding/json"
"encoding/pem"
"os"
"path/filepath"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/overlay"
"github.com/novox/mesh-host/internal/identity"
)
// Issue 1126 (ADR 0555 §3): the mesh's own certificate a composition carries is kept for the machine's record, and the
// record's keys read it; a composition carrying none changes nothing.
// record's keys read it; a composition carrying none changes nothing. And (L3 s6, s7; ADR 0550 §2) the record says what
// the node trusts a mesh signature from, and the node keeps the certificate the vault signed for it.
func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) {
state := filepath.Join(t.TempDir(), "state", "state.json")
res := func(id, content string) json.RawMessage {
@@ -26,7 +37,137 @@ func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) {
if err := keepAuthority(state, delivery.Composed{Resources: []json.RawMessage{res("ca-trust.certificate-authority", "CA-1")}}); err != nil {
t.Fatal(err)
}
if got := recordKeys(state)(); got.Authority != "CA-1" {
if got := recordKeys(state, nil)(); got.Authority != "CA-1" {
t.Errorf("the record's keys read the authority %q", got.Authority)
}
// L3 s6: before the vault's signers are read the node trusts the controller's key alone; once read, a message the
// vault signed is accepted and the record names both; once the lease is handed away the controller's key is refused
// and the record names the vault's alone. Each change tells the record's writer.
controllerPub, controller, _ := ed25519.GenerateKey(nil)
vaultPub, vault, _ := ed25519.GenerateKey(nil)
trust := newMeshTrust(controllerPub)
told := 0
trust.onChange(func() { told++ })
keys := recordKeys(state, trust.Trusts)
ctrlFP, vaultFP := delivery.SignerFingerprint(controllerPub), delivery.SignerFingerprint(vaultPub)
if got := keys().Trusts; !delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || delivery.TrustsKey(got, delivery.SignerVault, vaultFP) {
t.Errorf("before the signers are read the record trusts %v", got)
}
enrol, _ := delivery.SignContext(delivery.KindEnrolment)
msg := []byte("a reply")
if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(vault, enrol, msg)); err == nil {
t.Error("a vault-signed message is accepted before the vault's key is read")
}
vaultCA := testAuthority(t, vaultPub, vault)
keyValue, _ := json.Marshal(delivery.NewSignerKey(vaultPub, time.Now()))
fp, _ := delivery.AuthorityFingerprint(vaultCA)
authValue, _ := json.Marshal(delivery.SignerAuthority{Certificate: vaultCA, Fingerprint: fp, Made: time.Now()})
trust.heardSigner(delivery.SignersKeyKey, keyValue, false, func(string) {})
trust.heardSigner(delivery.SignersAuthorityKey, authValue, false, func(string) {})
if s, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(vault, enrol, msg)); err != nil || s.Name != delivery.SignerVault {
t.Errorf("a vault-signed message on a node that read signers: %+v, %v", s, err)
}
if got := keys().Trusts; !delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) ||
trust.VaultAuthority() != vaultCA || told != 2 {
t.Errorf("after the signers are read the record trusts %v (told %d)", got, told)
}
handed, _ := json.Marshal(buscompose.LeaseHolder{HandedTo: "mesh-vault", HandedBy: "person.operator"})
trust.heardLease(handed, true, nil)
if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(controller, enrol, msg)); err == nil || trust.Controller() != nil {
t.Error("after the lease is handed away the controller's key is still trusted")
}
if got := keys().Trusts; delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) || told != 3 {
t.Errorf("after the lease is handed away the record trusts %v (told %d)", got, told)
}
trust.heardLease(handed, true, nil)
if told != 3 {
t.Error("a lease heard again unchanged wrote the record again")
}
// A signers' key naming another fingerprint is not trusted.
bad, _ := json.Marshal(delivery.SignerKey{Key: base64.StdEncoding.EncodeToString(controllerPub), Fingerprint: vaultFP})
trust.heardSigner(delivery.SignersKeyKey, bad, false, func(string) {})
if !delivery.TrustsKey(keys().Trusts, delivery.SignerVault, vaultFP) {
t.Error("a signers' key that does not check replaced the vault's")
}
// L3 s7: the certificate the vault signs for this node's internal name and serving key is kept where the record reads
// it, and the record is written again; one signed by another authority is refused and nothing is kept.
serving, err := identity.GenerateServingKey()
if err != nil {
t.Fatal(err)
}
if err := identity.WriteServingKey(identity.ServingKeyPath(state), serving); err != nil {
t.Fatal(err)
}
servingPub, _ := base64.StdEncoding.DecodeString(serving.Public)
signedBy := func(ca string, key ed25519.PrivateKey) string {
parent, _ := delivery.ParseCertificate(ca)
der, err := x509.CreateCertificate(nil, delivery.NodeCertificateTemplate(overlay.InternalName("n1"), servingPub, time.Now().Add(-time.Hour)),
parent, ed25519.PublicKey(servingPub), key)
if err != nil {
t.Fatal(err)
}
return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))
}
var asked []any
answer := signedBy(vaultCA, vault)
ask := func(_ context.Context, verb string, args any) (json.RawMessage, error) {
asked = append(asked, verb, args)
return json.Marshal(delivery.SignAnswer{Kind: delivery.KindNodeCertificate, Node: "n1", Certificate: answer})
}
written := 0
if err := certifyThisNode(context.Background(), state, "n1", trust, ask, func() { written++ }); err != nil {
t.Fatal(err)
}
if got := recordKeys(state, nil)(); got.Certificate != answer || written != 1 || len(asked) != 2 || asked[0] != delivery.SignVerb {
t.Errorf("the vault's certificate: kept %v, written %d, asked %v", got.Certificate == answer, written, asked)
}
answer = signedBy(testAuthority(t, controllerPub, controller), controller)
if err := certifyThisNode(context.Background(), state, "n1", trust, ask, func() { written++ }); err == nil || written != 1 {
t.Errorf("a certificate of another authority was kept: %v", err)
}
// Issue 774: what the engine applied is kept beside the state too, one file per module, 0600 in a 0700 directory,
// written by a rename so a crash leaves the one before; a name that is no module's is refused, never a path.
t.Run("the compositions the engine applied are kept beside the state", func(t *testing.T) {
applied := appliedStoreFor(filepath.Join(t.TempDir(), "state", "state.json"))
if m, err := applied.Load(); err != nil || len(m) != 0 || applied.holds("node") {
t.Fatalf("an empty store loads %v, %v", m, err)
}
for _, tc := range []struct {
module string
takes bool
}{{"node", true}, {"nats", true}, {"mesh-delivery-view", true}, {"../state", false}, {"a/b", false}, {"..", false}, {"", false}} {
if err := applied.Keep(tc.module, []byte(`{"v":1}`)); (err == nil) != tc.takes {
t.Errorf("keeping %q: %v, want taken %v", tc.module, err, tc.takes)
}
}
if err := applied.Keep("node", []byte(`{"v":2}`)); err != nil {
t.Fatal(err)
}
entries, _ := os.ReadDir(applied.dir)
dirInfo, _ := os.Stat(applied.dir)
fileInfo, _ := os.Stat(filepath.Join(applied.dir, "node.json"))
if len(entries) != 3 || dirInfo.Mode().Perm() != 0o700 || fileInfo.Mode().Perm() != 0o600 {
t.Fatalf("the store holds %d files (want 3, no temporary one left), directory %v, file %v", len(entries), dirInfo.Mode(), fileInfo.Mode())
}
if err := applied.Drop("nats"); err != nil || applied.Drop("nats") != nil || applied.holds("nats") {
t.Fatalf("a dropped module is still held, or dropping twice fails: %v", err)
}
if m, err := applied.Load(); err != nil || string(m["node"]) != `{"v":2}` || len(m) != 2 {
t.Fatalf("the store loads %v, %v", m, err)
}
})
}
// testAuthority is a self-signed authority certificate for key, as the vault makes its own.
func testAuthority(t *testing.T, pub ed25519.PublicKey, key ed25519.PrivateKey) string {
t.Helper()
tpl := delivery.AuthorityTemplate(pub, time.Now().Add(-time.Hour))
der, err := x509.CreateCertificate(nil, tpl, tpl, pub, key)
if err != nil {
t.Fatal(err)
}
return string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}))
}
cmd/mesh-host/engineseat.go
+277
View File
@@ -0,0 +1,277 @@
package main
import (
"bufio"
"context"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"sync/atomic"
"syscall"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
meshasker "git.novox.be/novox/mesh-sdk/go/asker"
"git.novox.be/novox/mesh-sdk/go/asks"
"github.com/novox/mesh-host/internal/accounts"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/store"
)
// The node seat node-engine's judgements on this machine (novox/hq ADR 0563): what a hand-over and a library's take
// would set and what the disk holds now, and whether this machine is root-free.
// diskOf is what is on disk at a directory, as an act compares it when the operator's yes arrives: its owner and
// group, its mode, and which file it is.
func diskOf(info os.FileInfo) string {
owner := "unknown"
if uid, gid, ok := apply.OwnerOf(info); ok {
owner = fmt.Sprintf("%d:%d", uid, gid)
}
which := ""
if st, ok := info.Sys().(*syscall.Stat_t); ok {
which = fmt.Sprintf(" %v:%v", st.Dev, st.Ino)
}
return fmt.Sprintf("%s %o%s", owner, info.Mode().Perm(), which)
}
// handOverJudgeFrom judges a hand-over through the node seat as the controller's ask is judged (judgeFound): the
// declared owner and mode it will set, and the disk as it is.
func handOverJudgeFrom(statePath string) func(string) (link.Judged, error) {
return func(path string) (link.Judged, error) {
found, info, err := judgeFound(statePath, path)
if err != nil {
return link.Judged{}, err
}
return link.Judged{Subject: path, Path: path, Sets: orUnsaid(found.Declared), Disk: diskOf(info),
Words: "A directory of " + moduleOf(found.ID) + " is used as found: the mesh left its owner and mode."}, nil
}
}
func moduleOf(id string) string {
if at := strings.LastIndex(id, "."); at > 0 {
return id[:at]
}
return id
}
// accessJudgeFrom judges the take act on a library through the node seat (ADR 0328 §7, ADR 0563 §4): declared on this
// machine, its home, and its root holding data the mesh has not taken; what the take sets, and the disk as it is.
func accessJudgeFrom(statePath string) func(string, string) (link.Judged, error) {
return func(act, access string) (link.Judged, error) {
d, err := declaredNow(statePath, nil)
if errors.Is(err, store.ErrNothingDeclared) {
return link.Judged{}, errors.New("this node keeps no declaration, so it is declared no library")
}
if err != nil {
return link.Judged{}, fmt.Errorf("what runs on this node does not read: %w", err)
}
for _, l := range d.Libraries {
if apply.LibraryAccess(l.Library) != access {
continue
}
if !l.Home {
return link.Judged{}, fmt.Errorf("this machine is not the home of the library %s, and a library is "+
"taken on its home", l.Library)
}
ctx, cancel := context.WithTimeout(context.Background(), lookBound)
defer cancel()
if v := apply.JudgeLibrary(ctx, l); v.Kind != apply.VerdictWaitKeep {
return link.Judged{}, fmt.Errorf("the library %s does not wait for the operator's word to be kept "+
"(it is judged %s), so there is nothing to take", l.Library, v.Kind)
}
info, err := os.Lstat(l.Root)
if err != nil {
return link.Judged{}, fmt.Errorf("the library %s's root cannot be read: %v", l.Library, err)
}
return link.Judged{Subject: access, Path: filepath.Clean(l.Root),
Sets: "the group " + l.Group + ", group write and setgid on everything under its root",
Disk: diskOf(info), Words: "The library " + l.Library + " holds data the mesh has not taken."}, nil
}
return link.Judged{}, fmt.Errorf("the access %s names no library declared on this machine", access)
}
}
// askTakes asks the take of every library the look found waiting for the operator's word, once per library (ADR 0563
// §4): asked by the engine itself, never a second time while one is open or after one was asked; access-act asks again
// by hand.
func askTakes(acts *link.HandActs, waiting []string, say func(string)) {
if acts == nil {
return
}
for _, access := range waiting {
if acts.AskedFor(link.ActAccess, access) {
continue
}
if _, err := acts.AskAccess(link.EngineSeat+"-"+acts.Node, link.ActTake, access,
"its root holds data outside its group", true); err != nil && say != nil {
say("the take of " + access + " could not be asked: " + err.Error())
}
}
}
// lastAccounts is the newest look at every account (the health loop's), which root-free judges from.
var lastAccounts atomic.Pointer[accounts.Statement]
// rootFreshFor is how old the verdict root-free judges from may be (ADR 0563 §6, the controller's verdictFreshFor).
const rootFreshFor = 15 * time.Minute
// judgeRootFree is this machine's root-free (ADR 0259 §8, ADR 0563 §6): free only on the engine's own verdict, at most
// rootFreshFor old, that every account declared never to become root without a person cannot, at least one such
// account declared, and the login shell's execute not served here. Anything else is no pass, and says why.
func judgeRootFree(node string, st *accounts.Statement, now time.Time, served bool, servedWhy string) link.RootVerdict {
v := link.RootVerdict{Machine: node, Judged: now.UTC()}
var not []string
confined := ""
switch {
case st == nil:
not = append(not, "this machine's engine has judged no account yet")
case now.Sub(st.At) > rootFreshFor:
not = append(not, fmt.Sprintf("the engine's newest judgement of its accounts was made at %s, more than %d "+
"minutes ago, and a verdict that old is not a verdict on now", st.At.UTC().Format(time.RFC3339),
int(rootFreshFor.Minutes())))
default:
var named []string
for _, a := range st.Accounts {
if !a.Root {
continue
}
named = append(named, a.Name)
if a.State != accounts.Healthy {
not = append(not, "the account "+a.Name+" "+orNoneKnownWord(a.Reason))
}
}
if len(named) == 0 {
not = append(not, "no account here is declared never to become root, so agents run as the operator's "+
"account, which may become root")
} else if len(not) == 0 {
confined = "the account " + strings.Join(named, ", ") + " cannot become root without a person (judged " +
st.At.UTC().Format(time.RFC3339) + ")"
}
}
if served {
not = append(not, "the login shell runs any command an agent gives it as the machine's runtime account, which "+
"can become root ("+servedWhy+")")
}
if len(not) > 0 {
v.Why = strings.Join(not, "; ")
return v
}
v.Free, v.Why = true, confined+"; the login shell's execute is not served here"
return v
}
func orNoneKnownWord(s string) string {
if s == "" {
return "is not judged"
}
return s
}
// settingFileSuffix is the resource the login shell's holder renders its setting execute into (ADR 0268: zsh's
// tools-conf, `execute=serve` or `execute=withhold`), read beside the bus's discovery.
const settingFileSuffix = ".tools-conf"
// executeServedHere is whether the login shell's execute is served on this machine (ADR 0268), failing closed: served
// while the bus's discovery hears it answered here or could not be asked, or while the holder's setting file applied
// here says anything but withhold or cannot be read. why says which.
func executeServedHere(statePath string, heard func() (bool, error)) (bool, string) {
var why []string
if heard == nil {
why = append(why, "the bus could not be asked whether execute is answered here")
} else if h, err := heard(); err != nil {
why = append(why, "the bus could not be asked whether execute is answered here: "+err.Error())
} else if h {
why = append(why, "the bus hears execute answered here")
}
known, err := store.Load(statePath)
if err != nil {
why = append(why, "what this machine applied could not be read: "+err.Error())
}
for _, r := range known.Resources {
if r.Type != "file" || !strings.HasSuffix(r.ID, settingFileSuffix) {
continue
}
if value, err := settingIn(r.Target); err != nil {
why = append(why, moduleOf(r.ID)+"'s setting execute cannot be read: "+err.Error())
} else if value != "withhold" {
why = append(why, moduleOf(r.ID)+"'s setting execute here is "+value)
}
}
return len(why) > 0, strings.Join(why, "; ")
}
// settingIn is the value of execute in a setting file, "" when it says none.
func settingIn(path string) (string, error) {
f, err := os.Open(path)
if err != nil {
return "", err
}
defer f.Close()
value := ""
lines := bufio.NewScanner(f)
for lines.Scan() {
t := strings.TrimSpace(lines.Text())
if key, v, ok := strings.Cut(t, "="); ok && !strings.HasPrefix(t, "#") && strings.TrimSpace(key) == "execute" {
value = strings.TrimSpace(v)
}
}
if value == "" {
value = "not said"
}
return value, lines.Err()
}
// keepWaiting is every library a look found waiting for the operator's word to keep it (ADR 0328 §8).
func keepWaiting(vs []link.ResourceHealth) []string {
var out []string
for _, v := range vs {
for _, w := range v.AccessWaits {
if w.Reason == link.WaitKeep {
out = append(out, w.Access)
}
}
}
return out
}
// engineActs is the node seat this engine holds (ADR 0563): judged and performed by the same code as the controller's
// signed asks, its asks to the operator over the session's link, its acts kept beside the state.
func engineActs(opts options, node string, queue *link.Session, search *accounts.SetuidCache) *link.HandActs {
state := link.BusState{Session: queue, Node: node}
h := &link.HandActs{Node: node, State: state,
JudgeHandOver: handOverJudgeFrom(opts.state), HandOver: handOverFrom(opts.state),
JudgeAccess: accessJudgeFrom(opts.state), AccessAct: accessActFrom(opts.state, opts.timeout),
JudgeDirectories: judgeDirectoriesFrom(opts.state), SetuidSearch: queue.SetuidSearch, Searching: search.Running,
RootFree: func() link.RootVerdict {
served, why := executeServedHere(opts.state, func() (bool, error) {
return queue.HeardServed("node-login-shell", "execute")
})
return judgeRootFree(node, lastAccounts.Load(), time.Now(), served, why)
},
Say: queue.SayEngine}
asker := asks.EngineAsker(node)
h.Asker = &meshasker.Client{Module: asker, State: state, Publish: queue.PublishAsk,
Record: func(id string) (*stdio.StateEntry, error) { return queue.RouterRecord(asker + "." + id) }}
return h
}
// reconcileEvery is how often the engine catches up its open asks from the router's records (a warrant heard while no
// link was open, an ask past its expiry): besides each link's opening.
const reconcileEvery = 5 * time.Minute
func reconcileActs(ctx context.Context, h *link.HandActs) {
t := time.NewTicker(reconcileEvery)
defer t.Stop()
for {
select {
case <-ctx.Done():
return
case <-t.C:
h.Reconcile()
}
}
}
cmd/mesh-host/file_references_test.go
+73
View File
@@ -2,6 +2,7 @@ package main
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
@@ -10,6 +11,7 @@ import (
"testing"
"git.novox.be/novox/mesh-sdk/go/declaration"
"git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-host/internal/link"
)
@@ -82,6 +84,77 @@ func TestADeclarationWithAFileByDigestAppliesFromTheObject(t *testing.T) {
if err != nil || string(got) != string(body) {
t.Errorf("1695/AC1: the file holds %d bytes (%v), want %d", len(got), err, len(body))
}
// Issue 924 (novox, 2026-10-29 01:16): the controller's sweep removed an object a composition the node ran still named,
// and the module's apply failed while it ran. Content the node once had is its own keep's, and the bus is not asked.
gone := aFilesLink{err: fmt.Errorf("the object %s was deleted from the bucket declaration-files-novox", name)}
never := []byte(strings.Repeat("never had here\n", 6000))
for _, row := range []struct {
name string
raw []byte
onDisk []byte // nil removes the file
refused string // "" applies
}{
{"the file here drifted, its object deleted", raw, []byte("the module wrote it since"), ""},
{"the file here is gone, its object deleted", raw, nil, ""},
{"never had here, its object deleted", referenceDeclaration(target, never), nil, "never held it"},
} {
if row.onDisk == nil {
_ = os.Remove(target)
} else if err := os.WriteFile(target, row.onDisk, 0o600); err != nil {
t.Fatal(err)
}
report = applyAndKeep(link.WithFiles(context.Background(), gone), opts, row.raw, nil, nil, nil)
if row.refused == "" {
got, _ := os.ReadFile(target)
if report.Refused != "" || string(got) != string(body) {
t.Errorf("924: %s: refused %q, the file holds %d bytes; want it applied from the node's own keep",
row.name, report.Refused, len(got))
}
} else if !strings.Contains(report.Refused, row.refused) || !strings.Contains(report.Refused, "was deleted") {
t.Errorf("924: %s: refused %q; want a refusal saying the bucket and the keep both lack it", row.name, report.Refused)
}
}
// Issue 924, the live case: what runs came from the kept declaration, whose export names a digest swept long ago,
// while the node wrote a newer export since. Composed as it runs, the file names what the node wrote, and applies with
// no object; a file someone else changed since is left as named.
newer := []byte(strings.Repeat("the export the node wrote since\n", 3000))
newerName, _ := declaration.ObjectName(declaration.DigestOf(newer))
ctx = link.WithFiles(context.Background(), aFilesLink{objects: map[string][]byte{newerName: newer}})
if report = applyAndKeep(ctx, opts, referenceDeclaration(target, newer), nil, nil, nil); report.Refused != "" {
t.Fatalf("924: the newer export: %s", report.Refused)
}
stale := json.RawMessage(`{"id":"m.big","type":"file","path":"` + target + `","digest":"` + declaration.DigestOf(body) +
`","size":` + strconv.Itoa(len(body)) + `}`)
for _, row := range []struct {
name string
edit []byte // what someone else wrote there since; nil for nobody
digest string
}{
{"the file is what the node wrote", nil, declaration.DigestOf(newer)},
{"someone changed the file since", []byte("a hand edit"), declaration.DigestOf(body)},
} {
if row.edit != nil {
if err := os.WriteFile(target, row.edit, 0o600); err != nil {
t.Fatal(err)
}
}
runs := runsAsWritten([]delivery.Composed{{Node: "n", Module: "m", Resources: []json.RawMessage{stale}}}, opts.state)
var got struct {
Digest string `json:"digest"`
}
_ = json.Unmarshal(runs[0].Resources[0], &got)
if got.Digest != row.digest {
t.Errorf("924: %s: what runs names %s, want %s", row.name, got.Digest, row.digest)
}
if row.edit == nil {
raw := []byte(`{"declaration":1,"resources":[` + string(runs[0].Resources[0]) + `]}`)
if report = applyAndKeep(link.WithFiles(context.Background(), gone), opts, raw, nil, nil, nil); report.Refused != "" {
t.Errorf("924: %s: composed as it runs, refused %q; want applied with no object", row.name, report.Refused)
}
}
}
}
// 1695/AC8: the hand-run `apply <file>` and `reconcile` have no bus: a reference whose file here verifies is applied
cmd/mesh-host/filling.go
+76 -10
View File
@@ -3,22 +3,25 @@ package main
import (
"context"
"crypto/ed25519"
"encoding/json"
"errors"
"fmt"
"path/filepath"
"sync/atomic"
"git.novox.be/novox/mesh-sdk/go/declaration"
"git.novox.be/novox/mesh-sdk/go/provisions"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/filling"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
// serveFilling writes filled files on the provisioner client's request for as long as this engine serves (novox/hq
// ADR 0349 §4). A socket that cannot be served is said once, and this engine goes on applying.
func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, say func(string)) {
e := fillingEngine(opts, signer, say)
func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, queue *link.Session, node string, say func(string)) {
e := fillingEngine(opts, signer, provisionSecretsOn(queue, node, opts.state), say)
fillingNow.Store(e)
defer fillingNow.CompareAndSwap(e, nil)
if err := filling.Serve(ctx, filling.Socket, e, say); err != nil {
@@ -30,9 +33,10 @@ func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, s
var fillingNow atomic.Pointer[filling.Engine]
// knockFilling tells the provisioner's client to fill again what the filling refused or failed and the declaration
// applied now may let through (filling.Engine.Knock), once a module's apply or removal settled (agent.Env.Settled).
// What it told, or why it could not, is said.
func knockFilling(module string, say func(string)) {
// applied now may let through (filling.Engine.Knock), after what runs changed: a module's apply or removal settled
// (agent.Env.Settled, "<module> settled"), the haves were read at the start, or a module was taken as running
// (agent.Env.RunsChanged, issue 924). What it told, or why it could not, is said.
func knockFilling(after string, say func(string)) {
e := fillingNow.Load()
if e == nil {
return
@@ -40,16 +44,19 @@ func knockFilling(module string, say func(string)) {
told, err := e.Knock(filling.Socket)
switch {
case err != nil:
say(fmt.Sprintf("after %s settled: %v", module, err))
say(fmt.Sprintf("after %s: %v", after, err))
case told != "":
say(fmt.Sprintf("after %s settled, the provisioner's client is told to fill again: %s", module, told))
say(fmt.Sprintf("after %s, the provisioner's client is told to fill again: %s", after, told))
}
}
// fillingEngine is the write of a filled file as this engine does it: the declaration it keeps, verified, read under
// the apply's lock — the same two locks every apply takes, the process's and the one beside the state — and the file
// written through the apply (apply.WriteFilled), its outcome recorded in the node's state as an apply's is.
func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *filling.Engine {
//
// A provision's `${secret:}` the client left in the content is filled first (secrets, novox/hq ADR 0570 clause 6), and a
// file so filled is written 0600 when the declaration gives no mode.
func fillingEngine(opts options, signer ed25519.PublicKey, secrets provisionSecrets, say func(string)) *filling.Engine {
return &filling.Engine{
Declared: func() (*declaration.Declaration, error) {
return declaredNow(opts.state, signer)
@@ -73,8 +80,13 @@ func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *fi
if err != nil {
return "", "", err
}
report, updated, writeErr := apply.WriteFilled(ctx, built, d, known, path, content, apply.ExecRunner,
announceOr(say), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state)))
content, secret, err := secrets.fill(ctx, filledModule(d, path), content)
if err != nil {
return "", "", fmt.Errorf("%s was not written: %w", path, err)
}
report, updated, writeErr := apply.WriteFilledWith(ctx, built, d, known, path, content, apply.ExecRunner,
announceOr(say), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state)),
apply.Filled{Secret: secret})
// Saved whichever way it went, as an apply's: what was written is on the machine either way.
if err := store.Save(opts.state, updated); err != nil {
return "", "", errors.Join(writeErr, errors.New("written, and the node's state could not be saved: "+
@@ -93,3 +105,57 @@ func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *fi
},
}
}
// filledModule is the module whose file the declaration declares at path, or "" when none or several do.
func filledModule(d *declaration.Declaration, path string) string {
module := ""
for _, r := range d.Resources {
if f, ok := r.(*declaration.File); ok && f.Path == path {
if module != "" {
return ""
}
module, _ = d.ModuleOfID(f.ID)
}
}
return module
}
// provisionSecretsOn reads provisions and asks the vault over the link, and opens with this machine's sealing key.
func provisionSecretsOn(queue *link.Session, node, statePath string) provisionSecrets {
p := provisionSecrets{node: node, open: sealOpener(statePath)}
if queue == nil {
return p
}
// What the bus holds under prefix, read by a watch to its caught-up mark: no poll, no list of every key.
p.list = func(ctx context.Context, prefix string) (map[string][]byte, error) {
wctx, cancel := context.WithCancel(ctx)
defer cancel()
updates, err := queue.KVWatch(wctx, provisions.ProvisionsBucket, prefix+">")
if err != nil {
return nil, err
}
out := map[string][]byte{}
for {
select {
case <-ctx.Done():
return nil, ctx.Err()
case u, ok := <-updates:
if !ok {
return nil, fmt.Errorf("the watch of %s ended before it caught up", provisions.ProvisionsBucket)
}
switch {
case u.Caught:
return out, nil
case u.Deleted:
delete(out, u.Key)
default:
out[u.Key] = u.Value
}
}
}
}
p.own = func(ctx context.Context, args map[string]any) (json.RawMessage, error) {
return queue.AskVault(ctx, "own", args)
}
return p
}
cmd/mesh-host/filling_test.go
+102 -1
View File
@@ -7,9 +7,12 @@ import (
"os"
"os/user"
"path/filepath"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/provisions"
"github.com/novox/mesh-host/internal/filling"
"github.com/novox/mesh-host/internal/store"
)
@@ -45,7 +48,34 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) {
store.Declared{Declaration: raw, Signature: ed25519.Sign(private, raw)}); err != nil {
t.Fatal(err)
}
e := fillingEngine(opts, public, func(string) {})
// The provision m reads, as mesh-provisioner keeps it, and the vault's value of it, sealed to this machine (faked:
// the opener reads the box's "v").
provision := map[string]string{}
vaultName := "provision/n2/db"
secrets := provisionSecrets{node: "n1",
list: func(_ context.Context, prefix string) (map[string][]byte, error) {
out := map[string][]byte{}
for k, v := range provision {
if strings.HasPrefix(k, prefix) {
out[k] = []byte(v)
}
}
return out, nil
},
own: func(_ context.Context, args map[string]any) (json.RawMessage, error) {
raw, _ := json.Marshal(args)
var a struct{ Credentials []map[string]any }
_ = json.Unmarshal(raw, &a)
c := a.Credentials[0]
c["name"], c["generation"], c["sealed"] = vaultName, 2, map[string]string{"v": "s3cret"}
return json.Marshal(map[string]any{"node": "n1", "credentials": []any{c}})
},
open: func(sealed string) ([]byte, error) {
var b struct{ V string }
err := json.Unmarshal([]byte(sealed), &b)
return []byte(b.V), err
}}
e := fillingEngine(opts, public, secrets, func(string) {})
uid := uint32(os.Getuid())
if err := e.Accepts(uid); err != nil {
t.Fatal(err)
@@ -102,4 +132,75 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) {
if a := <-done; a.Action != "updated" {
t.Fatalf("the second write: %+v", a)
}
// ADR 0570 clause 6: the client leaves a provision's `${secret:}` to the node-engine, which fills it at the write
// from the vault's value of a provided provision, and writes the file 0600; a provision not provided refuses it.
keyed := func(state string, ref bool, id, local string) string {
p := provisions.Provision{Provision: "db", Local: local, Consumer: provisions.End{Module: "m", Node: "n1"},
Provider: provisions.End{Module: "postgres", Node: "n2"}, State: state, At: time.Now()}
if state != provisions.Provided {
p.Why = "the provider has not answered"
}
if ref {
p.Reference = &provisions.Reference{ID: id, Generation: 2}
}
raw, err := provisions.Encode(p)
if err != nil {
t.Fatal(err)
}
return string(raw)
}
for _, tc := range []struct {
name, state string
ref bool
content string
want string // the file after; "" keeps what was there
mode os.FileMode
vault string // the vault's credential name, when not the one the provision names
refID string // the reference's id, when not the provision's shared value
local string // the provision's local name (ADR 0094): the placeholder names it
}{
{name: "a provision not provided is refused", state: provisions.Asked, content: "password=${secret:db}\n"},
{name: "provided with no reference is refused", state: provisions.Provided, content: "password=${secret:db}\n"},
{name: "a reference to no credential of the provision is refused", state: provisions.Provided, ref: true,
refID: "provision/n9/db", content: "password=${secret:db}\n"},
{name: "a name no provision goes by is refused", state: provisions.Provided, ref: true, content: "password=${secret:other}\n"},
{name: "the vault's value is another than the provision names", state: provisions.Provided, ref: true,
vault: "pair/n2/db/n1/m", content: "password=${secret:db}\n"},
{name: "provided: filled at the write, 0600", state: provisions.Provided, ref: true,
content: "password=${secret:db}\n", want: "password=s3cret\n", mode: 0o600},
{name: "a local name is the provision's by its key", state: provisions.Provided, ref: true, local: "admin",
content: "admin=${secret:admin}\n", want: "admin=s3cret\n", mode: 0o600},
{name: "no placeholder is written as it came, as the declaration says", content: "plain\n", want: "plain\n", mode: 0o644},
} {
before, _ := os.ReadFile(path)
if tc.state != "" {
id := "provision/n2/db"
if tc.refID != "" {
id = tc.refID
}
clear(provision)
key := "n1.m.db"
if tc.local != "" {
key += "." + tc.local
}
provision[key] = keyed(tc.state, tc.ref, id, tc.local)
}
vaultName = "provision/n2/db"
if tc.vault != "" {
vaultName = tc.vault
}
a := e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte(tc.content)})
got, _ := os.ReadFile(path)
if tc.want == "" {
if a.Failed == "" && a.Refused == "" || string(got) != string(before) {
t.Errorf("%s: answered %+v, the file holds %q", tc.name, a, got)
}
continue
}
info, _ := os.Stat(path)
if a.Failed != "" || a.Refused != "" || string(got) != tc.want || info.Mode().Perm() != tc.mode {
t.Errorf("%s: answered %+v, the file holds %q, mode %v", tc.name, a, got, info.Mode())
}
}
}
cmd/mesh-host/found.go
+33 -14
View File
@@ -68,14 +68,23 @@ func foundDirectories(known store.State) []store.Applied {
// known (novox/hq ADR 0272): typed on the control-node, from the operator's own account. The directory's
// path is the resource's target beside the reason, never in it: the controller says a reason on the
// operator's channel, which withholds a path (ADR 0234 §6), and keeps the target as evidence.
//
// From the lease write, heard as the engine hears it (meshTrust), the line has no answerer: the way out is the node
// seat's hand-over with the directory's path, which asks the operator on their channel (novox/hq ADR 0563 §5).
func handOverLine() string {
node, _ := foundNode.Load().(string)
if node == "" {
node = "<node>"
}
if t := foundTrust.Load(); t != nil && t.HandedAway() {
return fmt.Sprintf("`%s/node-engine.hand-over` with the directory's path, then your yes on your channel", node)
}
return fmt.Sprintf("`nox node hand-over %s <directory>` on the control-node", node)
}
// foundTrust is what the engine trusts, whose lease decides the way out a directory used as found names.
var foundTrust atomic.Pointer[meshTrust]
// withFoundDirectories adds to a statement every directory used as found, as its module's resource of kind
// directory, unhealthy: the reason names its owner and mode as they are now, what was declared, and the line
// that hands it over. The path is the target, which the controller keeps as evidence.
@@ -129,16 +138,30 @@ func foundNow() []store.Applied {
// never follows a path it did not state: a hand-over is root giving a directory to an account at the next
// apply, and a path that resolves elsewhere would give that elsewhere.
func handOverFound(statePath, path, by string) (string, error) {
found, _, err := judgeFound(statePath, path)
if err != nil {
return "", err
}
if err := store.AddHandedOver(statePath, store.HandOver{ID: found.ID, Path: path, At: time.Now().UTC(), By: by}); err != nil {
return "", err
}
return fmt.Sprintf("%s (%s) is handed to the mesh by %s: the next apply gives it %s, and it is the mesh's from "+
"then on — removed when empty and no longer declared", path, found.ID, by, orUnsaid(found.Declared)), nil
}
// judgeFound is the judgement of a hand-over's path, recording nothing: the directory this machine uses as found at
// that path, as the engine states it, and what is on disk there.
func judgeFound(statePath, path string) (store.Applied, os.FileInfo, error) {
if !filepath.IsAbs(path) {
return "", fmt.Errorf("hand-over takes the directory's absolute path, and %q is not one; nothing was handed over", path)
return store.Applied{}, nil, fmt.Errorf("hand-over takes the directory's absolute path, and %q is not one; nothing was handed over", path)
}
if filepath.Clean(path) != path {
return "", fmt.Errorf("%q is not the directory's path as the engine states it (no `..`, no doubled or trailing "+
return store.Applied{}, nil, fmt.Errorf("%q is not the directory's path as the engine states it (no `..`, no doubled or trailing "+
"separator); nothing was handed over", path)
}
known, err := store.Load(statePath)
if err != nil {
return "", err
return store.Applied{}, nil, err
}
var found *store.Applied
for _, r := range foundDirectories(known) {
@@ -149,31 +172,27 @@ func handOverFound(statePath, path, by string) (string, error) {
}
}
if found == nil {
return "", errors.New(path + " is not a directory this machine uses as found (the module's condition names " +
return store.Applied{}, nil, errors.New(path + " is not a directory this machine uses as found (the module's condition names " +
"the ones that are); nothing was handed over")
}
info, err := os.Lstat(path)
switch {
case err != nil:
return "", fmt.Errorf("%s cannot be read on this machine: %v; nothing was handed over", path, err)
return store.Applied{}, nil, fmt.Errorf("%s cannot be read on this machine: %v; nothing was handed over", path, err)
case info.Mode()&os.ModeSymlink != 0:
return "", fmt.Errorf("%s is a link, and the mesh hands over no link (novox/hq ADR 0012); nothing was handed over", path)
return store.Applied{}, nil, fmt.Errorf("%s is a link, and the mesh hands over no link (novox/hq ADR 0012); nothing was handed over", path)
case !info.IsDir():
return "", fmt.Errorf("%s is not a directory on this machine; nothing was handed over", path)
return store.Applied{}, nil, fmt.Errorf("%s is not a directory on this machine; nothing was handed over", path)
}
resolved, err := filepath.EvalSymlinks(path)
if err != nil {
return "", fmt.Errorf("%s cannot be resolved on this machine: %v; nothing was handed over", path, err)
return store.Applied{}, nil, fmt.Errorf("%s cannot be resolved on this machine: %v; nothing was handed over", path, err)
}
if resolved != path {
return "", fmt.Errorf("%s is reached through a link (it is %s), and the mesh hands over nothing through one "+
return store.Applied{}, nil, fmt.Errorf("%s is reached through a link (it is %s), and the mesh hands over nothing through one "+
"(novox/hq issue 339); nothing was handed over", path, resolved)
}
if err := store.AddHandedOver(statePath, store.HandOver{ID: found.ID, Path: path, At: time.Now().UTC(), By: by}); err != nil {
return "", err
}
return fmt.Sprintf("%s (%s) is handed to the mesh by %s: the next apply gives it %s, and it is the mesh's from "+
"then on — removed when empty and no longer declared", path, found.ID, by, orUnsaid(found.Declared)), nil
return *found, info, nil
}
// handOverFrom is the engine's hand-over for the controller's ask: the state beside statePath, the path as asked.
cmd/mesh-host/found_test.go
+15
View File
@@ -40,6 +40,21 @@ func TestAFoundDirectoryIsStatedUnhealthy(t *testing.T) {
strings.Contains(r.Reason, "mesh-host") || strings.Contains(r.Reason, dir) {
t.Fatalf("the way out is not the operator's line, or carries the path: %q", r.Reason)
}
// ADR 0563 §5: from the lease write the remedy is the node seat's hand-over, answered on the operator's channel, and
// no longer the controller's terminal line; the lease is the one the engine hears.
handed := newMeshTrust(nil)
handed.handedAway = true
foundTrust.Store(handed)
if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason,
"`laptop/node-engine.hand-over`") || strings.Contains(r.Reason, "nox node hand-over") || strings.Contains(r.Reason, dir) {
t.Fatalf("after the lease write the way out is %q", r.Reason)
}
foundTrust.Store(newMeshTrust(nil))
if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason,
"`nox node hand-over laptop <directory>`") {
t.Fatalf("while the controller holds its lease the way out is %q", r.Reason)
}
foundTrust.Store((*meshTrust)(nil))
foundNode.Store("")
if r := withFoundDirectories(&link.Health{}, foundDirectories(known)).Resources[0]; !strings.Contains(r.Reason,
"`nox node hand-over <node> <directory>`") {
cmd/mesh-host/libraries_test.go
+40
View File
@@ -8,6 +8,8 @@ import (
"testing"
"time"
meshasker "git.novox.be/novox/mesh-sdk/go/asker"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/declaration"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/link"
@@ -53,4 +55,42 @@ func TestALibrarySaysItsWaitWithoutWordsToParse(t *testing.T) {
if strings.Contains(media.Reason, dir) || len(books.AccessWaits) != 1 || books.AccessWaits[0].Reason != link.WaitDisk {
t.Fatalf("a missing root does not wait for its disk, or a reason names a path: %+v %+v", books, media)
}
// 774 (ADR 0563 §4): the library's home engine asks its take of the operator itself, once: no second ask while one is
// open, nor after it ended; and a library waiting for its disk is not asked.
t.Run("the home engine asks the take once", func(t *testing.T) {
var asked []asks.Ask
state := &link.MemState{}
h := &link.HandActs{Node: "n1", State: state,
JudgeAccess: func(act, access string) (link.Judged, error) {
return link.Judged{Subject: access, Path: root, Sets: "the group mesh-media", Disk: "0:0 755"}, nil
},
AccessAct: func(link.AccessAct) (string, error) { return "taken", nil }}
h.Asker = &meshasker.Client{Module: asks.EngineAsker("n1"), State: state,
Publish: func(subject string, body any, id string) error {
if a, ok := body.(asks.Ask); ok {
asked = append(asked, a)
}
return nil
}}
waiting := keepWaiting(vs)
askTakes(h, waiting, nil)
askTakes(h, waiting, nil)
if len(waiting) != 1 || waiting[0] != "library:media" || len(asked) != 1 {
t.Fatalf("waiting %v asked %d times", waiting, len(asked))
}
h.Heard(asks.Warrant{Ask: asked[0].ID, Asker: asks.EngineAsker("n1"), Outcome: asks.OutcomeExpired, At: time.Now()})
askTakes(h, waiting, nil)
if len(asked) != 1 {
t.Fatalf("the take was asked again after its ask ended: %d", len(asked))
}
// Kept in a key of its own (rev-l3 round 1): the acts gone from the record never make it ask again.
keys, _ := state.Keys("act.")
for _, k := range keys {
_ = state.Delete(k)
}
askTakes(h, waiting, nil)
if len(asked) != 1 {
t.Fatalf("the take was asked again with the acts gone: %d", len(asked))
}
})
}
cmd/mesh-host/main.go
+72 -8
View File
@@ -516,7 +516,7 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, raw
// A file given by digest is had before the plan is drawn from it (novox/hq ADR 0486 §1, issue 1695). This command has
// no link to the bus, so a file already here that verifies is used, and any other is refused: never written empty.
if err := apply.ResolveFileReferences(ctx, d, link.FilesFrom(ctx)); err != nil {
if err := apply.ResolveFileReferencesKept(ctx, d, link.FilesFrom(ctx), objectKeep(opts)); err != nil {
return fmt.Errorf("a file of this declaration could not be had by its digest: %w", err)
}
@@ -1105,10 +1105,17 @@ func runLink(ctx context.Context, opts options) error {
// the bus, answer what the controller asks of this machine, and say its facts: when a link opens, and when the
// controller asks.
proof := &proving{statePath: opts.state, version: runningVersion(), say: say}
// What this engine trusts a mesh signature from: the controller's key until its lease is handed away, and the vault's
// once read from its signers (novox/hq ADR 0550 §2).
trust := newMeshTrust(mine.Membership.Signer)
// And the way out a directory used as found names follows the lease it hears (novox/hq ADR 0563 §5).
foundTrust.Store(trust)
queue := &link.Session{
Membership: membership,
// And the public halves and certificate its record says (novox/hq ADR 0555 §3), read at each facts report.
Keys: recordKeys(opts.state),
Keys: recordKeys(opts.state, trust.Trusts),
// And the controller's key, read per signed ask: none once its lease is handed away (novox/hq ADR 0563 §7).
ControllerKey: trust.Controller,
// And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8).
HaveWritten: proof.haveWritten,
// And the operator's hand-over of a directory used as found, carried by the controller from the
@@ -1129,6 +1136,10 @@ func runLink(ctx context.Context, opts options) error {
Say: say,
Timeout: opts.timeout,
}
// And the node seat node-engine this engine holds (novox/hq ADR 0563): its verbs on every link, its own asks to the
// operator under node-engine-<node>, and its acts kept beside its state.
queue.Acts = engineActs(opts, mine.Node, queue, setuidSearch)
go reconcileActs(ctx, queue.Acts)
current := &agentNow{}
setRunsHere(runsFromAgent(current))
factsOfNow := func(ctx context.Context, ag *agent.Agent) {
@@ -1173,7 +1184,10 @@ func runLink(ctx context.Context, opts options) error {
}
// The node follows its wants: one module at a time from the composition its want names (novox/hq ADR 0539), and the
// busy containers are asked for again soon (ADR 0360 §11).
go nodeAgent(aside, opts, mine.Membership.Signer, queue, sched, say, afterApply, current, factsOfNow)
// And what it trusts, followed on the bus, said in its record whenever it changes (ADR 0550 §2).
trust.onChange(func() { factsOfNow(aside, current.get()) })
go followTrust(aside, queue, trust, say)
go nodeAgent(aside, opts, mine.Membership.Signer, trust, queue, sched, say, afterApply, current, factsOfNow)
go busyPasses(aside, func() {
if ag := current.get(); ag != nil {
ag.Retry(busyModules()...)
@@ -1186,7 +1200,7 @@ func runLink(ctx context.Context, opts options) error {
go serveMeshCLI(aside, queue, say)
// And a filled file is written on the provisioner client's request, one at a time with the apply (novox/hq ADR
// 0349 §4).
go serveFilling(aside, opts, mine.Membership.Signer, say)
go serveFilling(aside, opts, mine.Membership.Signer, queue, mine.Node, say)
held := link.HoldRoused(aside, membership, queue, say, opts.timeout, rousedBySignal(ctx))
// **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back
@@ -1339,6 +1353,8 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
var lastSaid time.Time
owed := true
spaced := 1.0
// healthUnwritten is why the health report was last not written, said once each time it changes.
var healthUnwritten string
// A module whose change ended unjudged is looked at and said at once (review of fix 15): a module with nothing
// long-running is then ok within a look of its apply, not at the five-minute statement.
var nudged <-chan struct{}
@@ -1370,6 +1386,7 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
if a := accountJudge; a != nil {
as, accountsChanged := a.Look(ctx)
accountSt = &as
lastAccounts.Store(&as)
owed = owed || accountsChanged
if accountsChanged {
for _, v := range as.Accounts {
@@ -1384,6 +1401,9 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
accessSt, accessesChanged := accesses.look(time.Now())
keeping.poke(ctx)
owed = owed || accessesChanged
// And a library whose root holds data outside its group is asked of the operator by this engine, once (novox/hq
// ADR 0563 §4).
askTakes(queue.Acts, keepWaiting(accessSt), say)
if accessesChanged {
for _, v := range accessSt {
if v.State != link.StateHealthy {
@@ -1418,6 +1438,11 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
healthy := st.Healthy() && (netSt == nil || netSt.State != network.Unhealthy) &&
(unitSt == nil || len(unitSt.Failed) == 0) && (accountSt == nil || accountSt.Healthy()) && len(found) == 0 &&
accessesHealthy(accessSt)
h := *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt),
unitSt), accountSt), found), accessSt)
// The node's own health report in its key on the bus, compared on every look and written only when what it says
// changed (issue 774): what the watchman raises the machine's alerts from, without the controller.
writeHealthReport(ctx, queue, h, netSt, &healthUnwritten, say)
if !owed && !(!healthy && since >= sayUnhealthyAgain) && since < sayAnyway {
continue
}
@@ -1444,13 +1469,40 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Session,
reconcile(inactive...)
}
}
if queue.SayHealth(ctx, *withAccesses(withFoundDirectories(withAccounts(withUnits(healthAsReported(st, netSt),
unitSt), accountSt), found), accessSt)) {
if queue.SayHealth(ctx, h) {
lastSaid, owed = time.Now(), false
}
}
}
// writeHealthReport writes this node's health report to its key (link.WriteNodeHealth) when it changed. Why it could
// not is said once per reason, and said again when it is written after all.
func writeHealthReport(ctx context.Context, queue *link.Session, h link.Health, ns *network.Statement, unwritten *string,
say link.Announce) {
if queue == nil {
return
}
var peers []link.PeerSeen
if ns != nil {
for _, p := range ns.Peers {
peers = append(peers, link.PeerSeen{Key: p.Key, Addresses: p.Addresses, Seen: p.Seen})
}
}
var machines link.Machines
if c := machinesNow.Load(); c != nil {
machines.ByAddress, machines.Hub = c.Machines()
}
_, err := queue.WriteNodeHealth(ctx, link.NodeHealthOf(queue.Membership.Node, h, peers, machines))
switch {
case err != nil && err.Error() != *unwritten:
*unwritten = err.Error()
say("this machine's health report is not written to the bus: " + err.Error())
case err == nil && *unwritten != "":
*unwritten = ""
say("this machine's health report is written to the bus again")
}
}
// healthAsReported is a statement as the report and the event carry it.
func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health {
// LibraryContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a
@@ -1646,6 +1698,15 @@ func announceOr(say link.Announce) func(string) {
// run by hand beside this service — the lock beside the state does the same (store.Lock).
var applying sync.Mutex
// objectKeep is the node's own keep of the objects its compositions name by digest, beside its state (apply.ObjectKeep,
// issue 924): a composition the node once applied never again needs the controller's bucket to hold its files.
func objectKeep(opts options) *apply.ObjectKeep {
if opts.state == "" {
return nil
}
return apply.KeepObjectsIn(filepath.Join(filepath.Dir(opts.state), "objects"))
}
// appliedModule is how the apply of one module's declaration came out.
type appliedModule struct {
// Refused is why nothing was applied, when nothing was.
@@ -1685,7 +1746,7 @@ func applyModuleHeld(ctx context.Context, opts options, declared *declaration.De
// **A file given by digest is fetched and verified before anything of the composition is applied** (novox/hq ADR
// 0486 §1, issue 1695): one object missing or differing refuses the module whole.
if err := apply.ResolveFileReferences(ctx, declared, link.FilesFrom(ctx)); err != nil {
if err := apply.ResolveFileReferencesKept(ctx, declared, link.FilesFrom(ctx), objectKeep(opts)); err != nil {
return appliedModule{Refused: "a file of this composition could not be had by its digest: " + err.Error()}
}
@@ -2167,9 +2228,12 @@ func withMissing(st liveness.Statement) liveness.Statement {
// recordKeys reads the public halves of the node's keys and the certificate the mesh issued for its internal name from
// its state directory (novox/hq ADR 0555 §3): what the node's record says of them. A key that cannot be read is left
// empty, never invented; the certificate is there only after the first declaration that carried it.
func recordKeys(state string) func() link.RecordKeys {
func recordKeys(state string, trusts func() []string) func() link.RecordKeys {
return func() link.RecordKeys {
var k link.RecordKeys
if trusts != nil {
k.Trusts = trusts()
}
if id, err := identity.Load(identity.Path(state)); err == nil {
k.TunnelKey = id.Overlay.Public
}
cmd/mesh-host/nodeagent.go
+85 -22
View File
@@ -5,12 +5,14 @@ import (
"crypto/ed25519"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
@@ -343,13 +345,14 @@ func resourcesOf(cs []delivery.Composed) []declaration.Resource {
// controller's key the node was enrolled with (novox/hq ADR 0539 §3, §11); a node adopted later has it at its adoption.
const operatorKeysFile = "operator-keys"
// trustedKeys are the keys a composition may be signed by: the controller's, which the node was enrolled with, and the
// operator's, read from operatorKeysFile each time it is asked, so a key that arrives while the agent runs is trusted at
// the next composition. A file that cannot be read trusts the controller's alone, and says so.
func trustedKeys(opts options, controller ed25519.PublicKey) func() []ed25519.PublicKey {
// trustedKeys are the keys a composition may be signed by: the controller's, which the node was enrolled with, while the
// controller holds its lease (ADR 0550 §2: trusted no more once it is handed away), and the operator's, read from
// operatorKeysFile each time it is asked, so a key that arrives while the agent runs is trusted at the next composition.
// A file that cannot be read trusts the controller's alone, and says so.
func trustedKeys(opts options, controllerNow func() ed25519.PublicKey) func() []ed25519.PublicKey {
return func() []ed25519.PublicKey {
keys := []ed25519.PublicKey{}
if len(controller) == ed25519.PublicKeySize {
if controller := controllerNow(); len(controller) == ed25519.PublicKeySize {
keys = append(keys, controller)
}
raw, err := os.ReadFile(filepath.Join(filepath.Dir(opts.state), operatorKeysFile))
@@ -373,13 +376,16 @@ func trustedKeys(opts options, controller ed25519.PublicKey) func() []ed25519.Pu
}
// keptOnce reads the kept declaration once and splits it by module: its resources (the node's own are those that say
// none, ADR 0539 §9) and its filled paths.
// none, ADR 0539 §9) and its filled paths. Once the read is retired (keptRetired) nothing is read.
func keptOnce(opts options, signer ed25519.PublicKey) func() (map[string][]json.RawMessage, map[string][]string) {
var once sync.Once
var by map[string][]json.RawMessage
var filled map[string][]string
return func() (map[string][]json.RawMessage, map[string][]string) {
once.Do(func() {
if keptRetired.Load() {
return
}
raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer)
if err != nil {
return
@@ -429,7 +435,7 @@ func splitKept(raw []byte) (map[string][]json.RawMessage, map[string][]string) {
// nodeAgent runs the node agent under the session's links for as long as ctx lasts: it waits for a link, follows the
// node's wants and compositions on it, and begins again with the next link. A watch it cannot make (a bucket this node's
// account may not read) is said, and tried again with a growing wait, as the link is.
func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, session *link.Session, sched *apply.Scheduler,
func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trust *meshTrust, session *link.Session, sched *apply.Scheduler,
say link.Announce, afterApply func(), current *agentNow, sayFacts func(ctx context.Context, ag *agent.Agent)) {
wait := 2 * time.Second
for ctx.Err() == nil {
@@ -446,6 +452,18 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
return
}
var ag *agent.Agent
// The kept release-5 declaration is no longer read once every module the seat's placement puts here has a have of its
// own (ADR 0550 §2: its signature is the controller's, trusted no more after the lease write).
// Nor before this engine keeps the node's own as it runs (applied/node.json): until then the kept declaration is the one
// record of what the node's own composition carries, its folded modules among them (issue 774, review of the
// re-land, round 1: novox's first start would compose its own with nats running nothing).
if retired, why := placedAllHave(linked, session, node); retired && !keptRetired.Load() {
if !appliedStoreFor(opts.state).holds(agent.NodeModule) {
say("the kept declaration is still read: " + why + ", but this engine keeps no node's own composition yet")
} else if !keptRetired.Swap(true) {
say("the kept declaration is no longer read: " + why)
}
}
kept := keptOnce(opts, signer)
// The mesh's own certificate the declaration kept from before carries, kept for the record before this machine
// composes its own (ADR 0555 §3): without it a module that asks for a certificate would be stood in for.
@@ -458,7 +476,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
}
// The bus's user list, written alone from the newest composition of the bus module (ADR 0547 §8): on every
// node, and a node whose compositions carry no list writes nothing.
users := userListFollower(opts, session, composedBucket, trustedKeys(opts, signer), say, func() {
users := userListFollower(opts, session, composedBucket, trustedKeys(opts, trust.Controller), say, func() {
// This engine wrote the list, so the bus reloads its accounts and answers no JetStream request for a few
// seconds: what failed in them is tried again once that is over, not at its next wait.
time.AfterFunc(reloadSettles, func() {
@@ -483,7 +501,10 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
// The same kept declaration's filled paths, so a module still running from it keeps its files filled.
KeptFilled: func() map[string][]string { _, filled := kept(); return filled },
Recorded: recordedIn(opts.state, say),
Inactive: inactiveIn(&liveness.Exec{Run: apply.ExecRunner, System: builtFor}, heldBy(apply.WindowsIn(filepath.Dir(opts.state)))),
// What this engine applied, kept on the machine and read at the start, so what runs is known before the node
// composes its own (issue 774).
Applied: appliedStoreFor(opts.state),
Inactive: inactiveIn(&liveness.Exec{Run: apply.ExecRunner, System: builtFor}, heldBy(apply.WindowsIn(filepath.Dir(opts.state)))),
Revisions: func(ctx context.Context, key string) ([]agent.Update, error) {
got, err := session.KVHistory(ctx, composedBucket, key)
if err != nil {
@@ -495,12 +516,15 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
}
return out, nil
},
Trusted: trustedKeys(opts, signer),
Trusted: trustedKeys(opts, trust.Controller),
Backup: backupNode(session),
// What the filling refused before this apply may be named now: the provisioner's client is told to fill
// again (ace, 2026-10-25 21:17).
Settled: func(module string) { knockFilling(module, say) },
Say: say,
Settled: func(module string) { knockFilling(module+" settled", say) },
// And once the haves are read at the start, and whenever a module is taken as running without an apply, so
// a refusal from the engine's start does not wait for some module to settle (issue 924).
RunsChanged: func(why string) { knockFilling(why, say) },
Say: say,
// A failure of the bus not answering for a moment is tried again: 2 s, 5 s, 15 s, 60 s, then every 5 min.
Backoff: link.Backoff,
Apply: func(ctx context.Context, c delivery.Composed, others []delivery.Composed) agent.Result {
@@ -512,7 +536,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
if herr != nil {
res = failedWith(herr.Error())
} else {
res = applyComposition(actx, opts, held, others, sched, say)
res = applyComposition(actx, opts, held, others, ag.Placed(scopeName(c)), ag.RemovedByWant(), sched, say)
}
done()
// The mesh's own certificate it carries is kept for this machine's record (ADR 0555 §3).
@@ -531,7 +555,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
},
Remove: func(ctx context.Context, module string, others []delivery.Composed) agent.Result {
actx, done := applyContext(session.WithLink(ctx))
res := removeComposition(actx, opts, module, others, sched, say)
res := removeComposition(actx, opts, module, others, ag.Placed(module), sched, say)
done()
afterApply()
sayFacts(ctx, ag)
@@ -549,7 +573,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
hooks := composerHooks{
running: func() []delivery.Composed {
if now := own.get(); now != nil {
return now.Everything()
return runsAsWritten(now.Everything(), opts.state)
}
return nil
},
@@ -572,11 +596,17 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
now.NotComposedEach(run, why, held) // a module held alone marks that module, never the others (issue 1126)
}
},
// And the vault's authority beside the controller's, and this node's certificate asked of the vault when a
// module here asks for one (ADR 0550 §2).
vaultAuthority: trust.VaultAuthority,
certify: func(ctx context.Context) error {
return certifyThisNode(ctx, opts.state, node, trust, session.AskVault, func() { sayFacts(ctx, ag) })
},
}
// And its answer to every placement candidate, placed or not, applying nothing (novox/hq ADR 0559 §4): written into
// its own key, `<node>.node`, under candidates.
shadowHooks := hooks
shadowHooks.refused = nil
shadowHooks.refused, shadowHooks.certify = nil, nil
shadowEnv := composerEnv(session, node, say, shadowHooks)
// And, after each answer, what runs of an own secret the answer says the vault holds another value of, handed to
// the vault while the node applies the controller's compositions (issue 1126, cause C4; ownresync).
@@ -596,8 +626,10 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
continue
}
if composer != nil {
machinesNow.Store(composer) // the machines a failing network part points at, by the placements (issue 774)
composes.set(delivery.ComposesOwn)
env.Composed, env.SelfComposed, env.Trusted, env.Policy = composer, true, nil, composer.Policy
env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does
session.NoteComposes(linked, delivery.ComposesOwn, why)
// And once the catalogue no longer places it, the agent begins again on the controller's compositions: the
// placement key deleted ends this run, the way back (issue 1126, the seed's revert).
@@ -663,7 +695,13 @@ func (c *agentNow) get() *agent.Agent {
}
// applyComposition applies one composition: a module's, or the node's own.
func applyComposition(ctx context.Context, opts options, c delivery.Composed, others []delivery.Composed,
//
// placed is the modules a want still places here besides this one: none of their resources is removed by this apply,
// which is refused instead (issue 774).
//
// removed is the modules whose want says removed: in the node's own apply, a resource of any other module goes only when
// its module is among them (issue 774).
func applyComposition(ctx context.Context, opts options, c delivery.Composed, others []delivery.Composed, placed, removed map[string]bool,
sched *apply.Scheduler, say link.Announce) agent.Result {
name := scopeName(c)
if c.Module == "" {
@@ -679,7 +717,13 @@ func applyComposition(ctx context.Context, opts options, c delivery.Composed, ot
if err := refuseEmptyWrites(d); err != nil {
return failedWith(err.Error())
}
scope := apply.Scope{Module: name, Others: resourcesOf(others)}
scope := apply.Scope{Module: name, Others: resourcesOf(others), Placed: placed}
if name == agent.NodeModule {
scope.RemovedByWant = removed
if scope.RemovedByWant == nil {
scope.RemovedByWant = map[string]bool{}
}
}
got := applyModuleHeld(ctx, opts, d, &scope, sched, say)
if got.Refused != "" {
return failedWith(got.Refused)
@@ -698,9 +742,9 @@ func applyComposition(ctx context.Context, opts options, c delivery.Composed, ot
}
// removeComposition removes one module's own resources and nothing of another's.
func removeComposition(ctx context.Context, opts options, module string, others []delivery.Composed,
func removeComposition(ctx context.Context, opts options, module string, others []delivery.Composed, placed map[string]bool,
sched *apply.Scheduler, say link.Announce) agent.Result {
scope := apply.Scope{Module: module, Others: resourcesOf(others)}
scope := apply.Scope{Module: module, Others: resourcesOf(others), Placed: placed}
got := applyModuleHeld(ctx, opts, &declaration.Declaration{Version: declaration.Version}, &scope, sched, say)
switch {
case got.Refused != "":
@@ -725,8 +769,23 @@ func busyModules() []string {
return out
}
// keptDeclaration is the declaration the node kept before the switch, verified with the mesh's key when it is given.
// machinesNow is the composer whose placements name the mesh's machines by address, for the health report's targets
// (writeHealthReport); nil on a node that does not compose its own, whose parts then name no target.
var machinesNow atomic.Pointer[selfcompose.Composer]
// keptRetired says the kept declaration is read no more: every module the seat's placement puts on this node has a have of
// its own (placedAllHave), so what runs is the agent's alone (ADR 0550 §2, design 47's L3 s9).
var keptRetired atomic.Bool
// errKeptRetired is keptDeclaration's answer once the read is retired.
var errKeptRetired = errors.New("the kept declaration is read no more: every module placed on this node has a have of its own")
// keptDeclaration is the declaration the node kept before the switch, verified with the mesh's key when it is given;
// once the read is retired, nothing is read.
func keptDeclaration(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) {
if keptRetired.Load() {
return nil, errKeptRetired
}
var raw []byte
if len(signer) > 0 {
var err error
@@ -903,6 +962,9 @@ type composerHooks struct {
unseal func(string) ([]byte, error)
refused func(why string, held map[string]string)
held func(module string) (delivery.Seals, []string, error)
// vaultAuthority and certify are selfcompose.Env's VaultAuthority and Certify.
vaultAuthority func() string
certify func(ctx context.Context) error
}
func selfComposer(ctx context.Context, session *link.Session, node string, say func(string), hooks composerHooks) (*selfcompose.Composer, string) {
@@ -966,7 +1028,8 @@ func composerEnv(session *link.Session, node string, say func(string), hooks com
Hear: session.Hear,
// And a moved module's templates archive, in the artifact store, as every archive is fetched (issue 1126).
Fetch: apply.Fetch,
Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held}
Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held,
VaultAuthority: hooks.vaultAuthority, Certify: hooks.certify}
}
// endWhenUnplaced calls stop once this node's placement key is deleted from the seat's placement (after the watch
cmd/mesh-host/nodeagent_integration_test.go
+2 -1
View File
@@ -159,7 +159,8 @@ func (b *itBed) start(trustOperator bool) (stop func()) {
done := make(chan struct{})
go func() {
defer close(done)
nodeAgent(ctx, b.opts, b.controll.Public().(ed25519.PublicKey), b.session, nil, b.say, func() {},
controller := b.controll.Public().(ed25519.PublicKey)
nodeAgent(ctx, b.opts, controller, newMeshTrust(controller), b.session, nil, b.say, func() {},
&agentNow{}, func(context.Context, *agent.Agent) {})
}()
return func() { cancel(); <-done }
cmd/mesh-host/provision_secrets.go
+168
View File
@@ -0,0 +1,168 @@
package main
import (
"context"
"encoding/json"
"fmt"
"regexp"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/provisions"
)
// provisionSecrets fills a provision's value into a file the provisioner's client hands the node-engine to write
// (novox/hq ADR 0570 clause 6): the client fills the file but for each `${secret:<provision>}`, and the node-engine
// fills those at the write, so the client never holds the value. A whole `secrets` file of a handed consumer is the
// placeholder alone.
//
// A placeholder names the credential as the module calls it, `${secret:<name>}` (catalogue.SecretLocal, ADR 0094): the
// local name where the module keeps several values of one provision, else the provision. For each, it finds the one
// provision key `<node>.<consumer>.<provision>[.<local>]` of the seat mesh-provisioner's state `provisions` (clause 1)
// going by that name, which must be `provided` and carry the vault's reference; asks the vault for that pair
// credential sealed to this machine's key (the verb `own`, ADR 0349 §11 with the node-engine standing for the client,
// ADR 0550 §5); and opens it with this machine's sealing key. Anything short of that refuses the write, naming the
// provision and why: a file written with a placeholder left in it, or with no value, is a module that starts and cannot
// connect.
type provisionSecrets struct {
node string
// list is every key of mesh-provisioner's `provisions` under prefix (`<node>.<consumer>.`), by key: what this
// engine may read, its own node's (`<node>.>`).
list func(ctx context.Context, prefix string) (map[string][]byte, error)
// own asks the vault's verb `own` in this engine's name.
own func(ctx context.Context, args map[string]any) (json.RawMessage, error)
// open opens a value sealed to this machine.
open func(sealed string) ([]byte, error)
}
// secretPlaceholder is `${secret:<name>}`.
var secretPlaceholder = regexp.MustCompile(`\$\{secret:([A-Za-z0-9_-]+)\}`)
// fill is content with each provision placeholder of module's filled, and whether any was.
func (p provisionSecrets) fill(ctx context.Context, module string, content []byte) ([]byte, bool, error) {
found := secretPlaceholder.FindAllSubmatch(content, -1)
if len(found) == 0 {
return content, false, nil
}
if module == "" {
return nil, false, fmt.Errorf("the file names %d provision secret(s) and belongs to no module, so whose they are is not known", len(found))
}
if p.list == nil || p.own == nil || p.open == nil {
return nil, false, fmt.Errorf("%s's provisions cannot be read on this machine: the node-engine is not linked to the mesh", module)
}
prefix := p.node + "." + module + "."
keys, err := p.list(ctx, prefix)
if err != nil {
return nil, false, fmt.Errorf("%s's provisions cannot be read: %w", module, err)
}
out := string(content)
done := map[string]bool{}
for _, f := range found {
whole, name := string(f[0]), string(f[1])
if done[whole] {
continue
}
done[whole] = true
pr, key, err := byName(keys, name)
if err != nil {
return nil, false, fmt.Errorf("%s's secret %q: %w", module, name, err)
}
value, err := p.value(ctx, key, pr)
if err != nil {
return nil, false, err
}
out = strings.ReplaceAll(out, whole, string(value))
}
return []byte(out), true, nil
}
// byName is the one provision of keys the module calls name (catalogue.SecretLocal), and its key.
func byName(keys map[string][]byte, name string) (provisions.Provision, string, error) {
var hit []string
var got provisions.Provision
for key, raw := range keys {
pr, err := provisions.Decode(raw)
if err != nil {
return provisions.Provision{}, "", fmt.Errorf("the provision %s: %w", key, err)
}
if catalogue.SecretLocal(pr.Provision, pr.Local) == name {
hit, got = append(hit, key), pr
}
}
switch len(hit) {
case 0:
return provisions.Provision{}, "", fmt.Errorf("no provision of %s goes by that name: nothing asks for it", provisions.ProvisionsRead)
case 1:
return got, hit[0], nil
}
sort.Strings(hit)
return provisions.Provision{}, "", fmt.Errorf("the provisions %s all go by that name", strings.Join(hit, ", "))
}
// value is one provision's value on this node, opened.
func (p provisionSecrets) value(ctx context.Context, key string, pr provisions.Provision) ([]byte, error) {
if pr.State != provisions.Provided || pr.Reference == nil {
why := pr.State
if pr.Why != "" {
why += ": " + pr.Why
}
if pr.State == provisions.Provided {
why = "provided with no reference to the vault's value"
}
return nil, fmt.Errorf("the provision %s is %s, so its value is not written yet", key, why)
}
// The reference names a credential of this provision, as the vault names it (provisions.VaultName).
ours := false
for _, form := range []string{provisions.FormPair, provisions.FormShared, provisions.FormIssued} {
ours = ours || provisions.VaultName(form, pr) == pr.Reference.ID
}
if !ours {
return nil, fmt.Errorf("the provision %s names the vault's %s, which is no credential of it", key, pr.Reference.ID)
}
module, provision, local := pr.Consumer.Module, pr.Provision, pr.Local
type asked struct {
Role string `json:"role"`
Provision string `json:"provision"`
Consumer string `json:"consumer"`
ConsumerModule string `json:"consumer_module"`
Provider string `json:"provider"`
Local string `json:"local,omitempty"`
}
ask := asked{Role: "consumer", Provision: provision, Consumer: p.node, ConsumerModule: module, Provider: pr.Provider.Node, Local: local}
answer, err := p.own(ctx, map[string]any{"credentials": []asked{ask}})
if err != nil {
return nil, fmt.Errorf("the vault's value of %s: %w", key, err)
}
var a struct {
Node string `json:"node"`
Credentials []struct {
asked
Name string `json:"name"`
Generation int `json:"generation"`
Sealed json.RawMessage `json:"sealed"`
} `json:"credentials"`
}
if err := json.Unmarshal(answer, &a); err != nil {
return nil, fmt.Errorf("the vault's answer for %s is not one: %w", key, err)
}
if a.Node != p.node {
return nil, fmt.Errorf("the vault answered %s's credentials for %s on %s", a.Node, key, p.node)
}
for _, c := range a.Credentials {
if c.asked != ask {
continue
}
// The value the provision names, by the vault's credential name and its generation (clause 6): another is refused.
if c.Name != pr.Reference.ID || c.Generation != pr.Reference.Generation {
return nil, fmt.Errorf("the vault answered %s generation %d for %s, and the provision names %s generation %d: "+
"nothing is written until they agree", c.Name, c.Generation, key, pr.Reference.ID, pr.Reference.Generation)
}
value, err := p.open(string(c.Sealed))
if err != nil {
return nil, fmt.Errorf("the vault's value of %s does not open on this machine: %w", key, err)
}
return value, nil
}
return nil, fmt.Errorf("the vault answered no value of %s for this machine", key)
}
cmd/mesh-host/runs_as_written.go
+88
View File
@@ -0,0 +1,88 @@
package main
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"os"
"git.novox.be/novox/mesh-sdk/go/declaration"
"git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/novox/mesh-host/internal/store"
)
// runsAsWritten is what the node runs as the composer reads it (selfcompose.Env.Running), with every file given by digest
// named as this node last wrote it, where the two differ (issue 924, novox 2026-10-29 01:16). What runs of a module the
// agent has not applied since its start can come from the kept declaration, older than the compositions applied since:
// its export (mesh-vault.kept) then named a digest the controller's sweep had removed, while the file on disk was the
// newer export the node wrote. A node composing its own carries such a file as it runs (mesh-sdk frombus keptOf), so it
// names what is on disk: **only when the node's record says it wrote exactly that content there** (store Applied.Wrote,
// Target), never a file someone else changed. A file that cannot be read, or that the record does not vouch for, is
// left as the composition names it.
func runsAsWritten(running []delivery.Composed, statePath string) []delivery.Composed {
if len(running) == 0 || statePath == "" {
return running
}
known, err := store.Load(statePath)
if err != nil {
return running
}
out := make([]delivery.Composed, len(running))
for i, c := range running {
out[i] = c
var changed []json.RawMessage
for j, raw := range c.Resources {
if fixed, ok := asWritten(raw, known); ok {
if changed == nil {
changed = append([]json.RawMessage(nil), c.Resources...)
}
changed[j] = fixed
}
}
if changed != nil {
out[i].Resources = changed
}
}
return out
}
// asWritten is the file resource raw named by the digest and size of what this node last wrote at its path, when raw is a
// file given by another digest and the file there is what the record says the node wrote.
func asWritten(raw json.RawMessage, known store.State) (json.RawMessage, bool) {
var fields map[string]json.RawMessage
if json.Unmarshal(raw, &fields) != nil {
return nil, false
}
var kind, id, path, digest string
_ = json.Unmarshal(fields["type"], &kind)
_ = json.Unmarshal(fields["id"], &id)
_ = json.Unmarshal(fields["path"], &path)
_ = json.Unmarshal(fields["digest"], &digest)
if kind != string(declaration.TypeFile) || digest == "" || path == "" {
return nil, false
}
rec, ok := known.Find(id)
if !ok || rec.Wrote == "" || rec.Target != path {
return nil, false
}
body, err := os.ReadFile(path)
if err != nil {
return nil, false
}
sum := sha256.Sum256(body)
if hex.EncodeToString(sum[:]) != rec.Wrote {
return nil, false // changed since the node wrote it: not the node's to vouch for
}
if now := declaration.DigestOf(body); now == digest {
return nil, false
} else {
fields["digest"], _ = json.Marshal(now)
}
fields["size"], _ = json.Marshal(len(body))
fixed, err := json.Marshal(fields)
if err != nil {
return nil, false
}
return fixed, true
}
cmd/mesh-host/trust.go
+317
View File
@@ -0,0 +1,317 @@
package main
import (
"context"
"crypto/ed25519"
"encoding/base64"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/overlay"
"github.com/novox/mesh-host/internal/identity"
"github.com/novox/mesh-host/internal/link"
)
// **What this engine trusts a mesh signature from** (novox/hq ADR 0550 §2; design 47's L3 item d). Two signers: the
// controller's key, which the node was enrolled with (its membership's signer), and the vault's key, read from the vault's
// state `signers`, which only the vault may write: the bus grants are its anchor, as the wants' and the placement's (the
// lead's design limit, 2026-10-28). The controller's key is trusted until its lease is handed away, heard on the lease key
// (buscompose.LeaseHandedAway) and never on a clock; from then the vault's alone. What it trusts now the node states in its
// record (NodeRecord.Trusts), so the gate before the lease write reads every node's from the bus.
type meshTrust struct {
mu sync.Mutex
controller ed25519.PublicKey
handedAway bool
vault ed25519.PublicKey
authority string
// changed is told when what it trusts changed, so the record is written again.
changed func()
}
// newMeshTrust trusts the controller's key alone until the vault's is read.
func newMeshTrust(controller ed25519.PublicKey) *meshTrust {
t := &meshTrust{}
if len(controller) == ed25519.PublicKeySize {
t.controller = controller
}
return t
}
// onChange names who is told when what it trusts changed.
func (t *meshTrust) onChange(f func()) {
t.mu.Lock()
t.changed = f
t.mu.Unlock()
}
// Controller is the controller's key while the controller holds its lease, and nil once the lease is handed away.
func (t *meshTrust) Controller() ed25519.PublicKey {
if t == nil {
return nil
}
t.mu.Lock()
defer t.mu.Unlock()
if t.handedAway {
return nil
}
return t.controller
}
// HandedAway says the controller's lease was handed away, as heard on the lease key.
func (t *meshTrust) HandedAway() bool {
if t == nil {
return false
}
t.mu.Lock()
defer t.mu.Unlock()
return t.handedAway
}
// Signers are the keys a mesh signature is verified against now: the controller's while it holds the lease, and the
// vault's once read.
func (t *meshTrust) Signers() []delivery.Signer {
t.mu.Lock()
defer t.mu.Unlock()
var out []delivery.Signer
if t.controller != nil && !t.handedAway {
out = append(out, delivery.Signer{Name: delivery.SignerController, Key: t.controller})
}
if t.vault != nil {
out = append(out, delivery.Signer{Name: delivery.SignerVault, Key: t.vault})
}
return out
}
// Verify is the signer that signed msg as kind, or why none it trusts did.
func (t *meshTrust) Verify(kind string, msg, sig []byte) (delivery.Signer, error) {
context, err := delivery.SignContext(kind)
if err != nil {
return delivery.Signer{}, err
}
return delivery.VerifyBy(t.Signers(), context, msg, sig)
}
// VaultAuthority is the vault's authority certificate as read from `signers`, or "".
func (t *meshTrust) VaultAuthority() string {
if t == nil {
return ""
}
t.mu.Lock()
defer t.mu.Unlock()
return t.authority
}
// Trusts is what the record says it trusts: each signer's key and the vault's authority, by fingerprint.
func (t *meshTrust) Trusts() []string {
if t == nil {
return nil
}
authorities := map[string]string{}
if a := t.VaultAuthority(); a != "" {
authorities[delivery.SignerVault] = a
}
return delivery.Trusts(t.Signers(), authorities)
}
// change applies f under the lock and tells who follows when Trusts differs after it.
func (t *meshTrust) change(f func()) {
before := strings.Join(t.Trusts(), "\n")
t.mu.Lock()
f()
changed := t.changed
t.mu.Unlock()
if strings.Join(t.Trusts(), "\n") != before && changed != nil {
changed()
}
}
// heardSigner takes one key of the vault's `signers`: its key or its authority, refused (and said) when it does not
// check; a deleted key is no longer trusted.
func (t *meshTrust) heardSigner(key string, value []byte, deleted bool, say link.Announce) {
switch key {
case delivery.SignersKeyKey:
var pub ed25519.PublicKey
if !deleted && len(value) > 0 {
k, err := delivery.DecodeSignerKey(value)
if err == nil {
pub, err = k.PublicKey()
}
if err != nil {
say("the vault's signing key in " + delivery.SignersRead + " is not trusted: " + err.Error())
return
}
}
t.change(func() { t.vault = pub })
case delivery.SignersAuthorityKey:
pem := ""
if !deleted && len(value) > 0 {
a, err := delivery.DecodeSignerAuthority(value)
if err != nil {
say("the vault's authority in " + delivery.SignersRead + " is not trusted: " + err.Error())
return
}
pem = a.Certificate
}
t.change(func() { t.authority = pem })
}
}
// heardLease takes the controller's lease key as read: once handed away to a holder that is no controller, the
// controller's key is trusted no more. A lease that cannot be read changes nothing.
func (t *meshTrust) heardLease(raw []byte, found bool, readErr error) {
if readErr != nil {
return
}
handed, _ := buscompose.LeaseHandedAway(raw, found, nil)
t.change(func() { t.handedAway = handed })
}
// trustFollowAgain bounds the wait before a watch that could not be made is tried again on the same link: the vault may
// not serve `sign` yet, and until it does this node is not granted the reads.
var trustFollowAgain = 2 * time.Minute
// followTrust follows the vault's `signers` and the controller's lease on every link, as watches (never a poll), for as
// long as ctx lasts. A watch that cannot be made is said once per reason and tried again.
func followTrust(ctx context.Context, session *link.Session, trust *meshTrust, say link.Announce) {
said := ""
sayOnce := func(line string) {
if line != said {
said = line
if line != "" {
say(line)
}
}
}
for ctx.Err() == nil {
linked, cancel, err := session.WhenLinked(ctx)
if err != nil {
return
}
watching, stop := context.WithCancel(linked)
signers, serr := session.KVWatch(watching, delivery.SignersBucket, ">")
lease, lerr := session.KVWatch(watching, buscompose.LeaseBucket, buscompose.ControllerLeaseKey)
if serr != nil || lerr != nil {
stop()
sayOnce(fmt.Sprintf("what this node trusts a mesh signature from is not followed yet (the vault's signers: %v; "+
"the controller's lease: %v): it trusts what it last read, and looks again in %s", serr, lerr, trustFollowAgain))
signers, lease = nil, nil
} else {
sayOnce("")
}
for signers != nil || lease != nil {
select {
case u, ok := <-signers:
if !ok {
signers = nil
continue
}
if !u.Caught {
trust.heardSigner(u.Key, u.Value, u.Deleted, say)
}
case u, ok := <-lease:
if !ok {
lease = nil
continue
}
if !u.Caught {
trust.heardLease(u.Value, !u.Deleted && len(u.Value) > 0, nil)
}
}
}
stop()
select {
case <-linked.Done():
case <-time.After(trustFollowAgain):
}
cancel()
}
}
// placedAllHave says whether every module the seat mesh-placement places on node has a have of its own, read from the
// bus: then the kept declaration is read no more (keptRetired). A node the seat does not place, or whose placement or haves
// cannot be read, keeps reading it.
func placedAllHave(ctx context.Context, session *link.Session, node string) (bool, string) {
key, err := delivery.PlacementKey(node)
if err != nil {
return false, err.Error()
}
h, err := session.KVHistory(ctx, delivery.PlacementBucket, key)
if err != nil || len(h) == 0 || h[len(h)-1].Deleted {
return false, "its placement is not read"
}
p, err := delivery.DecodePlacement(h[len(h)-1].Value)
if err != nil {
return false, err.Error()
}
return allHave(p, func(module string) (bool, error) {
_, ok, err := session.ReadHaveErr(ctx, module)
return ok, err
})
}
// allHave says whether has finds a have for every module p places, and why not.
func allHave(p delivery.Placement, has func(module string) (bool, error)) (bool, string) {
if len(p.Modules) == 0 {
return false, "the seat mesh-placement places no module here"
}
for _, m := range p.Modules {
ok, err := has(m.Module)
if err != nil {
return false, "the have of " + m.Module + " is not read: " + err.Error()
}
if !ok {
return false, m.Module + " has no have yet"
}
}
return true, fmt.Sprintf("each of the %d modules placed here has a have of its own", len(p.Modules))
}
// certifyThisNode asks the vault's `sign` for this node's own certificate (kind node-certificate, ADR 0550 §2), checks
// it is for this node's internal name and serving key and signed by the authority the answer names, which must be the
// vault's this node trusts, keeps it where the record reads it, and writes the record again (written).
func certifyThisNode(ctx context.Context, state, node string, trust *meshTrust,
ask func(ctx context.Context, verb string, args any) (json.RawMessage, error), written func()) error {
serving, err := identity.LoadServingKey(identity.ServingKeyPath(state))
if err != nil {
return err
}
key, err := base64.StdEncoding.DecodeString(serving.Public)
if err != nil || len(key) != ed25519.PublicKeySize {
return fmt.Errorf("this node's serving key is not an ed25519 key")
}
raw, err := ask(ctx, delivery.SignVerb, map[string]any{"kind": delivery.KindNodeCertificate, "node": node})
if err != nil {
return err
}
var a delivery.SignAnswer
if err := json.Unmarshal(raw, &a); err != nil {
return fmt.Errorf("the vault's answer is not a signing: %w", err)
}
trusted := trust.VaultAuthority()
if trusted == "" {
return fmt.Errorf("this node has read no authority of the vault's to check the certificate against")
}
if err := delivery.CheckNodeCertificate(a.Certificate, []string{trusted}, overlay.InternalName(node),
ed25519.PublicKey(key), time.Now()); err != nil {
return err
}
path := identity.CertificatePath(state)
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
if err := os.WriteFile(path, []byte(a.Certificate), 0o644); err != nil {
return err
}
if written != nil {
written()
}
return nil
}
cmd/mesh-host/units_test.go
+59
View File
@@ -1,6 +1,9 @@
package main
import (
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -9,6 +12,7 @@ import (
"github.com/novox/mesh-host/internal/accounts"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/liveness"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/units"
)
@@ -103,6 +107,61 @@ func TestTheStatementSaysAnAccountJudgedForRoot(t *testing.T) {
if got["claude-code.agent"] != "never" || got["openrazer.account"] != "" {
t.Fatalf("root said: %v", got)
}
// 774 (ADR 0563 §6): the node seat's root-free passes only on the engine's own positive verdict at most 15 minutes old
// with execute not served here; a stale, missing or negative verdict, or execute served, is no pass and says why.
t.Run("this machine's root-free", func(t *testing.T) {
agent := accounts.Verdict{Account: accounts.Account{Module: "claude-code", ID: "claude-code.agent", Name: "agent", Root: true},
State: accounts.Healthy, Since: at}
free := &accounts.Statement{At: at, Accounts: []accounts.Verdict{agent}}
for _, row := range []struct {
name string
st *accounts.Statement
after time.Duration
served bool
free bool
why string
}{
{name: "a positive verdict", st: free, after: 14 * time.Minute, free: true, why: "the account agent cannot become root"},
{name: "a stale verdict", st: free, after: 16 * time.Minute, why: "more than 15 minutes ago"},
{name: "no verdict", st: nil, why: "judged no account yet"},
{name: "a way to root", st: as, why: "in the group docker, which grants root"},
{name: "no agent account", st: &accounts.Statement{At: at}, why: "agents run as the operator's account"},
{name: "execute served", st: free, served: true, why: "the login shell runs any command"},
} {
v := judgeRootFree("ace", row.st, at.Add(row.after), row.served, "the bus hears execute answered here")
if v.Free != row.free || !strings.Contains(v.Why, row.why) || v.Machine != "ace" {
t.Errorf("%s: %+v", row.name, v)
}
}
// execute is served while the holder's setting file says anything but withhold, or the bus hears it.
dir := t.TempDir()
statePath, conf := filepath.Join(dir, "state.json"), filepath.Join(dir, "tools.conf")
if err := store.Save(statePath, store.State{Resources: []store.Applied{{ID: "zsh.tools-conf", Type: "file", Target: conf}}}); err != nil {
t.Fatal(err)
}
quiet := func() (bool, error) { return false, nil }
for _, row := range []struct {
name, setting string
heard func() (bool, error)
served bool
}{
{name: "withheld and not heard", setting: "# the mesh's\nexecute=withhold\n", heard: quiet},
{name: "served", setting: "execute=serve\n", heard: quiet, served: true},
{name: "withheld and heard", setting: "execute=withhold\n", heard: func() (bool, error) { return true, nil }, served: true},
{name: "the bus not asked", setting: "execute=withhold\n", heard: func() (bool, error) { return false, errors.New("no link") }, served: true},
{name: "no setting file", heard: quiet, served: true},
} {
_ = os.Remove(conf)
if row.setting != "" {
if err := os.WriteFile(conf, []byte(row.setting), 0o644); err != nil {
t.Fatal(err)
}
}
if served, why := executeServedHere(statePath, row.heard); served != row.served {
t.Errorf("%s: served %v (%s)", row.name, served, why)
}
}
})
}
// A verdict waiting for the setuid search says how far it got, in the reason the root-free verb repeats and as
cmd/mesh-host/whatruns_test.go
+37
View File
@@ -4,6 +4,7 @@ import (
"crypto/ed25519"
"encoding/json"
"errors"
"path/filepath"
"strings"
"testing"
@@ -11,6 +12,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/novox/mesh-host/internal/agent"
"github.com/novox/mesh-host/internal/store"
)
func aFileOf(module, id string) json.RawMessage {
@@ -59,6 +61,41 @@ func TestDeclaredNowReadsWhatRunsWhenTheAgentIsRunning(t *testing.T) {
if err != nil || d.Provisioner != "from-agent" {
t.Fatalf("declaredNow read %+v (%v), not what runs", d, err)
}
// L3 s9 (ADR 0550 §2): a node whose placed modules all have a have reads the kept declaration no more, though the
// file is there and readable and the controller's key is dropped; until then it reads it.
state := filepath.Join(t.TempDir(), "state", "state.json")
if err := store.SaveDeclared(store.DeclaredPath(state), store.Declared{Declaration: []byte(`{"declaration":1,` +
`"resources":[{"id":"zsh.rc","type":"file","module":"zsh","path":"/etc/zshrc","content":"x"}]}`)}); err != nil {
t.Fatal(err)
}
if d, err := keptDeclaration(state, nil); err != nil || len(d.Resources) != 1 {
t.Fatalf("before the read is retired: %+v, %v", d, err)
}
placed := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "zsh"}, {Module: "lemurs"}}}
for _, row := range []struct {
name string
has map[string]bool
want bool
}{
{"every placed module has a have", map[string]bool{"zsh": true, "lemurs": true}, true},
{"one has none yet", map[string]bool{"zsh": true}, false},
} {
if got, why := allHave(placed, func(m string) (bool, error) { return row.has[m], nil }); got != row.want {
t.Errorf("%s: %v (%s)", row.name, got, why)
}
}
if got, _ := allHave(delivery.Placement{Node: "n1"}, nil); got {
t.Error("a node with nothing placed retires the read")
}
keptRetired.Store(true)
defer keptRetired.Store(false)
if _, err := keptDeclaration(state, nil); !errors.Is(err, errKeptRetired) {
t.Errorf("after the read is retired: %v", err)
}
if by, filled := keptOnce(options{state: state}, nil)(); by != nil || filled != nil {
t.Errorf("after the read is retired the kept declaration was split: %v %v", by, filled)
}
}
// 1945/AC17 (unit): a resource the node's own composition shares with a running module's composition is declared once,
go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module github.com/novox/mesh-host
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
golang.org/x/crypto v0.57.0
go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771 h1:Tt16HJk8JZRce1i3JzzDSNdCWOrv7dw8FQ/bckNQ5Ig=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029233535-78c9ec881771/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0 h1:lKU3HjNXGiVbLW2hUgFjU/TMTUYMlNnnmZxG5T7biQA=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261030001116-2da0de75c3f0/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
internal/accounts/setuid_cache.go
+11
View File
@@ -240,6 +240,17 @@ func (c *SetuidCache) SearchAgain() {
c.forgetProgress()
}
// Running says whether a search walks now (novox/hq ADR 0563 §2: the node seat starts no second while one does).
// Nil c runs none.
func (c *SetuidCache) Running() bool {
if c == nil {
return false
}
c.mu.Lock()
defer c.mu.Unlock()
return c.running
}
// SearchNow throws away the last complete search, kept or in memory, and any walk under way or kept, and has
// the next look start a full search at once, past AgainAfter and any back-off (novox/hq issue 361): the
// operator changed by hand what the last one found, which no apply says. Until the new one completes the
internal/agent/agent.go
+348 -14
View File
@@ -109,7 +109,9 @@ type Env struct {
Kept func() map[string][]json.RawMessage
// KeptFilled is the paths the kept declaration names as filled by the provisioner, by the module whose file is at
// each (the node's own under NodeModule). A module that still runs from the kept declaration keeps its filled files
// filled: without them the filling refused each one as "not named as filled" (issue 1945). Nil names none.
// filled: without them the filling refused each one as "not named as filled" (issue 1945). Nil names none. A module
// whose have names a composition the bucket keeps runs from that composition, never from the kept declaration
// (issue 924): the kept declaration is older than every composition applied since the switch.
KeptFilled func() map[string][]string
// Trusted is the public keys a composition may be signed by: the controller's and the operator's (ADR 0539 §3). A
// composition unsigned, signed by another key, or changed after it was signed is refused and nothing of it is
@@ -131,6 +133,12 @@ type Env struct {
// them. The node-engine's filling tells the provisioner's client then to fill again what it refused (mesh-host
// internal/filling Knock; ace, 2026-10-25 21:17). Nil tells nothing.
Settled func(module string)
// RunsChanged is told, outside the agent's lock, that what runs (Everything) changed other than by an apply or a
// removal: once at the start, when the haves and the compositions they name are read, and each time a module is
// taken as running from its have or the kept declaration. Like Settled, the filling then tells the provisioner's
// client to fill again what it refused (issue 924, the second meeting: ace, g14 and shanks on 2026-10-29 at 00:09
// UTC refused power's three new drop-ins from an engine start on, as no module settled after it). Nil tells nothing.
RunsChanged func(why string)
// Backoff is how long to wait before the try-th retry (1, 2, …) of an apply that failed transiently. Nil retries a
// transient failure only on RetryTransient or a change of either key.
Backoff func(try int) time.Duration
@@ -143,6 +151,27 @@ type Env struct {
// move of a module of policy record that recreates a container or moves its data waits for a person's yes in its
// want (ADR 0547 §3). Nil judges no move.
Policy func(ctx context.Context, module string) string
// HoldsData says whether a module's manifest declares data (catalogue.Manifest.DataItems), and whether its manifest
// could be read: what a composition's Data is, for what runs of a module from the kept declaration, which carries no
// Data of its own. A composition says Data, so without it a data module from the kept declaration is judged to move
// its data against its first composition after a restart (mongodb and mssql on novox, issue 924). Nil reads none.
HoldsData func(ctx context.Context, module string) (data, known bool)
// Applied keeps every composition the agent applied, by module (the node's own under NodeModule), beside the engine's
// state on the machine: read at the start before anything is watched, so what runs is known before the node composes
// its own for the first time, whatever the bucket and the kept declaration still hold (issue 774: on 2026-10-29 the
// bus node, its kept declaration retired and its own composition in no bucket, composed its own as if its folded
// modules ran nothing and removed the bus server and four held modules). Nil keeps none.
Applied AppliedStore
}
// AppliedStore is where the engine keeps the compositions it applied (Env.Applied): each module's last, as its bytes.
type AppliedStore interface {
// Load is every composition kept, by module; a store that cannot be read says so and loads none.
Load() (map[string][]byte, error)
// Keep keeps the composition the module was applied from, in place of the one before.
Keep(module string, raw []byte) error
// Drop forgets the module's, once it is removed.
Drop(module string) error
}
// transientFor is how many transient retries say the phase applying before the have says failed: past them the bus has
@@ -160,6 +189,13 @@ type Agent struct {
// applied is the composition each module was last applied from: a revision a want named from the key's history is
// not the key's latest value, and what it holds is still held out of the others' applies.
applied map[string]compNow
// had is, for each module whose have names a composition the bucket keeps (its latest value or a revision), that
// composition, read once at the start (loadHad): what runs of a module the agent has not taken yet, ahead of the kept
// declaration (issue 924).
had map[string]compNow
// keptData is, for each module of the kept declaration whose manifest could be read, whether it declares data
// (Env.HoldsData): the Data of what runs of it from the kept declaration.
keptData map[string]bool
// attempt is the last apply of a module that failed or is held, with the revisions it was made from.
attempt map[string]attempt
checked map[string]bool
@@ -218,7 +254,7 @@ func New(env Env) *Agent {
if env.Say == nil {
env.Say = func(string) {}
}
return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{},
return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{}, had: map[string]compNow{}, keptData: map[string]bool{},
attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, notComposed: map[string]notComposedMark{},
retry: make(chan []string, 8), reconcile: make(chan []string, 64), tries: map[string]int{}, again: make(chan againAsk, 64),
transient: make(chan struct{}, 1), stopped: make(chan struct{})}
@@ -228,6 +264,9 @@ func New(env Env) *Agent {
// before any module's, then each change as it comes.
func (a *Agent) Run(ctx context.Context) error {
defer close(a.stopped)
// What this engine last applied, before any composition is watched: the node composing its own reads what runs
// (Everything) from its first pass on, and that pass comes before the haves are read below (issue 774).
a.loadApplied(ctx)
wants, err := a.env.Wants.Watch(ctx, "want."+a.env.Node+".>")
if err != nil {
return fmt.Errorf("the agent cannot watch its wants: %w", err)
@@ -236,7 +275,7 @@ func (a *Agent) Run(ctx context.Context) error {
if err != nil {
return fmt.Errorf("the agent cannot watch its compositions: %w", err)
}
var wantsCaught, compsCaught bool
var wantsCaught, compsCaught, hadRead bool
dirty := map[string]bool{}
for {
select {
@@ -303,7 +342,17 @@ func (a *Agent) Run(ctx context.Context) error {
}
// Nothing is applied until what was there at the start is all in: a module's composition may arrive after its
// want, and the node's own composition comes before any module's.
if !(wantsCaught && compsCaught) || len(dirty) == 0 {
if !(wantsCaught && compsCaught) {
continue
}
if !hadRead {
// Before the first settle: what runs of each module with a have, from the composition it names, so the
// filling judges by it at once and not by the kept declaration until some module settles (issue 924).
hadRead = true
a.loadHad(ctx)
a.runsChanged("the haves were read at the start")
}
if len(dirty) == 0 {
continue
}
a.settle(ctx, dirty)
@@ -726,7 +775,12 @@ func (a *Agent) revisionNamed(ctx context.Context, module, fp, digest string) (c
if err != nil {
continue
}
dig, err := delivery.VerifyComposed(c, trusted)
var dig string
if a.env.SelfComposed {
dig, err = delivery.ComposedDigestOf(c) // the node's own: unsigned (ADR 0547 §2)
} else {
dig, err = delivery.VerifyComposed(c, trusted)
}
if err != nil || dig != digest {
continue
}
@@ -755,22 +809,33 @@ func (a *Agent) others(except string) []delivery.Composed {
out = append(out, c.C)
} else if c, ok := a.applied[m]; ok && c.Dig == h.Composed {
out = append(out, c.C)
} else if c, ok := a.had[m]; ok && c.Dig == h.Composed {
out = append(out, c.C) // a revision its have named, replaced since under the same key
}
}
hadNames := make([]string, 0, len(a.had))
for m := range a.had {
if _, held := a.running[m]; !held && m != except {
hadNames = append(hadNames, m)
}
}
sort.Strings(hadNames)
for _, m := range hadNames {
out = append(out, a.had[m].C)
}
if a.env.Kept != nil {
kept := a.env.Kept()
names := make([]string, 0, len(kept))
for m := range kept {
if _, held := a.running[m]; !held && m != except {
_, held := a.running[m]
_, had := a.had[m]
if !held && !had && m != except {
names = append(names, m)
}
}
sort.Strings(names)
for _, m := range names {
c := delivery.Composed{Node: a.env.Node, Module: m, Resources: kept[m]}
if a.env.KeptFilled != nil {
c.Filled = a.env.KeptFilled()[m]
}
c, _ := a.keptComposed(m, kept[m])
if m == NodeModule {
c.Module = ""
}
@@ -780,6 +845,188 @@ func (a *Agent) others(except string) []delivery.Composed {
return out
}
// loadHad reads, once at the start, the have of every module with a want or in the kept declaration and, where it names
// a composition, that composition: the bucket's value at its key when its digest is the have's, else the revision the
// bucket keeps with that digest. A have naming none, or one the bucket no longer keeps, leaves the module to the kept
// declaration, said once (issue 924).
func (a *Agent) loadHad(ctx context.Context) {
a.mu.Lock()
seen := map[string]bool{}
for m, w := range a.wants {
if m != NodeModule && w.Present && !w.Want.Removed {
seen[m] = true
}
}
if a.env.Kept != nil {
for m := range a.env.Kept() {
if m != NodeModule {
seen[m] = true
}
}
}
a.mu.Unlock()
names := make([]string, 0, len(seen))
for m := range seen {
names = append(names, m)
}
sort.Strings(names)
a.readKeptData(ctx, names...)
for _, m := range names {
if ctx.Err() != nil {
return
}
h, ok := a.env.Haves.Read(ctx, m)
if !ok || h.Composed == "" || !haveRuns(h) {
continue
}
a.mu.Lock()
if c, kept := a.had[m]; kept && c.Dig == h.Composed {
a.mu.Unlock()
continue // what this engine applied and kept itself (loadApplied)
}
c, found := a.compositionFor(m, h.FP)
a.mu.Unlock()
if !found || c.Err != nil || c.Dig != h.Composed {
c, found = a.revisionNamed(ctx, m, h.FP, h.Composed)
}
a.mu.Lock()
if found {
a.had[m] = c
} else if _, applied := a.had[m]; applied {
// Kept as this engine applied it (loadApplied), never composed as nothing.
} else if _, kept := a.keptHas(m); kept {
a.sayOnce("had:"+m, fmt.Sprintf("%s's have names the composition %s, which the bucket no longer keeps: it runs "+
"from the kept declaration until it is applied or taken", m, short(h.Composed)))
}
a.mu.Unlock()
}
}
// loadApplied takes, once at the start, what this engine last applied of each module as what runs of it, ahead of the
// bucket and the kept declaration (Env.Applied, issue 774): it is what the machine holds, whatever the bus still keeps.
// One whose have names another composition runs as applied until loadHad reads that one from the bucket; one whose
// have cannot be read, or that has none, is kept as applied, so a module is never composed as running nothing.
func (a *Agent) loadApplied(ctx context.Context) {
if a.env.Applied == nil {
return
}
kept, err := a.env.Applied.Load()
if err != nil {
a.env.Say(fmt.Sprintf("the compositions this engine applied cannot be read, so what runs is read from the haves alone: %v", err))
}
// At an engine's first start nothing is kept yet: the node's own runs as the kept declaration has it, which the store
// keeps now, so the node never composes its own as if its folded modules ran nothing once the declaration retires
// (issue 774, review of the re-land, round 1). Taken whatever digest the node's have names: the composition that names
// is in no bucket, and the declaration is the one record of what runs on disk.
seeded := false
if _, own := kept[NodeModule]; !own && err == nil && a.env.Kept != nil {
if rs, ok := a.env.Kept()[NodeModule]; ok && len(rs) > 0 {
c, _ := a.keptComposed(NodeModule, rs)
c.Module = ""
raw, eerr := delivery.EncodeComposed(c)
if eerr == nil {
eerr = a.env.Applied.Keep(NodeModule, raw)
}
if eerr != nil {
a.env.Say(fmt.Sprintf("the node's own as the kept declaration has it cannot be kept: %v", eerr))
} else {
if kept == nil {
kept = map[string][]byte{}
}
kept[NodeModule], seeded = raw, true
a.env.Say("the node's own runs as the kept declaration has it, kept now as what this engine runs")
}
}
}
names := make([]string, 0, len(kept))
for m := range kept {
names = append(names, m)
}
sort.Strings(names)
for _, m := range names {
if ctx.Err() != nil {
return
}
c, err := delivery.DecodeComposed(kept[m])
var dig string
if err == nil {
dig, err = delivery.ComposedDigestOf(c)
}
if err != nil {
a.env.Say(fmt.Sprintf("the composition this engine kept of %s cannot be read, so what runs of it is read from its have: %v", m, err))
continue
}
h, found, unread := a.readHave(ctx, m)
if seeded && m == NodeModule {
found, unread = true, false // what the kept declaration has runs, whatever the have names
} else if found && h.Composed != "" && h.Composed != dig && haveRuns(h) {
// The composition the have names replaces this once loadHad finds it in the bucket; until then, and when
// the bucket keeps it no more, what this engine applied runs (issue 774, review of the re-land, round 1).
a.env.Say(fmt.Sprintf("%s's have names the composition %s, not %s this engine applied last: it runs as applied "+
"until the one its have names is read", m, short(h.Composed), short(dig)))
}
if m == NodeModule {
c.Module, c.FP = "", ""
}
a.mu.Lock()
a.had[m] = compNow{Raw: kept[m], C: c, Dig: dig}
a.mu.Unlock()
switch {
case unread:
a.env.Say(fmt.Sprintf("%s's have cannot be read: it runs as this engine applied it, %s", m, short(dig)))
case !found:
a.env.Say(fmt.Sprintf("%s has no have: it runs as this engine applied it, %s", m, short(dig)))
}
}
}
// keepApplied keeps the composition a module was applied from (Env.Applied); one that cannot be kept is said, and the
// apply stands.
func (a *Agent) keepApplied(module string, comp compNow) {
if a.env.Applied == nil {
return
}
raw := comp.Raw
if len(raw) == 0 {
var err error
if raw, err = delivery.EncodeComposed(comp.C); err != nil {
a.env.Say(fmt.Sprintf("the composition %s was applied from cannot be kept: %v", module, err))
return
}
}
if err := a.env.Applied.Keep(module, raw); err != nil {
a.env.Say(fmt.Sprintf("the composition %s was applied from cannot be kept: %v", module, err))
}
}
// haveRuns says a have names what runs, in the phases baseline takes as running: running (or empty), held for a failed
// backup, or a composer's mark over what still runs. A have backing up, applying, checking, failed or waiting names a
// composition not applied, or not yet (issue 924, review of 031e106).
func haveRuns(h Have) bool {
switch delivery.Phase(h.Phase) {
case "", delivery.PhaseRunning, delivery.PhaseHeld:
return true
}
return composerMark(h)
}
// keptHas says whether the kept declaration has resources for the module. Called holding a.mu or not: Kept takes no lock
// of the agent's.
func (a *Agent) keptHas(module string) ([]json.RawMessage, bool) {
if a.env.Kept == nil {
return nil, false
}
rs, ok := a.env.Kept()[module]
return rs, ok
}
// runsChanged tells RunsChanged, outside the agent's lock.
func (a *Agent) runsChanged(why string) {
if a.env.RunsChanged != nil {
a.env.RunsChanged(why)
}
}
// unreadableModules are the keys of the module compositions in the node's bucket that cannot be read or are refused, of a
// module the assigner has not removed: for each module only the composition its want names, else its newest, since a
// stale one nobody applies (written before a rule the SDK now holds) must not hold the node's own back. Called holding
@@ -901,7 +1148,8 @@ func (a *Agent) heldOut(except string) []delivery.Composed {
}
}
if len(rs) > 0 {
out = append(out, delivery.Composed{Node: a.env.Node, Module: m, Resources: rs})
// Held out, never judged: its Data from its manifest where known, as every other composition says it.
out = append(out, delivery.Composed{Node: a.env.Node, Module: m, Resources: rs, Data: a.keptData[m]})
}
}
return out
@@ -916,6 +1164,34 @@ func (a *Agent) Everything() []delivery.Composed {
return a.others("\x00")
}
// Placed is the modules a want still places on the node (present and not removed), held ones included, but except:
// what no apply but the module's own may remove (apply.Scope.Placed, issue 774). Called without a.mu held.
func (a *Agent) Placed(except string) map[string]bool {
a.mu.Lock()
defer a.mu.Unlock()
out := map[string]bool{}
for m, w := range a.wants {
if m != NodeModule && m != except && w.Present && !w.Want.Removed {
out[m] = true
}
}
return out
}
// RemovedByWant is the modules whose want says removed (apply.Scope.RemovedByWant, issue 774): in the node's own apply,
// only their resources may go. Called without a.mu held.
func (a *Agent) RemovedByWant() map[string]bool {
a.mu.Lock()
defer a.mu.Unlock()
out := map[string]bool{}
for m, w := range a.wants {
if m != NodeModule && w.Present && w.Want.Removed {
out[m] = true
}
}
return out
}
// HoldsNode says whether the agent runs the node's own composition, so the node's parts are its and not the kept
// declaration's.
func (a *Agent) HoldsNode() bool {
@@ -1048,6 +1324,7 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning}
}
a.mu.Unlock()
a.runsChanged(module + " was taken as held")
return true
}
if h.Composed != "" {
@@ -1068,6 +1345,7 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
a.mu.Lock()
a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning}
a.mu.Unlock()
a.runsChanged(module + " was taken from its have")
// Said again, so the session holds the module and its key carries the health this engine judges,
// not the one an older engine left there (xorg's unknown on g14 and shanks, 2026-10-25).
a.writeHave(ctx, Have{Module: module, FP: h.FP, Composed: h.Composed,
@@ -1096,6 +1374,7 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
a.mu.Lock()
a.running[module] = delivery.Have{FP: w.Want.FP, Composed: w.Want.Composed, Phase: delivery.PhaseRunning}
a.mu.Unlock()
a.runsChanged(module + " was taken from the kept declaration")
a.writeHave(ctx, Have{Module: module, FP: w.Want.FP, Composed: w.Want.Composed, Phase: string(delivery.PhaseRunning)})
return true
}
@@ -1184,6 +1463,7 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
a.running[module] = delivery.Have{FP: fp, Composed: dig, Phase: delivery.PhaseRunning}
a.applied[module] = comp
a.mu.Unlock()
a.keepApplied(module, comp)
if a.env.Settled != nil {
a.env.Settled(module)
}
@@ -1213,8 +1493,14 @@ func (a *Agent) remove(ctx context.Context, module string, w wantNow) {
return
}
delete(a.attempt, module)
delete(a.had, module)
a.running[module] = delivery.Have{Phase: delivery.PhaseRemoved}
a.mu.Unlock()
if a.env.Applied != nil {
if err := a.env.Applied.Drop(module); err != nil {
a.env.Say(fmt.Sprintf("the composition %s was applied from cannot be forgotten now it is removed: %v", module, err))
}
}
if a.env.Settled != nil {
a.env.Settled(module)
}
@@ -1286,6 +1572,7 @@ func (a *Agent) waitsForYes(ctx context.Context, module string, w wantNow, comp
if module == NodeModule || a.env.Policy == nil || a.env.Policy(ctx, module) != catalogue.PolicyRecord || w.Want.Yes == comp.Dig {
return false
}
a.readKeptData(ctx, module) // the manifest may read now though it did not at the start
a.mu.Lock()
r, ran := a.running[module]
if ran && r.Composed == comp.Dig && r.Phase != delivery.PhaseRemoved {
@@ -1331,14 +1618,61 @@ func (a *Agent) waitsForYes(ctx context.Context, module string, w wantNow, comp
// runningComposition is what the module runs now: the composition last applied, else its resources in the declaration
// the node kept from before the switch. Called with a.mu held.
//
// The composition its have named at the start comes before the kept declaration (issue 924). From the kept declaration,
// what runs carries Data as its manifest declares it; a module whose manifest could not be read cannot be judged against
// it (false), and that is said, rather than taking it to hold no data and judging a false move of its data.
func (a *Agent) runningComposition(module string) (delivery.Composed, bool) {
if c, ok := a.applied[module]; ok && c.Err == nil {
return c.C, true
}
if a.env.Kept != nil {
if rs, ok := a.env.Kept()[module]; ok {
return delivery.Composed{Node: a.env.Node, Module: module, Resources: rs}, true
if c, ok := a.had[module]; ok {
return c.C, true
}
if rs, ok := a.keptHas(module); ok {
c, known := a.keptComposed(module, rs)
if !known {
a.sayOnce("kept-data:"+module, fmt.Sprintf("%s runs from the kept declaration, and whether it holds data cannot be "+
"read from its manifest: a move of it cannot be judged", module))
}
return c, known
}
return delivery.Composed{}, false
}
// keptComposed is what runs of a module from the kept declaration: its resources there, the paths it names as filled,
// and Data as the module's manifest declares it; false when the manifest could not be read, so Data is not known. Called
// with a.mu held.
func (a *Agent) keptComposed(module string, rs []json.RawMessage) (delivery.Composed, bool) {
c := delivery.Composed{Node: a.env.Node, Module: module, Resources: rs}
if a.env.KeptFilled != nil {
c.Filled = a.env.KeptFilled()[module]
}
data, known := a.keptData[module]
c.Data = data
return c, known || module == NodeModule
}
// readKeptData reads, outside the agent's lock, whether each module of the kept declaration not known yet declares data
// (Env.HoldsData), and keeps the answers it could read.
func (a *Agent) readKeptData(ctx context.Context, modules ...string) {
if a.env.HoldsData == nil {
return
}
for _, m := range modules {
if m == NodeModule {
continue
}
a.mu.Lock()
_, done := a.keptData[m]
a.mu.Unlock()
if _, kept := a.keptHas(m); done || !kept {
continue
}
if data, known := a.env.HoldsData(ctx, m); known {
a.mu.Lock()
a.keptData[m] = data
a.mu.Unlock()
}
}
}
internal/agent/agent_test.go
+251 -11
View File
@@ -769,6 +769,162 @@ func TestEverythingIsWhatTheAgentRunsAndWhatIsKeptForTheRest(t *testing.T) {
if a.HoldsNode() {
t.Fatal("the agent holds no node composition")
}
// Issue 774 (novox, 2026-10-29 07:54 UTC): the kept declaration retired and the node's own composition in no bucket, a
// fresh engine's first own composition was composed from what runs with the node's own missing, so its folded modules
// (the bus server among them) were composed as nothing and removed. What this engine applied is kept on the machine
// and is what runs from before the first composition is watched; an apply keeps, a removal forgets.
t.Run("what this engine applied is what runs before the node composes its own", func(t *testing.T) {
b := newBed(t)
b.env.SelfComposed, b.env.Trusted, b.kept = true, nil, nil
store := &memApplied{kept: map[string][]byte{}}
b.env.Applied = store
// A folded module's resource in the node's own composition: its id names the module, as the controller folds it.
natsConf, _ := json.Marshal(map[string]any{"id": "nats.conf", "type": "file", "path": "/tmp/r6/nats/conf", "content": "listen 4222"})
own := delivery.Composed{Node: node, Resources: []json.RawMessage{natsConf}}
ownRaw, err := delivery.EncodeComposed(own)
if err != nil {
t.Fatal(err)
}
ownDig, _ := delivery.ComposedDigestOf(own)
web := delivery.Composed{Node: node, Module: "web", FP: fpOf("a"), Resources: []json.RawMessage{file("web", "x", "1")}}
webRaw, _ := delivery.EncodeComposed(web)
webDig, _ := delivery.ComposedDigestOf(web)
store.kept[NodeModule], store.kept["web"] = ownRaw, webRaw
b.haves.Write(context.Background(), Have{Module: NodeModule, Composed: ownDig})
b.haves.Write(context.Background(), Have{Module: "web", FP: fpOf("a"), Composed: webDig, Phase: string(delivery.PhaseRunning)})
// The composer's first pass reads what runs when its watch begins, before the agent reads any have.
firstCh := make(chan []delivery.Composed, 1)
comps := b.comps
var ag *Agent
b.env.Composed = watchFunc(func(ctx context.Context, p string) (<-chan Update, error) {
firstCh <- ag.Everything()
return comps.Watch(ctx, p)
})
ag = New(b.env)
b.ag = ag
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { defer close(done); _ = ag.Run(ctx) }()
defer func() { cancel(); <-done }()
var first []delivery.Composed
select {
case first = <-firstCh:
case <-time.After(5 * time.Second):
t.Fatal("the composer was not watched")
}
var ranOwn, ranWeb bool
for _, c := range first {
ranOwn = ranOwn || (c.Module == "" && len(c.Resources) == 1 && strings.Contains(string(c.Resources[0]), "4222"))
ranWeb = ranWeb || c.Module == "web"
}
if !ranOwn || !ranWeb {
t.Fatalf("what runs at the node's first own composition: own %v, web %v (%d compositions); want both as this engine applied them",
ranOwn, ranWeb, len(first))
}
// An apply keeps the composition it applied; a removal forgets it.
web2 := delivery.Composed{Node: node, Module: "web", FP: fpOf("b"), Resources: []json.RawMessage{file("web", "x", "2")}}
ck, _ := delivery.ComposedKey(node, "web", fpOf("b"))
raw2, _ := delivery.EncodeComposed(web2)
comps.put(ck, raw2)
wk, _ := delivery.WantKey(node, "web")
wraw, _ := delivery.EncodeWant(delivery.Want{FP: fpOf("b"), Self: true})
b.wants.put(wk, wraw)
eventually(t, "the applied composition was not kept", func() bool { return string(store.get("web")) == string(raw2) })
rraw, _ := delivery.EncodeWant(delivery.Want{Removed: true})
b.wants.put(wk, rraw)
eventually(t, "the removed module's composition was not forgotten", func() bool { return store.get("web") == nil })
})
// Issue 774, review of the re-land, round 1: at the re-landed engine's first start nothing is kept yet, and the node's
// have names a composition the node composed itself, in no bucket. The node's own runs as the kept declaration has it
// and is kept then, so the next start, the declaration retired, still runs nats.
t.Run("at the first start nothing is kept: the node's own runs as the kept declaration has it", func(t *testing.T) {
natsConf, _ := json.Marshal(map[string]any{"id": "nats.conf", "type": "file", "path": "/tmp/r6/nats/conf", "content": "listen 4222"})
store := &memApplied{kept: map[string][]byte{}}
h := &haves{}
h.Write(context.Background(), Have{Module: NodeModule, Composed: "sha256:" + strings.Repeat("c", 64)})
firstPass := func(kept map[string][]json.RawMessage) []delivery.Composed {
b := newBed(t)
b.env.SelfComposed, b.env.Trusted, b.kept, b.env.Applied, b.env.Haves = true, nil, kept, store, h
firstCh := make(chan []delivery.Composed, 1)
comps := b.comps
var ag *Agent
b.env.Composed = watchFunc(func(ctx context.Context, p string) (<-chan Update, error) {
firstCh <- ag.Everything()
return comps.Watch(ctx, p)
})
ag = New(b.env)
ctx, cancel := context.WithCancel(context.Background())
done := make(chan struct{})
go func() { defer close(done); _ = ag.Run(ctx) }()
defer func() { cancel(); <-done }()
select {
case got := <-firstCh:
return got
case <-time.After(5 * time.Second):
t.Fatal("the composer was not watched")
}
return nil
}
runsNats := func(cs []delivery.Composed) bool {
for _, c := range cs {
if c.Module == "" && len(c.Resources) == 1 && strings.Contains(string(c.Resources[0]), "4222") {
return true
}
}
return false
}
if got := firstPass(map[string][]json.RawMessage{NodeModule: {natsConf}}); !runsNats(got) {
t.Fatalf("the first start does not run nats in the node's own (%d compositions)", len(got))
}
if store.get(NodeModule) == nil {
t.Fatal("the first start kept no node's own composition")
}
if got := firstPass(nil); !runsNats(got) {
t.Fatalf("the next start, the kept declaration retired, does not run nats in the node's own (%d compositions)", len(got))
}
})
}
// watchFunc is a Watcher from a function.
type watchFunc func(ctx context.Context, pattern string) (<-chan Update, error)
func (f watchFunc) Watch(ctx context.Context, pattern string) (<-chan Update, error) {
return f(ctx, pattern)
}
// memApplied is an AppliedStore in memory.
type memApplied struct {
mu sync.Mutex
kept map[string][]byte
}
func (m *memApplied) Load() (map[string][]byte, error) {
m.mu.Lock()
defer m.mu.Unlock()
out := map[string][]byte{}
for k, v := range m.kept {
out[k] = v
}
return out, nil
}
func (m *memApplied) Keep(module string, raw []byte) error {
m.mu.Lock()
defer m.mu.Unlock()
m.kept[module] = raw
return nil
}
func (m *memApplied) Drop(module string) error {
m.mu.Lock()
defer m.mu.Unlock()
delete(m.kept, module)
return nil
}
func (m *memApplied) get(module string) []byte {
m.mu.Lock()
defer m.mu.Unlock()
return m.kept[module]
}
// A failed apply is applied again after a restart even when the kept declaration holds the composition: once this
@@ -1026,18 +1182,102 @@ func TestTheRevisionAWantNamesIsAppliedFromHistory(t *testing.T) {
}
}
// 1945/AC18 (unit): a module that still runs from the kept declaration carries the kept declaration's filled paths of
// its files, so the filling writes them ("not named as filled" for claude-code and screen-lock, 2026-10-25).
func TestAModuleRunningFromTheKeptDeclarationKeepsItsFilledPaths(t *testing.T) {
b := newBed(t)
b.kept = map[string][]json.RawMessage{"claude-code": {file("claude-code", "settings", "")}}
b.env.KeptFilled = func() map[string][]string {
return map[string][]string{"claude-code": {"/tmp/r6/claude-code/settings"}}
// 1945/AC18 and issue 924 (unit): what runs of a module the agent has not taken yet carries the filled paths the
// filling judges by. From the kept declaration while the module has no have naming a composition ("not named as filled"
// for claude-code and screen-lock, 2026-10-25); from the composition its have names once it has one, the kept
// declaration being older, read at the start before any module settles and told to the filling then, so a file the
// newer composition fills is accepted on the first fill (power's three drop-ins on ace, g14 and shanks, 2026-10-29).
// Only a have in a phase baseline takes as running names what runs; and what runs carries Data, from the composition
// or, from the kept declaration, from the module's manifest, so a data module is not judged to move its data against
// its first composition after a restart (mongodb and mssql on novox).
func TestWhatRunsBeforeTheAgentTakesAModuleNamesItsFilledPaths(t *testing.T) {
const old, added = "/tmp/r6/power/settings", "/tmp/r6/power/bound"
filledFile := func(id string) json.RawMessage {
return json.RawMessage(`{"id":"power.` + id + `","type":"file","module":"power","path":"/tmp/r6/power/` + id +
`","mode":"0644","owner-at-runtime":"owned"}`)
}
a := New(b.env)
got := a.Everything()
if len(got) != 1 || got[0].Module != "claude-code" || strings.Join(got[0].Filled, ",") != "/tmp/r6/claude-code/settings" {
t.Fatalf("what runs from the kept declaration: %+v", got)
fp := fpOf("a")
ck, _ := delivery.ComposedKey(node, "power", fp)
applied := delivery.Composed{Node: node, Module: "power", FP: fp, Filled: []string{old, added}, Data: true,
Resources: []json.RawMessage{filledFile("settings"), filledFile("bound")}}
_, appliedRaw, appliedDig := signedBy(t, applied, ck, controllerKey)
newer := applied
newer.Resources = append([]json.RawMessage{}, applied.Resources...)
newer.Resources = append(newer.Resources, file("power", "later", "1"))
_, newerRaw, _ := signedBy(t, newer, ck, controllerKey)
for _, tc := range []struct {
name string
// have is the module's have; bucket the value its composition key holds now, history the revisions kept.
have *Have
bucket []byte
history []Update
filled, told string
// data is what runs' Data; holdsData the manifest's answer (nil: none read).
data bool
holdsData func(context.Context, string) (bool, bool)
}{
{name: "no have: the kept declaration's", filled: old},
{name: "a have naming the bucket's value", have: &Have{Module: "power", FP: fp, Composed: appliedDig},
bucket: appliedRaw, filled: old + "," + added, told: "the haves were read at the start", data: true},
{name: "a have applying names nothing that runs yet", have: &Have{Module: "power", FP: fp, Composed: appliedDig,
Phase: string(delivery.PhaseApplying)}, bucket: appliedRaw, filled: old, told: "the haves were read at the start"},
{name: "a data module from the kept declaration says Data as its manifest does", filled: old,
told: "the haves were read at the start", data: true,
holdsData: func(context.Context, string) (bool, bool) { return true, true }},
{name: "a have naming a revision replaced since", have: &Have{Module: "power", FP: fp, Composed: appliedDig},
bucket: newerRaw, history: []Update{{Key: ck, Value: appliedRaw, Revision: 3}, {Key: ck, Value: newerRaw, Revision: 7}},
filled: old + "," + added, told: "the haves were read at the start", data: true},
{name: "a have naming a composition the bucket no longer keeps", have: &Have{Module: "power", FP: fp, Composed: appliedDig},
bucket: newerRaw, filled: old, told: "the haves were read at the start"},
} {
t.Run(tc.name, func(t *testing.T) {
b := newBed(t)
b.kept = map[string][]json.RawMessage{"power": {filledFile("settings")}}
b.env.KeptFilled = func() map[string][]string { return map[string][]string{"power": {old}} }
if tc.have != nil {
b.haves.stored = map[string]Have{"power": *tc.have}
}
if tc.bucket != nil {
b.comps.before(ck, tc.bucket)
}
b.env.Revisions = func(_ context.Context, key string) ([]Update, error) { return tc.history, nil }
b.env.HoldsData = tc.holdsData
var told []string
var mu sync.Mutex
b.env.RunsChanged = func(why string) { mu.Lock(); told = append(told, why); mu.Unlock() }
filledNow := func() string {
for _, c := range b.ag.Everything() {
if c.Module == "power" {
if c.Data != tc.data {
t.Errorf("what runs says Data %v, want %v", c.Data, tc.data)
}
return strings.Join(c.Filled, ",")
}
}
return ""
}
if tc.told == "" { // no want, no have: nothing to start, and the kept declaration is what runs
a := New(b.env)
b.ag = a
if got := filledNow(); got != tc.filled {
t.Fatalf("filled %q, want %q", got, tc.filled)
}
return
}
stop := b.run()
defer stop()
eventually(t, "the filling was not told what runs at the start", func() bool {
mu.Lock()
defer mu.Unlock()
return len(told) > 0 && told[0] == tc.told
})
if got := filledNow(); got != tc.filled {
t.Fatalf("filled %q, want %q", got, tc.filled)
}
if got := b.appliedList(); len(got) != 0 {
t.Fatalf("applied %v: nothing has a want", got)
}
})
}
}
internal/apply/apply.go
+57
View File
@@ -312,6 +312,38 @@ type Scope struct {
Module string
// Others are the resources of every other module the node has a composition for.
Others []declaration.Resource
// Placed are the modules a want still places on the node (present, not removed), held ones included: no apply but
// the module's own removes a resource of one of them (issue 774: on 2026-10-29 the node's own apply on the bus node,
// composed without what its folded modules ran, removed the bus server and four held modules). Nil guards nothing.
Placed map[string]bool
// RemovedByWant are the modules whose want says removed. In the node's own apply (Module NodeModule) with this set
// non-nil, a resource of any other module is removed only when that module is here: a module the node runs folded
// in its own composition has no want, so Placed does not name it (issue 774, review of the re-land, round 1: nats on
// the bus node has none, and its nats.conf was guessed as nats's and removed). Nil guards by Placed alone.
RemovedByWant map[string]bool
}
// placedOwner is the module a want still places on the node that the record belongs to, other than the scope's own:
// the module recorded with it, else the one its id names (a resource the node's own composition carried for a folded
// module is recorded as the node's, its id naming the module). Empty when there is none.
func (sc *Scope) placedOwner(a store.Applied) string {
if sc == nil || (len(sc.Placed) == 0 && sc.RemovedByWant == nil) {
return ""
}
m := a.Module
if m == "" || m == store.NodeModule {
m, _ = declaration.ModuleGuess(a.ID)
}
if m == "" || m == sc.Module || m == store.NodeModule {
return ""
}
if sc.Placed[m] {
return m
}
if sc.Module == store.NodeModule && sc.RemovedByWant != nil && !sc.RemovedByWant[m] {
return m // a module the node's own composition carried, which no want removed
}
return ""
}
// owns reports whether a record belongs to the scope's module.
@@ -441,6 +473,31 @@ func applyScoped(
}
}
// **A module a want still places here is never removed by another's apply** (issue 774): its resources go only with
// its own apply or its want's removal. One that would is refused whole, before anything is changed, naming them: a
// composition that leaves them out lost what runs, and removing what runs is the worst a node can do with that.
if scope != nil {
var refused []string
for _, orphan := range known.Orphans(declared, origin) {
if !scope.owns(orphan) {
continue
}
if _, kept := heldByOthers[orphan.ID]; kept {
continue
}
if m := scope.placedOwner(orphan); m != "" {
refused = append(refused, orphan.ID+" of "+m)
}
}
if len(refused) > 0 {
sort.Strings(refused)
err := fmt.Errorf("the apply of %s would remove %s, of modules a want still places here; a module's resources "+
"go only with its own apply or its want's removal, so nothing was applied", scope.Module, strings.Join(refused, ", "))
log(" refused: " + err.Error())
return report, known, &Error{Resource: "the resources of modules placed here", Err: err, Done: report}
}
}
removeOrphan := func(orphan store.Applied) error {
var action, detail string
var err error
internal/apply/filled.go
+27 -3
View File
@@ -45,6 +45,30 @@ func WriteFilled(
log func(string),
keep Keep,
windows *Windows,
) (report Report, _ store.State, _ error) {
return WriteFilledWith(ctx, sys, d, known, path, content, run, log, keep, windows, Filled{})
}
// Filled is what the node-engine did to a filled file's content before its write.
type Filled struct {
// Secret says the node-engine filled a value into the content, a provision's `${secret:}` (novox/hq ADR 0570
// clause 6): the file is then written 0600 when the declaration gives no mode, as a file carrying a secret is.
Secret bool
}
// WriteFilledWith is WriteFilled for content the node-engine filled before the write (Filled).
func WriteFilledWith(
ctx context.Context,
sys system.System,
d *declaration.Declaration,
known store.State,
path string,
content []byte,
run Runner,
log func(string),
keep Keep,
windows *Windows,
filled Filled,
) (report Report, _ store.State, _ error) {
if log == nil {
log = func(string) {}
@@ -65,7 +89,7 @@ func WriteFilled(
return report, known, &Error{Resource: path, Err: fmt.Errorf("%d resources declare %s, and a filled file "+
"is written for exactly one, so nothing was written", len(declared), path)}
}
file := filledAs(declared[0], content)
file := filledAs(declared[0], content, filled.Secret)
if held, isHeld := known.HeldAt(file.ID); isHeld {
return report, known, &Error{Resource: file.ID, Err: fmt.Errorf("%s is held as found for a module not "+
"yet taken (%s), so nothing was written", path, held.Why)}
@@ -163,10 +187,10 @@ func WriteFilled(
}
// filledAs is the declared file with the filled content in place of the declaration's.
func filledAs(declared *declaration.File, content []byte) *declaration.File {
func filledAs(declared *declaration.File, content []byte, secret bool) *declaration.File {
f := *declared
f.Content, f.Sealed, f.VaultSealed, f.Bytes, f.Secrets = string(content), "", nil, "", nil
if f.Mode == "" && (declared.Secret() || len(declared.Secrets) > 0) {
if f.Mode == "" && (secret || declared.Secret() || len(declared.Secrets) > 0) {
// Written as the declaration would have written it: a file carrying a credential is not world-readable.
f.Mode = "0600"
}
internal/apply/module_scope_test.go
+72 -25
View File
@@ -3,6 +3,7 @@ package apply
import (
"context"
"path/filepath"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/declaration"
@@ -116,32 +117,78 @@ func TestARecordWithoutAnOwnerBelongsToTheModuleItsIdNames(t *testing.T) {
// leaves it as it is, whatever its type, until the module's own apply takes it (issue 1945, live on 2026-10-25 at 02:32:
// shanks and g14 lost systemd-resolved's guard, its resolver files and sudo's operator file, ace qbittorrent's server,
// because the node's own apply removed every record no longer in it, the others holding the very same resources or not).
//
// And a resource the node's own composition carried for a module a want still places here (a folded module of the old
// form, held) is never removed by the node's own apply when a composition leaves it out: the apply is refused whole and
// says why (issue 774, live on 2026-10-29 at 07:54 UTC on the bus node: the node's own composition, composed without what
// it ran once the kept declaration was read no more, removed the bus server, the proxy and three held modules). Only its
// want's removal lets it go.
func TestAResourceThatMovedToItsModulesCompositionIsLeftByTheNodesOwnApply(t *testing.T) {
dir := t.TempDir()
fact := filepath.Join(dir, "suffix")
gone := filepath.Join(dir, "gone.conf")
own := declare(t, `{"id":"systemd-resolved.fact-suffix","type":"file","path":"`+fact+`","content":"mesh\n"},
{"id":"old.gone","type":"file","path":"`+gone+`","content":"g\n"}`)
_, state, err := ApplyModule(context.Background(), archHost(t), own, Scope{Module: store.NodeModule},
state0(), store.OriginDeclared, noServices, nil, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
// systemd-resolved now has a composition of its own, not applied yet; old.gone is no module's any more.
moved := declare(t, `{"id":"systemd-resolved.fact-suffix","type":"file","path":"`+fact+`","content":"mesh\n","module":"systemd-resolved"}`)
report, state, err := ApplyModule(context.Background(), archHost(t), &declaration.Declaration{},
Scope{Module: store.NodeModule, Others: moved.Resources}, state, store.OriginDeclared, noServices, nil, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
if !exists(fact) {
t.Fatalf("the moved resource was removed: %+v", outcomeOf(report, "systemd-resolved.fact-suffix"))
}
if exists(gone) {
t.Fatal("a resource no composition holds any more was kept")
}
if _, still := state.Find("systemd-resolved.fact-suffix"); !still {
t.Fatal("the moved resource's record was dropped, so its module's apply would not know it")
for _, tc := range []struct {
name string
id string // the resource the node's own composition carried, and no longer does
moved bool // its module's own composition now declares it (Others)
placed bool // a want still places its module here
refused bool // the node's own apply is refused whole
stays bool // the resource is still on the machine and recorded
// wants names the modules whose want says removed (Scope.RemovedByWant); nil leaves the guard to Placed alone.
wants []string
}{
{name: "moved to its module's own composition", id: "systemd-resolved.fact-suffix", moved: true, placed: true, stays: true},
{name: "a held folded module, in no composition", id: "nats.server-conf", placed: true, refused: true, stays: true},
{name: "its module's want removed", id: "nats.server-conf"},
// Issue 774, review of the re-land, round 1: a module folded in the node's own has no want, so Placed does not name
// it; in the node's own apply only a module a want removed loses its resources.
{name: "a folded module with no want", id: "nats.server-conf", wants: []string{}, refused: true, stays: true},
{name: "a folded module a want removed", id: "nats.server-conf", wants: []string{"nats", "old"}},
} {
t.Run(tc.name, func(t *testing.T) {
dir := t.TempDir()
fact := filepath.Join(dir, "carried")
gone := filepath.Join(dir, "gone.conf")
own := declare(t, `{"id":"`+tc.id+`","type":"file","path":"`+fact+`","content":"mesh\n"},
{"id":"old.gone","type":"file","path":"`+gone+`","content":"g\n"}`)
_, state, err := ApplyModule(context.Background(), archHost(t), own, Scope{Module: store.NodeModule},
state0(), store.OriginDeclared, noServices, nil, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
module, _ := declaration.ModuleGuess(tc.id)
scope := Scope{Module: store.NodeModule}
if tc.moved {
moved := declare(t, `{"id":"`+tc.id+`","type":"file","path":"`+fact+`","content":"mesh\n","module":"`+module+`"}`)
scope.Others = moved.Resources
}
if tc.placed {
scope.Placed = map[string]bool{module: true}
}
if tc.wants != nil {
scope.RemovedByWant = map[string]bool{}
for _, m := range tc.wants {
scope.RemovedByWant[m] = true
}
}
report, after, err := ApplyModule(context.Background(), archHost(t), &declaration.Declaration{}, scope, state,
store.OriginDeclared, noServices, nil, nil, nil, nil)
if tc.refused != (err != nil) {
t.Fatalf("refused %v, want %v: %v", err != nil, tc.refused, err)
}
if tc.refused {
if !strings.Contains(err.Error(), tc.id+" of "+module) {
t.Fatalf("the refusal does not name what it kept: %v", err)
}
after = state
}
if exists(fact) != tc.stays {
t.Fatalf("the carried resource exists %v, want %v: %+v", exists(fact), tc.stays, outcomeOf(report, tc.id))
}
if _, still := after.Find(tc.id); still != tc.stays {
t.Fatalf("the carried resource's record kept %v, want %v", still, tc.stays)
}
if exists(gone) == !tc.refused {
t.Fatalf("a resource no composition holds any more exists %v, with the apply refused %v", exists(gone), tc.refused)
}
})
}
}
internal/apply/objectkeep.go
+111
View File
@@ -0,0 +1,111 @@
package apply
import (
"fmt"
"os"
"path/filepath"
"time"
"git.novox.be/novox/mesh-sdk/go/declaration"
)
// ObjectKeep is the node-engine's own keep of the content of files given by digest (novox/hq ADR 0486 §1, issue 924): one
// file per object, named as the object is, beside the node's state. **What a node once had it never asks the bus for
// again**, so neither the controller's sweep of the bucket (an object is removed a day after no composition of the
// controller names it) nor a bus that does not answer can refuse a composition the node already ran. The node's account
// may only read its bucket (ADR 0486 §2), so the keep is the node's, not the bucket's: nothing is written back.
//
// An object unused for KeptObjectsFor is let go when another is kept, so the keep holds what the node applied lately and
// not every export it ever ran. Every failure to keep is silent and costs only a fetch: the keep is a copy.
type ObjectKeep struct {
Dir string
// Now is the clock; nil is time.Now.
Now func() time.Time
}
// KeptObjectsFor is how long an object of the keep is held after it was last used.
const KeptObjectsFor = 14 * 24 * time.Hour
// KeepObjectsIn is the keep at dir; an empty dir keeps nothing.
func KeepObjectsIn(dir string) *ObjectKeep {
if dir == "" {
return nil
}
return &ObjectKeep{Dir: dir}
}
func (k *ObjectKeep) now() time.Time {
if k.Now != nil {
return k.Now()
}
return time.Now()
}
// get is the kept content of the object name when it is what f names, and marks it used.
func (k *ObjectKeep) get(name string, f *declaration.File) ([]byte, bool) {
if k == nil {
return nil, false
}
path := filepath.Join(k.Dir, name)
body, err := os.ReadFile(path)
if err != nil || int64(len(body)) != f.Size || f.Verify(body) != nil {
return nil, false
}
now := k.now()
_ = os.Chtimes(path, now, now)
return body, true
}
// put keeps body under name, unless it is kept already (then it is marked used), and lets go of what was unused too long.
func (k *ObjectKeep) put(name string, body []byte) {
if k == nil {
return
}
path := filepath.Join(k.Dir, name)
now := k.now()
if info, err := os.Stat(path); err == nil && info.Size() == int64(len(body)) {
_ = os.Chtimes(path, now, now)
return
}
if err := os.MkdirAll(k.Dir, 0o700); err != nil {
return
}
tmp, err := os.CreateTemp(k.Dir, ".put-*")
if err != nil {
return
}
_, werr := tmp.Write(body)
cerr := tmp.Close()
if werr != nil || cerr != nil || os.Chmod(tmp.Name(), 0o600) != nil || os.Rename(tmp.Name(), path) != nil {
_ = os.Remove(tmp.Name())
return
}
_ = os.Chtimes(path, now, now)
k.prune(now)
}
// prune lets go of every object unused for KeptObjectsFor.
func (k *ObjectKeep) prune(now time.Time) {
entries, err := os.ReadDir(k.Dir)
if err != nil {
return
}
for _, e := range entries {
info, err := e.Info()
if err != nil || !info.Mode().IsRegular() {
continue
}
if now.Sub(info.ModTime()) >= KeptObjectsFor {
_ = os.Remove(filepath.Join(k.Dir, e.Name()))
}
}
}
// never says, in a failed fetch's error before its cause (a transient one's detail keeps the tail), that the node's own
// keep did not hold the object either.
func (k *ObjectKeep) never() string {
if k == nil {
return ""
}
return fmt.Sprintf(" (this node's own keep of objects, %s, never held it)", k.Dir)
}
internal/apply/references.go
+24 -8
View File
@@ -27,13 +27,23 @@ type FetchObject func(ctx context.Context, name string, limit int64) ([]byte, er
// unchanged declaration, and a node that is asleep and has no bus, apply as they always did. One object is fetched
// once however many files name its digest.
func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetch FetchObject) error {
return ResolveFileReferencesKept(ctx, d, fetch, nil)
}
// ResolveFileReferencesKept is ResolveFileReferences with the node's own keep of objects (ObjectKeep): every content had
// (a file here that verifies, or an object of the bus) is kept, and a kept one is used before the bus is asked. **A node
// depends on its bucket only for content it never had** (issue 924, 2026-10-29: novox's own composition named the
// export mesh-vault runs by a digest whose object the controller's sweep had removed, so the vault's apply failed while
// it ran; the bucket's objects are kept only while a composition of the controller names them, and a node composing its
// own names objects the controller no longer knows of). A nil keep keeps nothing.
func ResolveFileReferencesKept(ctx context.Context, d *declaration.Declaration, fetch FetchObject, keep *ObjectKeep) error {
fetched := map[string][]byte{}
for i, r := range d.Resources {
f, ok := r.(*declaration.File)
if !ok || !f.IsReference() {
continue
}
body, err := referencedContent(ctx, f, fetch, fetched)
body, err := referencedContent(ctx, f, fetch, fetched, keep)
if err != nil {
return err
}
@@ -50,31 +60,37 @@ func ResolveFileReferences(ctx context.Context, d *declaration.Declaration, fetc
}
func referencedContent(ctx context.Context, f *declaration.File, fetch FetchObject,
fetched map[string][]byte) ([]byte, error) {
fetched map[string][]byte, keep *ObjectKeep) ([]byte, error) {
if body, ok := fetched[f.Digest]; ok {
return body, nil
}
if body, ok := alreadyThere(f); ok {
fetched[f.Digest] = body
return body, nil
}
name, err := declaration.ObjectName(f.Digest)
if err != nil {
return nil, fmt.Errorf("file %s (%s): %w", f.ID, f.Path, err)
}
if body, ok := alreadyThere(f); ok {
fetched[f.Digest] = body
keep.put(name, body)
return body, nil
}
if body, ok := keep.get(name, f); ok {
fetched[f.Digest] = body
return body, nil
}
if fetch == nil {
return nil, fmt.Errorf("file %s (%s) is given by digest %s and is not on this machine, and this node-engine "+
"has no link to the bus to fetch it from", f.ID, f.Path, f.Digest)
}
body, err := fetch(ctx, name, f.Size)
if err != nil {
return nil, fmt.Errorf("file %s (%s): fetching the object %s of this node's own object bucket: %w",
f.ID, f.Path, name, err)
return nil, fmt.Errorf("file %s (%s): fetching the object %s of this node's own object bucket%s: %w",
f.ID, f.Path, name, keep.never(), err)
}
if err := f.Verify(body); err != nil {
return nil, fmt.Errorf("%s (%s): %w; nothing was applied", f.ID, f.Path, err)
}
fetched[f.Digest] = body
keep.put(name, body)
return body, nil
}
internal/link/access_act_test.go
+110
View File
@@ -6,6 +6,10 @@ import (
"strings"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asker"
"git.novox.be/novox/mesh-sdk/go/asks"
)
// The take act's ask (novox/hq ADR 0325 §8): a hand-over's fields and the act's own in one signed object, under a
@@ -68,6 +72,112 @@ func TestTheTakeActRunsOnlyOnAWarrant(t *testing.T) {
if a.Said != "started" || len(given) != 1 || given[0].Warrant != "w-1" || given[0].Path != "/srv/media/movies" {
t.Fatalf("a warranted ask did not reach the act whole: %+v %+v", a, given)
}
// 774 (ADR 0563 §3): a hand-over asked through the node seat is the engine's own ask, performed only on the
// operator's warrant to it: never on the call, not on a warrant for another digest, once on a warrant heard twice,
// and not when the disk changed since it was asked.
t.Run("the node seat's hand-over acts only on the operator's warrant", func(t *testing.T) {
for _, row := range []struct {
name string
warrant func(asked asks.Ask, w asks.Warrant) asks.Warrant
diskAfter string
twice bool
done int
state string
}{
{name: "no warrant", done: 0, state: HandAsked},
{name: "the operator's yes", warrant: yes, done: 1, state: HandDone},
{name: "the yes heard twice", warrant: yes, twice: true, done: 1, state: HandDone},
{name: "a warrant for another digest", warrant: func(a asks.Ask, w asks.Warrant) asks.Warrant {
w = yes(a, w)
w.AskDigest = "sha256:another"
return w
}, done: 0, state: HandAsked},
{name: "a no", warrant: func(a asks.Ask, w asks.Warrant) asks.Warrant {
w = yes(a, w)
w.Option, w.Label, w.Level = "no", "No, leave it", asks.Acknowledge
return w
}, done: 0, state: HandRefused},
{name: "the disk changed", warrant: yes, diskAfter: "0:0 700 9:2", done: 0, state: HandRefused},
{name: "expired", warrant: func(a asks.Ask, w asks.Warrant) asks.Warrant {
return asks.Warrant{Ask: a.ID, Asker: w.Asker, Outcome: asks.OutcomeExpired, At: w.At}
}, done: 0, state: HandRefused},
} {
t.Run(row.name, func(t *testing.T) {
h, asked, done := anEngine(t)
disk := "1000:1000 755 9:1"
h.JudgeHandOver = func(path string) (Judged, error) {
return Judged{Subject: path, Path: path, Sets: "owner jo, group jo, mode 0750", Disk: disk,
Words: "A directory used as found."}, nil
}
if _, err := h.Call(EngineCall{Verb: ActHandOver, Caller: "shanks.node-tools",
Args: map[string]any{"path": "/srv/x", "why": "a test"}}); err != nil {
t.Fatal(err)
}
if len(*asked) != 1 || *done != 0 {
t.Fatalf("the call asked %d and performed %d", len(*asked), *done)
}
// The act is one key, and the open ask one key made by a create (ADR 0563 §3).
if acts, _ := h.State.Keys("act."); len(acts) != 1 || acts[0] != "act."+(*asked)[0].ID {
t.Fatalf("the act is kept as %v", acts)
}
if open, _ := h.State.Keys("open."); len(open) != 1 || !strings.HasPrefix(open[0], "open.hand-over.") {
t.Fatalf("the open ask is kept as %v", open)
}
if _, err := h.Call(EngineCall{Verb: ActHandOver, Caller: "x", Args: map[string]any{"path": "/srv/x"}}); err == nil {
t.Fatal("a second ask of the same path was asked while one is open")
}
if row.diskAfter != "" {
disk = row.diskAfter
}
if row.warrant != nil {
w := row.warrant((*asked)[0], asks.Warrant{Asker: asks.EngineAsker("n1"), At: time.Now()})
h.Heard(w)
if row.twice {
h.Heard(w)
}
}
got, _ := h.Acts(0)
acts := got.(map[string]any)["acts"].([]HandAct)
if *done != row.done || len(acts) != 1 || acts[0].State != row.state {
t.Fatalf("performed %d (want %d), acts %+v (want %s)", *done, row.done, acts, row.state)
}
if open, _ := h.State.Keys("open."); (len(open) == 1) != (row.state == HandAsked) {
t.Fatalf("an ask %s leaves its open key as %v", row.state, open)
}
if row.state == HandDone && (acts[0].By != "operator" || acts[0].Via == "" || acts[0].Ask == "") {
t.Fatalf("the act does not record the warrant: %+v", acts[0])
}
})
}
})
}
// yes is the operator's yes to an ask, as the router writes it.
func yes(a asks.Ask, w asks.Warrant) asks.Warrant {
w.Ask, w.Outcome, w.Option, w.Label, w.Level = a.ID, asks.OutcomeChosen, "yes", "Yes", asks.Approve
w.By = &asks.Person{Who: asks.Operator, Kind: "telegram", Verified: "user id verified"}
w.AskDigest, w.Proofs = a.Digest(), []string{"P1"}
return w
}
// anEngine is a node seat with its own state in a temporary folder, a router that keeps what it was asked, and a
// hand-over that counts what it performed.
func anEngine(t *testing.T) (*HandActs, *[]asks.Ask, *int) {
t.Helper()
var asked []asks.Ask
done := 0
state := &MemState{}
h := &HandActs{Node: "n1", State: state,
HandOver: func(path, by string) (string, error) { done++; return path + " handed over by " + by, nil }}
h.Asker = &asker.Client{Module: asks.EngineAsker("n1"), State: state,
Publish: func(subject string, body any, id string) error {
if a, ok := body.(asks.Ask); ok && subject == asks.AskSubject(asks.EngineAsker("n1")) {
asked = append(asked, a)
}
return nil
},
Record: func(string) (*stdio.StateEntry, error) { return nil, nil }}
return h, &asked, &done
}
// The controller names an access by its declared resource id exactly, which may hold a `/`: it reaches the act,
internal/link/engineseat.go
+382
View File
@@ -0,0 +1,382 @@
package link
import (
"context"
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asks"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/micro"
)
// The node seat node-engine, served on the engine's own link (novox/hq ADR 0563 §1).
//
// Each verb is answered on this node's subject alone, `mesh.seat.node-engine.tool.<verb>.<node>`, and on the same
// address naming its caller, `mesh.seat.node-engine.call.<verb>.<node>.<caller>`, whose last token the bus lets each user
// publish with its own name alone (novox/hq issue 365): the caller is that token, and a call on the subject naming none is
// from an unproven caller, recorded so. The body is the call's arguments; the answer is {result, error, node}, as every
// tool's (mesh-tools node-tools). One service, `node-engine`, answers discovery with each verb's seat, tool and node.
// EngineSeat is the seat.
const EngineSeat = catalogue.EngineSeat
// EngineToolSubject is where a verb of this node's seat is asked naming no caller.
func EngineToolSubject(verb, node string) string {
return "mesh.seat." + EngineSeat + ".tool." + verb + "." + node
}
// EngineCallPattern is where a verb of this node's seat is asked naming its caller, any caller.
func EngineCallPattern(verb, node string) string {
return "mesh.seat." + EngineSeat + ".call." + verb + "." + node + ".*"
}
// EngineEventSubject is where the seat says one of its events about this node.
func EngineEventSubject(event, node string) string {
return "mesh.seat." + EngineSeat + ".event." + event + "." + node
}
// EngineCall is one call of the seat: its verb, its caller as the bus proved it, and its arguments.
type EngineCall struct {
Verb string
Caller string
Args map[string]any
}
// EngineFunc answers one call: its result, or why it is refused.
type EngineFunc func(EngineCall) (any, error)
// CallerOfSubject is the caller a call's subject names for this node's verb: the last token of a caller-named subject,
// each `~` a dot, or stdio.Unproven on the subject that names none or on any other.
func CallerOfSubject(subject, verb, node string) string {
token, ok := strings.CutPrefix(subject, "mesh.seat."+EngineSeat+".call."+verb+"."+node+".")
if !ok || token == "" || strings.ContainsAny(token, ".*> ") {
return stdio.Unproven
}
return strings.ReplaceAll(token, "~", ".")
}
// MaxEngineCall bounds a call's arguments: a path, an access and a few words.
const MaxEngineCall = 16 << 10
// AnswerEngineCall is the envelope one call is answered with: the verb's result, or its refusal in words.
func AnswerEngineCall(handle EngineFunc, verb, node, subject string, body []byte) []byte {
envelope := map[string]any{"node": node}
say := func() []byte {
raw, err := json.Marshal(envelope)
if err != nil {
raw, _ = json.Marshal(map[string]any{"node": node, "error": "the answer could not be written: " + err.Error()})
}
return raw
}
args := map[string]any{}
switch {
case handle == nil:
envelope["error"] = "this engine serves no verb of " + EngineSeat
return say()
case len(body) > MaxEngineCall:
envelope["error"] = fmt.Sprintf("the call is %d bytes, more than a call of %s takes", len(body), EngineSeat)
return say()
case len(strings.TrimSpace(string(body))) > 0:
if err := json.Unmarshal(body, &args); err != nil {
envelope["error"] = "the call's arguments are not an object: " + err.Error()
return say()
}
}
result, err := handle(EngineCall{Verb: verb, Caller: CallerOfSubject(subject, verb, node), Args: args})
if err != nil {
envelope["error"] = err.Error()
} else {
envelope["result"] = result
}
return say()
}
// EngineSeats is a link that serves this node's seat. Optional, as Asked is.
type EngineSeats interface {
ServeEngineSeat(EngineFunc)
}
// ServeEngineSeat answers every verb of the seat on this node's subjects, as the service node-engine, which says each
// verb's seat, tool and node to discovery (novox/hq ADR 0197). A bus whose user list is older than the grant refuses the
// subscriptions, said by the client; a caller then hears nobody answer.
func (l *natsLink) ServeEngineSeat(handle EngineFunc) {
if handle == nil || l.conn == nil {
return
}
svc, err := micro.AddService(l.conn, micro.Config{Name: EngineSeat, Version: "1.0.0",
Description: "the node seat node-engine of " + l.node + " (novox/hq ADR 0563)", QueueGroupDisabled: true})
if err != nil {
return
}
for _, v := range catalogue.EngineVerbs() {
verb := v.Name
answer := micro.HandlerFunc(func(r micro.Request) {
_ = r.Respond(AnswerEngineCall(handle, verb, l.node, r.Subject(), r.Data()))
})
meta := map[string]string{"seat": EngineSeat, "tool": verb, "node": l.node}
_ = svc.AddEndpoint(verb, answer, micro.WithEndpointSubject(EngineToolSubject(verb, l.node)),
micro.WithEndpointMetadata(meta))
_ = svc.AddEndpoint(verb+"-called", answer, micro.WithEndpointSubject(EngineCallPattern(verb, l.node)),
micro.WithEndpointMetadata(meta))
}
l.engine = svc
}
// SayEngine says one of the seat's events about this node, on the events stream.
func (l *natsLink) SayEngine(event string, body any) error {
raw, err := json.Marshal(body)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, err = l.js.Publish(EngineEventSubject(event, l.node), raw, nats.Context(ctx))
return err
}
// The engine asks the operator under its own asker name (asks.EngineAsker) on the router's seat (ADR 0259 §3, as ADR
// 0563 §3 changes it): its ask and cancel on the seat's work stream, its warrants heard on its own decided subject, and
// its asks' state read from the router's records under its own name.
// RouterRecords is the router's bucket of asks, which each asker reads under its own name: the state `asks` of the
// operator-channel's holder, the messenger (mesh-catalog modules/messenger).
const RouterRecords = "messenger_asks"
// OperatorAsks is a link that asks the operator as this node's engine. Optional.
type OperatorAsks interface {
PublishAsk(subject string, body any, id string) error
RouterRecord(key string) (*stdio.StateEntry, error)
HearWarrants(asker string, heard func(asks.Warrant)) error
}
// PublishAsk publishes an ask or a cancel on the router's seat, once per id within the stream's duplicate window.
func (l *natsLink) PublishAsk(subject string, body any, id string) error {
raw, err := json.Marshal(body)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, err = l.js.Publish(subject, raw, nats.MsgId(id), nats.Context(ctx))
return err
}
// RouterRecord reads one key of the router's records, `<asker>.<id>`; nil when it holds none.
func (l *natsLink) RouterRecord(key string) (*stdio.StateEntry, error) {
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
msg, err := l.conn.RequestWithContext(ctx, "$JS.API.DIRECT.GET.KV_"+RouterRecords+".$KV."+RouterRecords+"."+key, nil)
if err != nil {
return nil, err
}
if status := msg.Header.Get("Status"); status == "404" {
return nil, nil
} else if status != "" {
return nil, fmt.Errorf("the router's record %s could not be read: %s %s", key, status, msg.Header.Get("Description"))
}
return &stdio.StateEntry{Key: key, Value: msg.Data}, nil
}
// HearWarrants hears the warrants for asker on its decided subject, each handed to heard as it arrives.
func (l *natsLink) HearWarrants(asker string, heard func(asks.Warrant)) error {
sub, err := l.conn.Subscribe(asks.DecidedSubject(asker), func(msg *nats.Msg) {
var w asks.Warrant
if json.Unmarshal(msg.Data, &w) == nil {
heard(w)
}
})
if err != nil {
return err
}
l.warrants = sub
return nil
}
// errNoAsking is a link that cannot ask the operator.
var errNoAsking = errors.New("the link open now cannot ask the operator")
// discoveryQuiet is how long the engine waits after the last answer to the bus's discovery, and discoveryMost the
// longest it waits in all.
const (
discoveryQuiet = 300 * time.Millisecond
discoveryMost = 2 * time.Second
)
// HeardServed asks the bus's discovery (novox/hq ADR 0197) whether a seat's verb is answered on this machine: an
// endpoint naming the seat, the verb as its tool and this node.
func (l *natsLink) HeardServed(seat, verb string) (bool, error) {
inbox := l.conn.NewRespInbox()
sub, err := l.conn.SubscribeSync(inbox)
if err != nil {
return false, err
}
defer func() { _ = sub.Unsubscribe() }()
if err := l.conn.PublishRequest("$SRV.INFO", inbox, nil); err != nil {
return false, fmt.Errorf("asking the bus who serves what: %w", err)
}
heard := false
deadline := time.Now().Add(discoveryMost)
for time.Now().Before(deadline) {
msg, err := sub.NextMsg(discoveryQuiet)
if err != nil {
break
}
var info micro.Info
if json.Unmarshal(msg.Data, &info) != nil {
continue
}
for _, e := range info.Endpoints {
node := e.Metadata["node"]
if node == "" {
node = info.ID
}
if e.Metadata["seat"] == seat && e.Metadata["tool"] == verb && node == l.node {
heard = true
}
}
}
return heard, nil
}
// HeardServed is the link open now asking the bus's discovery whether a seat's verb is answered on this machine.
func (q *Session) HeardServed(seat, verb string) (bool, error) {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
asker, ok := bus.(interface {
HeardServed(string, string) (bool, error)
})
if bus == nil || !ok {
return false, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked)
}
return asker.HeardServed(seat, verb)
}
// ---- the engine's acts in the bus's own storage (ADR 0563 §2, §3) -------------------------------------------------
// actsStore is this node's keys of ActsBucket over the link open now: bound, never created (the grants part creates
// it, as the node-engine's declared state).
func (l *natsLink) actsStore() (nats.KeyValue, error) {
kv, err := l.js.KeyValue(ActsBucket)
if err != nil {
return nil, fmt.Errorf("the node-engine's state %s cannot be bound (it is created from the node-engine's "+
"manifest; asked again once it is): %w", ActsBucket, err)
}
return kv, nil
}
// BusState is the engine's ActStore: its own node's keys of ActsBucket, `<node>.<key>`, over whichever link the session
// has open; with none open every read and write is refused, and nothing is kept anywhere else.
type BusState struct {
Session *Session
Node string
}
func (b BusState) kv() (nats.KeyValue, error) {
b.Session.init()
b.Session.mu.Lock()
bus := b.Session.bus
b.Session.mu.Unlock()
l, ok := bus.(interface{ actsStore() (nats.KeyValue, error) })
if bus == nil || !ok {
return nil, fmt.Errorf("%w: this machine has no link to the bus open now, where its acts are kept", ErrNotAsked)
}
return l.actsStore()
}
func (b BusState) key(k string) string { return b.Node + "." + k }
// changed is a write that lost its compare-and-set, as stdio.ErrChanged.
func changed(err error) error {
var api *nats.APIError
if errors.Is(err, nats.ErrKeyExists) || (errors.As(err, &api) && api.ErrorCode == nats.JSErrCodeStreamWrongLastSequence) {
return stdio.ErrChanged
}
return err
}
func (b BusState) Get(key string) (*stdio.StateEntry, error) {
kv, err := b.kv()
if err != nil {
return nil, err
}
e, err := kv.Get(b.key(key))
if errors.Is(err, nats.ErrKeyNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
return &stdio.StateEntry{Key: key, Value: e.Value(), Revision: e.Revision()}, nil
}
func (b BusState) Create(key string, value any) (uint64, error) {
kv, err := b.kv()
if err != nil {
return 0, err
}
raw, err := json.Marshal(value)
if err != nil {
return 0, err
}
rev, err := kv.Create(b.key(key), raw)
return rev, changed(err)
}
func (b BusState) Update(key string, value any, revision uint64) (uint64, error) {
kv, err := b.kv()
if err != nil {
return 0, err
}
raw, err := json.Marshal(value)
if err != nil {
return 0, err
}
rev, err := kv.Update(b.key(key), raw, revision)
return rev, changed(err)
}
func (b BusState) Delete(key string) error {
kv, err := b.kv()
if err != nil {
return err
}
return kv.Delete(b.key(key))
}
// Keys are this node's keys under prefix, without the node: a watch of `<node>.>` alone, the one filter the node's
// grant names, its values left out.
func (b BusState) Keys(prefix string) ([]string, error) {
kv, err := b.kv()
if err != nil {
return nil, err
}
w, err := kv.Watch(b.Node+".>", nats.MetaOnly(), nats.IgnoreDeletes())
if err != nil {
return nil, err
}
defer func() { _ = w.Stop() }()
var out []string
for {
select {
case e := <-w.Updates():
if e == nil {
sort.Strings(out)
return out, nil
}
if k := strings.TrimPrefix(e.Key(), b.Node+"."); strings.HasPrefix(k, prefix) {
out = append(out, k)
}
case <-time.After(10 * time.Second):
return nil, errors.New("the node-engine's acts could not be listed within 10s")
}
}
}
internal/link/handacts.go
+769
View File
@@ -0,0 +1,769 @@
package link
import (
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"sort"
"strconv"
"strings"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/asker"
"git.novox.be/novox/mesh-sdk/go/asks"
)
// The node seat node-engine's acts (novox/hq ADR 0563).
//
// A hand-over and an access act are the engine's own asks, performed only on the warrant (§3): a call performs nothing.
// The engine judges every value as it does for the controller's signed ask, then refuses in words or asks the operator
// under its own asker name (asks.EngineAsker) at the level approve, binding the digest of the verb, the node, every
// argument and the owner, group and mode it will set. A warrant is claimed once by compare-and-set in the engine's own
// state (asker.Client.Take), held to the ask it answers, and to the act bound and the disk as judged, judged again now;
// only then is the act performed and recorded with the warrant. A decline, an expiry or a changed disk says refused and
// changes nothing. One ask is open per path or access.
//
// The read-only verbs need no warrant (§2): judge-directories changes nothing; setuid-search is bounded here, one at a
// time and at most one started in an hour. Every act, with who asked as the runtime proved it, is kept in the engine's own
// declared state acts (ActsBucket, created by the grants part), under its own node's keys alone, read by the verb acts.
// The acts of the seat, as its verbs name them.
const (
ActHandOver = "hand-over"
ActAccess = "access-act"
ActJudgeDirectories = "judge-directories"
ActSetuidSearch = "setuid-search"
)
// The states of an act.
const (
HandAsked = "asked"
HandDone = "done"
HandRefused = "refused"
HandAnswered = "answered"
)
// SearchEvery is how often a setuid search may be started on a node (ADR 0563 §2): one in any hour.
const SearchEvery = time.Hour
// askMargin is how long past an ask's expiry the router is given to say it ended, before the engine closes it itself.
const askMargin = 5 * time.Minute
// ActsBucket is the node-engine's declared state acts (its manifest's state, ADR 0563 §2, §3), the bucket the grants
// part creates (mesh-sdk buscompose EngineActsBucket): keys `<node>.act.<id>`, `<node>.open.<act>.<digest>` and
// `<node>.ask.<id>`, each written by this node's engine alone.
const ActsBucket = "mesh-host_acts"
// Judged is what the engine judged of one act before it asks: what it is about, what it will set (the owner, group and
// mode, in words), and what the disk holds now (Disk), compared again when the warrant arrives.
type Judged struct {
// Subject is the path of a hand-over, or the access of an access act.
Subject string
// Path is the directory the act changes, as the engine states it.
Path string
// Sets is the owner, group and mode the act will set, in words; bound in the ask's digest.
Sets string
// Disk is what the engine read on disk when it judged: owner, group, mode and the file it is.
Disk string
// Words is what the operator is shown of it, without a path (ADR 0234 §6): the path is in the whole words alone.
Words string
}
// HandAct is one act asked through the seat, as acts reads it.
type HandAct struct {
ID string `json:"id"`
Act string `json:"act"`
Subject string `json:"subject,omitempty"`
Caller string `json:"caller"`
Why string `json:"why,omitempty"`
At time.Time `json:"at"`
State string `json:"state"`
Said string `json:"said,omitempty"`
Ended time.Time `json:"ended,omitempty"`
// Ask is the operator's ask of a hand-over or an access act; Sets and Disk what was judged when it was asked.
Ask string `json:"ask,omitempty"`
Sets string `json:"sets,omitempty"`
Disk string `json:"disk,omitempty"`
Path string `json:"path,omitempty"`
// Expires is when the ask ends unanswered.
Expires time.Time `json:"expires,omitempty"`
// By, Via, Proofs are the operator's warrant, recorded with the act (ADR 0563 §3).
By string `json:"by,omitempty"`
Via string `json:"via,omitempty"`
Proofs []string `json:"proofs,omitempty"`
// Auto says the engine asked it itself, on finding a library's root holding data outside its group (§4).
Auto bool `json:"auto,omitempty"`
}
// RootVerdict is the engine's answer to root-free (ADR 0563 §6), in the controller's words.
type RootVerdict struct {
Machine string `json:"machine"`
Free bool `json:"free"`
Why string `json:"why"`
Judged time.Time `json:"judged"`
}
// HandActs is the engine's side of the seat. Every func is the engine's own judgement or act; nil refuses.
type HandActs struct {
Node string
// State is where the acts and the asks are kept: this node's keys of ActsBucket (BusState), each key one act.
State ActStore
// Asker asks the operator as asks.EngineAsker(Node), with State as its store.
Asker *asker.Client
// JudgeHandOver judges a hand-over's path, refusing in words; HandOver records it on the warrant.
JudgeHandOver func(path string) (Judged, error)
HandOver HandOverFunc
// JudgeAccess judges an access act on an access, refusing in words; AccessAct runs it on the warrant.
JudgeAccess func(act, access string) (Judged, error)
AccessAct AccessActFunc
// JudgeDirectories, SetuidSearch and Searching are the read-only verbs' and whether a search runs now.
JudgeDirectories JudgeDirectoriesFunc
SetuidSearch SetuidSearchFunc
Searching func() bool
// RootFree is this machine's root-free verdict now.
RootFree func() RootVerdict
// Say says one of the seat's events, asked, done or refused; nil says nothing.
Say func(event string, body any)
Now func() time.Time
// Nonce keeps two asks of one second apart; nil is random.
Nonce func() string
mu sync.Mutex
}
func (h *HandActs) now() time.Time {
if h.Now != nil {
return h.Now()
}
return time.Now()
}
func (h *HandActs) nonce() string {
if h.Nonce != nil {
return h.Nonce()
}
var b [2]byte
_, _ = rand.Read(b[:])
return hex.EncodeToString(b[:])
}
func (h *HandActs) say(event string, a HandAct) {
if h.Say != nil {
h.Say(event, map[string]any{"act": a.Act, "subject": a.Subject, "caller": a.Caller, "ask": a.Ask,
"state": a.State, "said": a.Said, "node": h.Node})
}
}
// Call is the seat's one entry: a verb, the caller as the runtime proved it, and its arguments.
func (h *HandActs) Call(c EngineCall) (any, error) {
arg := func(k string) string { s, _ := c.Args[k].(string); return strings.TrimSpace(s) }
switch c.Verb {
case ActHandOver:
return h.AskHandOver(c.Caller, arg("path"), arg("why"), false)
case ActAccess:
return h.AskAccess(c.Caller, arg("act"), arg("access"), arg("why"), false)
case ActJudgeDirectories:
return h.Judge(c.Caller)
case ActSetuidSearch:
return h.Search(c.Caller, arg("why"))
case "root-free":
if h.RootFree == nil {
return nil, errors.New("this engine judges no machine root-free")
}
return map[string]any{"machines": []RootVerdict{h.RootFree()}}, nil
case "acts":
if id := arg("id"); id != "" {
return h.Act(id)
}
limit, _ := strconv.Atoi(arg("limit"))
if f, ok := c.Args["limit"].(float64); ok {
limit = int(f)
}
return h.Acts(limit)
}
return nil, fmt.Errorf("the node seat %s serves no verb %q", EngineSeat, c.Verb)
}
// ---- the record --------------------------------------------------------------------------------------------------
// ActStore is the engine's own state, this node's keys alone (BusState prefixes them with the node): each act one key,
// written by compare-and-set on its revision, an open ask one key made by a create.
type ActStore interface {
Get(key string) (*stdio.StateEntry, error)
Create(key string, value any) (uint64, error)
Update(key string, value any, revision uint64) (uint64, error)
Delete(key string) error
// Keys are the keys under prefix.
Keys(prefix string) ([]string, error)
}
func actKey(id string) string { return "act." + id }
// openKey is the one key an open ask of act on subject holds: made by a create, so a second ask of it is refused.
func openKey(act, subject string) string {
sum := sha256.Sum256([]byte(subject))
return "open." + act + "." + hex.EncodeToString(sum[:12])
}
// read is every act kept, oldest first.
func (h *HandActs) read() ([]HandAct, error) {
keys, err := h.State.Keys("act.")
if err != nil {
return nil, fmt.Errorf("the engine's acts cannot be listed: %w", err)
}
var out []HandAct
for _, k := range keys {
a, _, err := h.readOne(k)
if err != nil {
return nil, err
}
if a != nil {
out = append(out, *a)
}
}
sort.SliceStable(out, func(i, j int) bool { return out[i].At.Before(out[j].At) })
return out, nil
}
func (h *HandActs) readOne(key string) (*HandAct, uint64, error) {
e, err := h.State.Get(key)
if err != nil || e == nil {
return nil, 0, err
}
var a HandAct
if err := json.Unmarshal(e.Value, &a); err != nil {
return nil, 0, fmt.Errorf("the engine's act %s cannot be read: %w", key, err)
}
return &a, e.Revision, nil
}
// judgedKey keeps the node's judgements and refused searches, the last keptJudged of them: any caller may ask them, so
// they roll in one key and never add one (ADR 0563 §2: a wrong caller gains nothing, and fills no bucket).
const (
judgedKey = "judged"
keptJudged = 50
)
// roll adds a to the rolling key by compare-and-set, the oldest falling out.
func (h *HandActs) roll(a HandAct) error {
for range 5 {
e, err := h.State.Get(judgedKey)
if err != nil {
return err
}
var kept []HandAct
if e != nil {
if err := json.Unmarshal(e.Value, &kept); err != nil {
kept = nil // an unreadable roll is started again, never a reason to refuse
}
}
kept = append(kept, a)
if len(kept) > keptJudged {
kept = kept[len(kept)-keptJudged:]
}
if e == nil {
_, err = h.State.Create(judgedKey, kept)
} else {
_, err = h.State.Update(judgedKey, kept, e.Revision)
}
if !errors.Is(err, stdio.ErrChanged) {
return err
}
}
return errors.New("the engine's judgements kept changing under its write")
}
func (h *HandActs) rolled() ([]HandAct, error) {
e, err := h.State.Get(judgedKey)
if err != nil || e == nil {
return nil, err
}
var kept []HandAct
if err := json.Unmarshal(e.Value, &kept); err != nil {
return nil, fmt.Errorf("the engine's judgements cannot be read: %w", err)
}
return kept, nil
}
func (h *HandActs) record(a HandAct) error {
_, err := h.State.Create(actKey(a.ID), a)
return err
}
// end marks the open act of an ask ended, once, by compare-and-set on its revision, and frees its open key: false when
// it was not open.
func (h *HandActs) end(ask string, set func(*HandAct)) (HandAct, bool, error) {
for range 5 {
a, rev, err := h.readOne(actKey(ask))
if err != nil || a == nil || a.State != HandAsked {
return HandAct{}, false, err
}
set(a)
a.Ended = h.now().UTC()
if _, err := h.State.Update(actKey(ask), *a, rev); errors.Is(err, stdio.ErrChanged) {
continue
} else if err != nil {
return HandAct{}, false, err
}
_ = h.State.Delete(openKey(a.Act, a.Subject))
return *a, true, nil
}
return HandAct{}, false, errors.New("the engine's act " + ask + " kept changing under its write")
}
// Act is the verb acts with an id: that one act.
func (h *HandActs) Act(id string) (any, error) {
a, _, err := h.readOne(actKey(id))
if err != nil {
return nil, err
}
if a == nil {
return nil, fmt.Errorf("this engine keeps no act %q", id)
}
return map[string]any{"node": h.Node, "acts": []HandAct{*a}}, nil
}
// Acts is the verb acts: every act kept, newest first, at most limit when given.
func (h *HandActs) Acts(limit int) (any, error) {
acts, err := h.read()
if err != nil {
return nil, err
}
judged, err := h.rolled()
if err != nil {
return nil, err
}
acts = append(acts, judged...)
sort.SliceStable(acts, func(i, j int) bool { return acts[i].At.Before(acts[j].At) })
out := make([]HandAct, 0, len(acts))
for i := len(acts) - 1; i >= 0; i-- {
out = append(out, acts[i])
}
if limit > 0 && len(out) > limit {
out = out[:limit]
}
return map[string]any{"node": h.Node, "acts": out}, nil
}
// ---- the read-only verbs (§2) -------------------------------------------------------------------------------------
// Judge is judge-directories: the engine's judgement, read-only, recorded with its caller.
func (h *HandActs) Judge(caller string) (any, error) {
if h.JudgeDirectories == nil {
return nil, errors.New("this engine judges no directory")
}
said, err := h.JudgeDirectories(caller)
a := HandAct{ID: h.id(ActJudgeDirectories), Act: ActJudgeDirectories, Caller: caller, At: h.now().UTC(), State: HandAnswered}
if err != nil {
a.State, a.Said = HandRefused, err.Error()
}
if rerr := h.roll(a); rerr != nil && err == nil {
return nil, rerr
}
if err != nil {
return nil, err
}
return map[string]any{"node": h.Node, "said": said}, nil
}
// Search is setuid-search: refused while one runs and when one was started within the hour, naming when the next may
// start; otherwise the last search is thrown away and a full one starts, recorded with its caller.
func (h *HandActs) Search(caller, why string) (any, error) {
if h.SetuidSearch == nil {
return nil, errors.New("this engine keeps no search for setuid programs")
}
h.mu.Lock()
defer h.mu.Unlock()
now := h.now().UTC()
if h.Searching != nil && h.Searching() {
return nil, errors.New("a search for setuid programs runs on this machine now; no second is started while it does")
}
// The last start is a key of its own, claimed by compare-and-set before the search starts: no number of other acts,
// nor the record of acts, moves it (ADR 0563 §2).
if err := h.claimSearch(now, caller); err != nil {
return nil, err
}
said, err := h.SetuidSearch(caller)
a := HandAct{ID: h.id(ActSetuidSearch), Act: ActSetuidSearch, Caller: caller, Why: why, At: now, State: HandDone, Said: said}
if err != nil {
a.State, a.Said = HandRefused, err.Error()
}
// A search started is bounded to one an hour and kept as an act of its own; a refused one, which any caller may
// cause, rolls with the judgements.
record := h.record
if err != nil {
record = h.roll
}
if rerr := record(a); rerr != nil {
return nil, rerr
}
if err != nil {
return nil, err
}
return map[string]any{"node": h.Node, "said": said}, nil
}
// searchKey keeps the last search's start on this node, and libKey each library's own take ask: keys of their own, which
// no other act's key moves, kept for the bucket's age (ninety days, the node-engine's manifest).
const searchKey = "search.last"
func libKey(access string) string {
sum := sha256.Sum256([]byte(access))
return "lib." + hex.EncodeToString(sum[:12]) + ".asked"
}
// started is the last search's start as kept, and who asked it.
type started struct {
At time.Time `json:"at"`
By string `json:"by"`
}
// claimSearch refuses a search within SearchEvery of the last start, naming when the next may start, and otherwise
// claims this one by compare-and-set: a second claim at the same moment loses.
func (h *HandActs) claimSearch(now time.Time, caller string) error {
e, err := h.State.Get(searchKey)
if err != nil {
return err
}
var rev uint64
if e != nil {
var last started
if err := json.Unmarshal(e.Value, &last); err != nil {
return fmt.Errorf("the last search's start cannot be read, so none is started: %w", err)
}
if now.Sub(last.At) < SearchEvery {
return fmt.Errorf("a search was started at %s by %s, and one is started in any hour: the next may start "+
"at %s", last.At.Format(time.RFC3339), last.By, last.At.Add(SearchEvery).Format(time.RFC3339))
}
rev = e.Revision
}
claim := started{At: now, By: caller}
if rev == 0 {
_, err = h.State.Create(searchKey, claim)
} else {
_, err = h.State.Update(searchKey, claim, rev)
}
if errors.Is(err, stdio.ErrChanged) {
return errors.New("another search was started at this moment; no second is started")
}
return err
}
// ---- the asks (§3, §4) --------------------------------------------------------------------------------------------
func (h *HandActs) id(act string) string {
return act + "-" + h.now().UTC().Format("20060102t150405") + "-" + h.nonce()
}
// bound is the act an ask's yes binds: the verb, the node, every argument and what it will set (ADR 0563 §3).
func (h *HandActs) bound(a HandAct) asks.Act {
act := asks.Act{"verb": a.Act, "node": h.Node, "arg.subject": a.Subject, "arg.why": a.Why, "sets": a.Sets}
if a.Path != "" {
act["path"] = a.Path
}
return act
}
// AskHandOver is hand-over: judged, refused in words or asked of the operator. auto is the engine asking by itself.
func (h *HandActs) AskHandOver(caller, path, why string, auto bool) (any, error) {
if h.JudgeHandOver == nil || h.HandOver == nil {
return nil, errors.New("this engine takes no hand-over")
}
j, err := h.JudgeHandOver(path)
if err != nil {
return nil, err
}
return h.ask(ActHandOver, caller, why, auto, j)
}
// AskAccess is access-act: the act take on an access, judged on the library's home, refused or asked of the operator.
func (h *HandActs) AskAccess(caller, act, access, why string, auto bool) (any, error) {
if h.JudgeAccess == nil || h.AccessAct == nil {
return nil, errors.New("this engine runs no act on an accessed folder")
}
if act != ActTake {
return nil, fmt.Errorf("the act asked is %q, and the one act on an access is %q", act, ActTake)
}
j, err := h.JudgeAccess(act, access)
if err != nil {
return nil, err
}
return h.ask(ActAccess, caller, why, auto, j)
}
// AskedFor says whether an act on subject was ever asked by the engine itself: a library's take is asked by itself once
// (ADR 0563 §4), and again only by hand.
func (h *HandActs) AskedFor(act, subject string) bool {
if act == ActAccess {
e, err := h.State.Get(libKey(subject))
return err != nil || e != nil // unread is never a reason to ask again
}
acts, err := h.read()
if err != nil {
return true
}
for _, a := range acts {
if a.Act == act && a.Subject == subject && (a.Auto || a.State == HandAsked) {
return true
}
}
return false
}
func (h *HandActs) ask(act, caller, why string, auto bool, j Judged) (any, error) {
if h.Asker == nil {
return nil, errors.New("this engine reaches no operator to ask: nothing was asked and nothing changed")
}
h.mu.Lock()
defer h.mu.Unlock()
now := h.now().UTC()
a := HandAct{ID: h.id(act), Act: act, Subject: j.Subject, Path: j.Path, Caller: caller, Why: why, At: now,
State: HandAsked, Sets: j.Sets, Disk: j.Disk, Expires: now.Add(asks.ApproveLasts - time.Minute), Auto: auto}
a.Ask = a.ID
// One ask open per path or access: its open key made by a create, which a second ask cannot make.
if _, err := h.State.Create(openKey(act, j.Subject), map[string]string{"ask": a.Ask}); errors.Is(err, stdio.ErrChanged) {
held := "another"
if e, _ := h.State.Get(openKey(act, j.Subject)); e != nil {
var o struct {
Ask string `json:"ask"`
}
if json.Unmarshal(e.Value, &o) == nil && o.Ask != "" {
held = o.Ask
}
}
return nil, fmt.Errorf("%s of %s is asked already: the ask %s waits for the operator's answer", act, j.Subject, held)
} else if err != nil {
return nil, err
}
if err := h.record(a); err != nil {
_ = h.State.Delete(openKey(act, j.Subject))
return nil, err
}
bind, err := asks.ActDigest(h.bound(a))
if err != nil {
return nil, err
}
headline, does, stays := "Hand a directory on "+h.Node+" to the mesh?", "The mesh gives it "+j.Sets+".",
"The directory stays as it was found."
if act == ActAccess {
headline, does, stays = "Keep the library "+strings.TrimPrefix(j.Subject, "library:")+" on "+h.Node+"?",
"The library's group, group write and setgid are set on everything under its root.",
"The library waits and nothing under its root changes."
}
if len([]rune(headline)) > asks.HeadlineLength {
headline = string([]rune(headline)[:asks.HeadlineLength])
}
asked := asks.Ask{ID: a.Ask, Headline: headline, Who: asks.Operator,
Explanation: j.Words + " Your yes sets " + j.Sets + "; nothing changes until you answer.",
Whole: j.Words + " (" + j.Path + ") Your yes sets " + j.Sets + "; nothing changes until you answer.",
Details: "Asked by " + caller + " on " + h.Node + orWhy(why),
Options: []asks.Option{{ID: "yes", Label: "Yes", Does: does, Level: asks.Approve, Binds: bind},
{ID: "no", Label: "No, leave it", Does: stays, Level: asks.Acknowledge}},
Expires: a.Expires, OnExpiry: stays, About: EngineSeat + "-" + h.Node + "-" + act}
if err := h.Asker.Ask(asked); err != nil {
_, _, _ = h.end(a.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, "the operator could not be asked: "+err.Error() })
return nil, fmt.Errorf("the operator could not be asked, and nothing changed: %w", err)
}
// Once asked, a library's own take ask is kept's own take ask: once, in a key of its own (ADR 0563 §4).
if auto && act == ActAccess {
_, _ = h.State.Create(libKey(j.Subject), map[string]any{"ask": a.Ask, "at": now})
}
h.say(HandAsked, a)
return map[string]any{"asked": a.Ask, "node": h.Node, "says": "the operator is asked on their channel; only " +
"their yes changes anything. acts shows where it stands."}, nil
}
func orWhy(why string) string {
if why == "" {
return ""
}
return ": " + why
}
// Warranted is the act on the operator's answer, handed once by asker.Client.Take after it held the warrant to the
// ask: a no is recorded refused; a yes is held to the act the option bound and the disk as judged, judged again now,
// and only then performed and recorded done with the warrant.
func (h *HandActs) Warranted(o asks.Option, w asks.Warrant) error {
a, _, err := h.readOne(actKey(w.Ask))
if err != nil {
return err
}
if a == nil || a.State != HandAsked {
return fmt.Errorf("the warrant answers the ask %s, which is not open here: nothing is done", w.Ask)
}
refuse := func(why string) error {
ended, ok, err := h.end(w.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, why; warrantOn(x, w) })
if err == nil && ok {
h.say(HandRefused, ended)
}
return err
}
if o.ID != "yes" {
return refuse("the operator said no")
}
var j Judged
switch a.Act {
case ActHandOver:
j, err = h.JudgeHandOver(a.Subject)
case ActAccess:
j, err = h.JudgeAccess(ActTake, a.Subject)
default:
err = fmt.Errorf("the act %s is not performed on a warrant", a.Act)
}
if err != nil {
return refuse("judged again on the operator's yes, it is refused: " + err.Error())
}
now := *a
now.Sets, now.Path = j.Sets, j.Path
if err := o.Performs(h.bound(now)); err != nil {
return refuse("what the yes bound is not what would be done now: " + err.Error())
}
if j.Disk != a.Disk {
return refuse("the disk changed since the operator was asked (" + a.Disk + ", now " + j.Disk + "): ask again")
}
// Recorded with the way it came (ADR 0563 §5): the operator's yes to this engine's ask, not a terminal's line.
by := operatorOf(w) + ", to " + EngineSeat + "'s ask " + w.Ask
var said string
switch a.Act {
case ActHandOver:
said, err = h.HandOver(a.Subject, by)
case ActAccess:
said, err = h.AccessAct(AccessAct{Act: ActTake, Access: a.Subject, Path: j.Path, Warrant: w.Ask, By: by})
}
if err != nil {
return refuse(err.Error())
}
ended, ok, err := h.end(w.Ask, func(x *HandAct) { x.State, x.Said = HandDone, said; warrantOn(x, w) })
if err == nil && ok {
h.say(HandDone, ended)
}
return err
}
func warrantOn(a *HandAct, w asks.Warrant) {
a.Proofs = w.Proofs
if w.By != nil {
a.By, a.Via = w.By.Who, w.By.Kind
if w.By.Verified != "" {
a.Via += " (" + w.By.Verified + ")"
}
}
}
func operatorOf(w asks.Warrant) string {
if w.By == nil {
return "the operator"
}
return "the " + w.By.Who + " via " + w.By.Kind
}
// Heard is a warrant heard on the engine's decided subject: taken once (asker.Client.Take), and an ask that ended with
// no choice recorded refused.
func (h *HandActs) Heard(w asks.Warrant) {
if h.Asker == nil {
return
}
// A warrant the ask kept here does not hold to (another digest, another option, too late) ends nothing: it is no
// answer to the ask, which stays open for the operator's.
if handled, _ := h.Asker.Take(w, h.Warranted); handled == asker.Recorded {
why := "the ask ended " + string(w.Outcome)
if w.Words != "" {
why += ": " + w.Words
}
if ended, ok, _ := h.end(w.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, why; warrantOn(x, w) }); ok {
h.say(HandRefused, ended)
}
}
}
// Reconcile catches up what the engine missed while it was away: a warrant the router holds for an open ask is taken,
// and an ask past its expiry and a margin is closed refused, expired.
func (h *HandActs) Reconcile() {
if h.Asker == nil {
return
}
acts, err := h.read()
if err != nil {
return
}
for _, a := range acts {
if a.State != HandAsked || a.Ask == "" {
continue
}
if _, w, err := h.Asker.Where(a.Ask); err == nil && w != nil {
h.Heard(*w)
continue
}
if !a.Expires.IsZero() && h.now().After(a.Expires.Add(askMargin)) {
if ended, ok, _ := h.end(a.Ask, func(x *HandAct) { x.State, x.Said = HandRefused, "the ask expired unanswered" }); ok {
h.say(HandRefused, ended)
}
}
}
}
// MemState is an ActStore in memory: an engine's acts while no bus is reached in a test.
type MemState struct {
mu sync.Mutex
keys map[string]stdio.StateEntry
top uint64
}
func (m *MemState) Get(key string) (*stdio.StateEntry, error) {
m.mu.Lock()
defer m.mu.Unlock()
if e, ok := m.keys[key]; ok {
return &e, nil
}
return nil, nil
}
func (m *MemState) Create(key string, value any) (uint64, error) { return m.put(key, value, 0, true) }
func (m *MemState) Update(key string, value any, revision uint64) (uint64, error) {
return m.put(key, value, revision, false)
}
func (m *MemState) put(key string, value any, revision uint64, create bool) (uint64, error) {
raw, err := json.Marshal(value)
if err != nil {
return 0, err
}
m.mu.Lock()
defer m.mu.Unlock()
was, kept := m.keys[key]
if (create && kept) || (!create && (!kept || was.Revision != revision)) {
return 0, stdio.ErrChanged
}
if m.keys == nil {
m.keys = map[string]stdio.StateEntry{}
}
m.top++
m.keys[key] = stdio.StateEntry{Key: key, Value: raw, Revision: m.top}
return m.top, nil
}
func (m *MemState) Delete(key string) error {
m.mu.Lock()
defer m.mu.Unlock()
delete(m.keys, key)
return nil
}
func (m *MemState) Keys(prefix string) ([]string, error) {
m.mu.Lock()
defer m.mu.Unlock()
var out []string
for k := range m.keys {
if strings.HasPrefix(k, prefix) {
out = append(out, k)
}
}
sort.Strings(out)
return out, nil
}
internal/link/handover.go
+15 -5
View File
@@ -76,9 +76,13 @@ const MaxHandOverBy = 200
// HandOverCheck is what an ask is held to: the mesh's key, this node's name, the clock, and the nonces taken.
type HandOverCheck struct {
Signer ed25519.PublicKey
Node string
Now func() time.Time
Nonces *Nonces
// SignerNow, when set, is the controller's key as the engine trusts it at this ask, read per ask in place of Signer:
// nil once the controller's lease is handed away, which refuses every ask signed by its key (novox/hq ADR 0563 §7,
// ADR 0550 §2).
SignerNow func() ed25519.PublicKey
Node string
Now func() time.Time
Nonces *Nonces
}
// Nonces are the nonces of asks taken, each kept until its ask expires: an ask is taken once. Kept in memory —
@@ -157,8 +161,14 @@ func believedBytesWithin(check HandOverCheck, context string, withPath bool, bod
if err := json.Unmarshal(body, &signed); err != nil {
return HandOverAsk{}, nil, "the ask is not a signed ask: " + err.Error()
}
if len(check.Signer) != ed25519.PublicKeySize ||
!ed25519.Verify(check.Signer, append([]byte(context), signed.Ask...), signed.Signature) {
signer := check.Signer
if check.SignerNow != nil {
if signer = check.SignerNow(); signer == nil {
return HandOverAsk{}, nil, "the controller's lease is handed away, and no ask signed by its key is taken"
}
}
if len(signer) != ed25519.PublicKeySize ||
!ed25519.Verify(signer, append([]byte(context), signed.Ask...), signed.Signature) {
return HandOverAsk{}, nil, "the ask is not signed by the mesh"
}
var ask HandOverAsk
internal/link/handover_test.go
+117 -2
View File
@@ -9,6 +9,9 @@ import (
"strings"
"testing"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"github.com/nats-io/nats.go"
)
// The hand-over's ask and answer hold these field names; the controller's side holds the same list
@@ -145,6 +148,21 @@ func TestAHandOverAskIsBelievedOnlySignedForThisNodeUnexpiredAndOnce(t *testing.
if strings.Join(given, "; ") != "/srv/notes by jo through mesh-cli on anchor; /refused by jo through mesh-cli on anchor" {
t.Fatalf("the handler was given %v", given)
}
// ADR 0563 §7, ADR 0550 §2: after the lease write the engine answers no ask signed by the controller's key. The key
// is read per ask from what the engine trusts now; nil (the lease handed away) refuses, and nothing is handed over.
trusted := m.Signer
following := HandOverCheck{Signer: m.Signer, SignerNow: func() ed25519.PublicKey { return trusted }, Node: m.Node,
Now: func() time.Time { return now }, Nonces: &Nonces{}}
before := len(given)
trusted = nil
late := signedAsk(t, key, with(func(a *HandOverAsk) { a.Expires = now.Add(time.Minute); a.Nonce = strings.Repeat("cd", 16) }))
if a := AnswerHandOver(following, handle, late); !strings.Contains(a.Refused, "lease is handed away") || len(given) != before {
t.Fatalf("after the lease write an ask signed by the controller was answered %+v", a)
}
trusted = m.Signer
if a := AnswerHandOver(following, handle, late); a.Refused != "" || len(given) != before+1 {
t.Fatalf("while the controller holds the lease its ask was answered %+v", a)
}
// A nonce is forgotten once its ask has expired, so the set does not grow for ever.
now = now.Add(HandOverGood + time.Second)
check.Nonces.take("ff", now.Add(time.Second), now)
@@ -174,17 +192,23 @@ func TestAHandOverIsAnsweredOnlyWhenItCarriesAReply(t *testing.T) {
// handingLink is a quiet link that hears hand-overs, keeping the handler it was given.
type handingLink struct {
*quietLink
given chan HandOverFunc
given chan HandOverFunc
signer chan func() ed25519.PublicKey
}
func (l handingLink) HearHandOvers(h HandOverFunc) { l.given <- h }
// FollowSigner keeps what the link is told to verify the controller's asks with.
func (l handingLink) FollowSigner(f func() ed25519.PublicKey) { l.signer <- f }
// **The link is given the queue's hand-over** when it is served: what the controller asks reaches the engine's
// judgement, and nothing else.
func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
m, _ := aMember(t)
l := handingLink{quietLink: newQuietLink(), given: make(chan HandOverFunc, 1)}
l := handingLink{quietLink: newQuietLink(), given: make(chan HandOverFunc, 1), signer: make(chan func() ed25519.PublicKey, 1)}
q := &Session{Membership: m, Timeout: time.Second}
// The link verifies the controller's asks with the key the engine trusts now (ADR 0563 §7).
q.ControllerKey = func() ed25519.PublicKey { return nil }
q.HandOver = func(path, by string) (string, error) { return "the queue's: " + path + " by " + by, nil }
ctx, stop := context.WithCancel(context.Background())
served := make(chan error, 1)
@@ -197,8 +221,40 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
case <-time.After(5 * time.Second):
t.Fatal("the link was never given the queue's hand-over")
}
select {
case f := <-l.signer:
if f() != nil {
t.Fatal("the link was given another key than the queue's")
}
case <-time.After(5 * time.Second):
t.Fatal("the link was never given the key the engine trusts now")
}
stop()
<-served
// 774 (ADR 0563 §1): a call of the node seat reaches its verb with its caller as the subject proves it, unproven on
// the subject that names none; a verb the seat does not serve is refused, and the answer is a tool's envelope.
t.Run("a call of the node seat reaches its verb with its proven caller", func(t *testing.T) {
h := &HandActs{Node: "n1", State: &MemState{},
RootFree: func() RootVerdict { return RootVerdict{Machine: "n1", Free: true, Why: "judged"} },
JudgeDirectories: func(by string) (string, error) { return "judged for " + by, nil }}
for _, row := range []struct{ verb, subject, want string }{
{"judge-directories", "mesh.seat.node-engine.call.judge-directories.n1.shanks~node-tools", `"said":"judged for shanks.node-tools"`},
{"judge-directories", EngineToolSubject("judge-directories", "n1"), `"said":"judged for unproven"`},
{"judge-directories", "mesh.seat.node-engine.call.judge-directories.n2.shanks~node-tools", `"said":"judged for unproven"`},
{"root-free", "mesh.seat.node-engine.call.root-free.n1.ace~messenger", `"free":true`},
{"unserved", "mesh.seat.node-engine.call.unserved.n1.x", `"error":"the node seat node-engine serves no verb \"unserved\""`},
} {
if got := string(AnswerEngineCall(h.Call, row.verb, "n1", row.subject, []byte(`{}`))); !strings.Contains(got, row.want) ||
!strings.Contains(got, `"node":"n1"`) {
t.Errorf("%s on %s: %s", row.verb, row.subject, got)
}
}
acts, _ := h.Acts(0)
if got := acts.(map[string]any)["acts"].([]HandAct); len(got) != 3 || got[0].Caller != "unproven" ||
got[2].Caller != "shanks.node-tools" || got[2].Act != ActJudgeDirectories {
t.Errorf("the judgements are recorded as %+v", got)
}
})
}
// **Against a real server**: a signed request on the node's own subject is answered; one on another node's is
@@ -237,4 +293,63 @@ func TestNatsAHandOverIsAskedOfTheNode(t *testing.T) {
if len(handled) != 1 {
t.Fatalf("handled %d asks; want only the one with a reply", len(handled))
}
// 774 (ADR 0563 §2, §3): the engine's acts are its node's keys of its declared state on the bus, each written by
// compare-and-set: a second create and a stale revision lose, and another node's keys are not this node's.
t.Run("the acts are kept in the bus's own storage under this node's keys", func(t *testing.T) {
_ = js.DeleteKeyValue(ActsBucket)
if _, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: ActsBucket}); err != nil {
t.Fatal(err)
}
q := &Session{Membership: m, Timeout: time.Second}
q.bus = l
mine, theirs := BusState{Session: q, Node: m.Node}, BusState{Session: q, Node: "another"}
rev, err := mine.Create("act.a-1", map[string]string{"state": "asked"})
if err != nil {
t.Fatal(err)
}
if _, err := mine.Create("act.a-1", map[string]string{}); !errors.Is(err, stdio.ErrChanged) {
t.Errorf("a second create: %v", err)
}
if _, err := mine.Update("act.a-1", map[string]string{"state": "done"}, rev+7); !errors.Is(err, stdio.ErrChanged) {
t.Errorf("an update at a stale revision: %v", err)
}
if _, err := mine.Update("act.a-1", map[string]string{"state": "done"}, rev); err != nil {
t.Errorf("an update at its revision: %v", err)
}
_, _ = theirs.Create("act.b-1", map[string]string{})
if keys, err := mine.Keys("act."); err != nil || len(keys) != 1 || keys[0] != "act.a-1" {
t.Errorf("this node's acts are %v, %v", keys, err)
}
if e, _ := js.KeyValue(ActsBucket); e != nil {
if got, err := e.Get(m.Node + ".act.a-1"); err != nil || !strings.Contains(string(got.Value()), "done") {
t.Errorf("the bucket holds %v, %v", got, err)
}
}
if err := mine.Delete("act.a-1"); err != nil {
t.Fatal(err)
}
if e, err := mine.Get("act.a-1"); err != nil || e != nil {
t.Errorf("a deleted act reads %v, %v", e, err)
}
})
// 774 (ADR 0563 §1): the node seat is served on this node's subjects alone, naming its caller, and said to discovery.
t.Run("the node seat is served on this node and discovered", func(t *testing.T) {
l.ServeEngineSeat(func(c EngineCall) (any, error) { return c.Verb + " for " + c.Caller, nil })
reply, err := conn.Request("mesh.seat.node-engine.call.acts."+m.Node+".shanks~node-tools", []byte(`{}`), 5*time.Second)
if err != nil || !strings.Contains(string(reply.Data), `"result":"acts for shanks.node-tools"`) {
t.Fatalf("the call answered %v, %v", reply, err)
}
if _, err := conn.Request(EngineToolSubject("acts", "another"), []byte(`{}`), 300*time.Millisecond); err == nil {
t.Fatal("another node's verb was answered here")
}
for seat, want := range map[string]bool{EngineSeat: true, "node-login-shell": false} {
verb := "acts"
if !want {
verb = "execute"
}
if heard, err := l.HeardServed(seat, verb); err != nil || heard != want {
t.Errorf("discovery hears %s.%s here: %v, %v", seat, verb, heard, err)
}
}
})
}
internal/link/have_test.go
+3 -2
View File
@@ -274,7 +274,8 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
// 1126 (ADR 0555 §3): with its keys, the node key carries the node's record: the profile's present capabilities and
// networks, the outward links, the health's contract and the public keys; a report that changes none of it writes
// nothing, a changed fact writes the key once, and a report without a profile or health leaves them standing.
keys := RecordKeys{TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert"}
keys := RecordKeys{TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert",
Trusts: []string{"controller key sha256:aa", "vault key sha256:bb"}} // and what it trusts (L3 s6, ADR 0550 §2)
q.Keys = func() RecordKeys { return keys }
rep.Profile = map[string]any{"capabilities": []any{map[string]any{"name": "systemd", "present": true},
map[string]any{"name": "docker", "present": false}, map[string]any{"name": "apt", "present": true}},
@@ -286,7 +287,7 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
t.Fatalf("the node key holds no record: %+v, %v", got, err)
}
wantRecord := delivery.NodeRecord{V: delivery.Version, Node: "n1", TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk",
Certificate: "cert", Capabilities: []string{"apt", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true,
Certificate: "cert", Trusts: keys.Trusts, Capabilities: []string{"apt", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true,
Networks: []catalogue.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, Contract: NodeKeyContract}
if !delivery.SameNodeRecord(*got.Record, wantRecord) {
t.Errorf("record = %+v, want %+v", *got.Record, wantRecord)
internal/link/health_test.go
+98
View File
@@ -3,9 +3,12 @@ package link
import (
"context"
"encoding/json"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// A health statement is said on the link open now, as the machine's own word on its own subject, and
@@ -50,6 +53,101 @@ func TestAHealthStatementIsSaidOnTheLinkOpenNow(t *testing.T) {
if HealthSubject("anchor") != "mesh.control.anchor.health" {
t.Fatalf("the subject %s is not inside the host's own grant", HealthSubject("anchor"))
}
// Issue 774: the node's own health report in its key of mesh-host_health, for the watchman's six alerts: written on
// a change only, the streaks and the time of the look not being one, and read first after a restart.
t.Run("the node's health report is written on a change only", func(t *testing.T) {
since := time.Date(2026, 10, 29, 20, 0, 0, 0, time.UTC)
look := func(streak int, writer string) Health {
return Health{Contract: 7, At: time.Now(), Network: &NetworkHealth{State: StateUnhealthy, Since: since, Parts: []NetworkPart{
{Part: "resolv-conf", State: StateUnhealthy, Writer: writer, Owner: "dnsmasq", Since: since, Streak: streak}}},
Units: &UnitsHealth{State: "degraded", Failed: []FailedUnit{{Unit: "cups.service", Scope: "system", Since: since}}},
Resources: []ResourceHealth{
{Module: "brother", Resource: "brother.scanner", Kind: KindAccount, Account: "jochen", State: StateUnhealthy,
Reason: "relogin needed: the session began before scanner", Since: since},
{Module: "photos", Resource: "photos.library", Kind: KindDirectory, Target: "/srv/photos", State: StateUnhealthy,
Reason: ReasonUsedAsFound + " owned by root", Since: since},
{Module: "letta", Resource: "letta.server", Kind: "container", State: StateHealthy, Since: since},
{Module: "claude-code", Resource: "claude-code.agent", Kind: KindAccount, Account: "agent", Root: "never",
State: StateHealthy, Since: since}}}
}
peers := []PeerSeen{{Key: "B=", Seen: true}, {Key: "A=", Addresses: []string{"10.10.0.2/32"}}}
r := NodeHealthOf("anchor", look(1, "NetworkManager"), peers, Machines{})
if r.Network.Parts[0].Writer != "NetworkManager" || r.Network.Parts[0].Streak != 0 || r.Peers[0].Key != "A=" ||
len(r.Units.Failed) != 1 || len(r.Relogin) != 1 || r.Relogin[0].Account != "jochen" ||
len(r.Found) != 1 || r.Found[0].Path != "/srv/photos" ||
len(r.Root) != 1 || r.Root[0].Account != "agent" || r.Root[0].State != StateHealthy || r.Contract != 7 {
t.Fatalf("the report is %+v", r)
}
// The machines an unhealthy part points at, as the controller holds it there; none for the machine's own.
machines := Machines{ByAddress: map[string]string{"10.10.0.1": "novox", "10.10.0.4": "ace", "10.10.0.9": "anchor"}, Hub: "novox"}
for _, tc := range []struct {
name string
toward []string
state string
want []string
}{
{"the tunnel to the hub", []string{"hub"}, StateUnhealthy, []string{"novox"}},
{"two mesh resolvers", []string{"10.10.0.1", "10.10.0.4"}, StateUnhealthy, []string{"novox", "ace"}},
{"a resolver a VPN client wrote in is the machine's own", []string{"10.10.0.1", "192.0.2.53"}, StateUnhealthy, nil},
{"its own resolver is its own", []string{"10.10.0.9"}, StateUnhealthy, nil},
{"a healthy part points at nothing", []string{"hub"}, StateHealthy, nil},
} {
h := Health{Network: &NetworkHealth{State: tc.state, Parts: []NetworkPart{{Part: "names", State: tc.state, Toward: tc.toward}}}}
got := NodeHealthOf("anchor", h, nil, machines).Network.Parts[0].Targets
if !slices.Equal(got, tc.want) {
t.Errorf("%s: targets %v, want %v", tc.name, got, tc.want)
}
}
store := &memHealth{}
var last *NodeHealth
clock := func() time.Time { return since }
for i, step := range []struct {
h NodeHealth
fresh bool // a restart: nothing written in this run yet
wrote bool
}{
{h: r, wrote: true},
{h: NodeHealthOf("anchor", look(2, "NetworkManager"), peers, Machines{})},
{h: NodeHealthOf("anchor", look(3, "NetworkManager"), peers, Machines{}), fresh: true},
{h: NodeHealthOf("anchor", look(3, "openvpn"), peers, Machines{}), wrote: true},
{h: NodeHealthOf("anchor", look(3, "openvpn"), []PeerSeen{{Key: "A=", Seen: true}}, Machines{}), wrote: true},
} {
if step.fresh {
last = nil
}
wrote, err := writeNodeHealth(t.Context(), store, &last, step.h, clock)
if err != nil || wrote != step.wrote {
t.Fatalf("step %d: wrote %v (want %v), %v", i, wrote, step.wrote, err)
}
}
got, err := delivery.DecodeNodeHealth(store.values["anchor"])
if err != nil || got.Node != "anchor" || got.Network.Parts[0].Writer != "openvpn" || !got.Peers[0].Seen || got.At.IsZero() {
t.Fatalf("the key holds %+v, %v", got, err)
}
// The key's bytes are mesh-sdk's layout: decoded and encoded again by it, the same bytes.
if again, err := delivery.EncodeNodeHealth(got); err != nil || string(again) != string(store.values["anchor"]) {
t.Fatalf("the key's bytes are not the sdk's layout: %s\n%s (%v)", store.values["anchor"], again, err)
}
if store.puts != 3 {
t.Fatalf("%d writes, want 3: one per change", store.puts)
}
})
}
// memHealth is a HealthStore in memory.
type memHealth struct {
values map[string][]byte
puts int
}
func (m *memHealth) Get(_ context.Context, key string) ([]byte, error) { return m.values[key], nil }
func (m *memHealth) Put(_ context.Context, key string, value []byte) error {
if m.values == nil {
m.values = map[string][]byte{}
}
m.values[key], m.puts = value, m.puts+1
return nil
}
// A declared tool check reads the node tools' reply envelope: healthy only when the tool says so (novox/hq
internal/link/hearing_nats.go
+48 -17
View File
@@ -6,9 +6,11 @@ import (
"errors"
"fmt"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/micro"
)
// The host's link on the bus: one connection, and what is asked of this node over it. The node hears no declaration
@@ -35,11 +37,14 @@ func natsURL(address string) string {
// natsLink is this node's connection as a JetStream subscription.
type natsLink struct {
conn *nats.Conn
js nats.JetStreamContext
asking *nats.Subscription
asked chan struct{}
handing *nats.Subscription
// mu guards signerNow (FollowSigner).
mu sync.Mutex
signerNow func() ed25519.PublicKey
conn *nats.Conn
js nats.JetStreamContext
asking *nats.Subscription
asked chan struct{}
handing *nats.Subscription
// searching hears the controller's terminal asking for a fresh setuid search (novox/hq issue 361).
searching *nats.Subscription
// judging hears the controller asking for a read-only judgement of this node's directories (issue 358).
@@ -48,7 +53,10 @@ type natsLink struct {
acting *nats.Subscription
// moving hears the controller asking to move a value sealed to this machine into the vault (issue 676).
moving *nats.Subscription
node string
// engine serves the node seat node-engine, and warrants hears the operator's answers to its asks (ADR 0563).
engine micro.Service
warrants *nats.Subscription
node string
// signer is the mesh's key, which a hand-over ask is verified with as a declaration is (issue 356).
signer ed25519.PublicKey
lost chan error
@@ -141,6 +149,21 @@ func (l *natsLink) hearAsks() {
}
}
// FollowSigner has every signed ask heard on this link verified with the key the engine trusts at that ask, the
// controller's while it holds its lease and none from its hand-over (novox/hq ADR 0563 §7, ADR 0550 §2).
func (l *natsLink) FollowSigner(f func() ed25519.PublicKey) {
l.mu.Lock()
l.signerNow = f
l.mu.Unlock()
}
// check is how a signed ask is believed on this link: the static membership signer until FollowSigner names the trust.
func (l *natsLink) check() HandOverCheck {
l.mu.Lock()
defer l.mu.Unlock()
return HandOverCheck{Signer: l.signer, SignerNow: l.signerNow, Node: l.node}
}
// HearHandOvers listens for the controller asking this node to hand a directory used as found to the mesh
// (novox/hq issue 356): a request on this machine's own subject, answered once on the reply it carried. The
// ask is verified with the mesh's key before anything is read out of it (AnswerHandOver), because a responder
@@ -151,7 +174,7 @@ func (l *natsLink) HearHandOvers(handle HandOverFunc) {
if handle == nil {
return
}
check := HandOverCheck{Signer: l.signer, Node: l.node}
check := l.check()
handing, err := l.conn.Subscribe(AskHandOverSubject(l.node), func(msg *nats.Msg) {
if body, answer := HandOverReply(check, handle, msg.Reply, msg.Data); answer {
_ = msg.Respond(body)
@@ -170,7 +193,7 @@ func (l *natsLink) HearSetuidSearches(handle SetuidSearchFunc) {
if handle == nil {
return
}
check := HandOverCheck{Signer: l.signer, Node: l.node}
check := l.check()
searching, err := l.conn.Subscribe(AskSetuidSearchSubject(l.node), func(msg *nats.Msg) {
if body, answer := SetuidSearchReply(check, handle, msg.Reply, msg.Data); answer {
_ = msg.Respond(body)
@@ -189,7 +212,7 @@ func (l *natsLink) HearDirectoryJudgements(handle JudgeDirectoriesFunc) {
if handle == nil {
return
}
check := HandOverCheck{Signer: l.signer, Node: l.node}
check := l.check()
judging, err := l.conn.Subscribe(AskJudgeDirectoriesSubject(l.node), func(msg *nats.Msg) {
if body, answer := JudgeDirectoriesReply(check, handle, msg.Reply, msg.Data); answer {
_ = msg.Respond(body)
@@ -207,7 +230,7 @@ func (l *natsLink) HearAccessActs(handle AccessActFunc) {
if handle == nil {
return
}
check := HandOverCheck{Signer: l.signer, Node: l.node}
check := l.check()
acting, err := l.conn.Subscribe(AskAccessActSubject(l.node), func(msg *nats.Msg) {
if body, answer := AccessActReply(check, handle, msg.Reply, msg.Data); answer {
_ = msg.Respond(body)
@@ -225,7 +248,7 @@ func (l *natsLink) HearMoveSecrets(handle MoveSecretFunc) {
if handle == nil {
return
}
check := HandOverCheck{Signer: l.signer, Node: l.node}
check := l.check()
moving, err := l.conn.Subscribe(AskMoveSecretSubject(l.node), func(msg *nats.Msg) {
if body, answer := MoveSecretReply(check, handle, msg.Reply, msg.Data); answer {
_ = msg.Respond(body)
@@ -258,6 +281,12 @@ func (l *natsLink) Close() {
if l.judging != nil {
_ = l.judging.Unsubscribe()
}
if l.engine != nil {
_ = l.engine.Stop()
}
if l.warrants != nil {
_ = l.warrants.Unsubscribe()
}
if l.conn != nil {
l.conn.Close()
}
@@ -301,10 +330,12 @@ func (l *natsLink) Ask(ctx context.Context, subject string, body []byte) ([]byte
// What the queue asks of a live link beside Bus, each by a type assertion that fails quietly: said here
// so a link that drops one does not build.
var (
_ Bus = (*natsLink)(nil)
_ HealthBus = (*natsLink)(nil)
_ HaveBus = (*natsLink)(nil)
_ ToolBus = (*natsLink)(nil)
_ Asker = (*natsLink)(nil)
_ Asked = (*natsLink)(nil)
_ Bus = (*natsLink)(nil)
_ HealthBus = (*natsLink)(nil)
_ HaveBus = (*natsLink)(nil)
_ ToolBus = (*natsLink)(nil)
_ Asker = (*natsLink)(nil)
_ Asked = (*natsLink)(nil)
_ EngineSeats = (*natsLink)(nil)
_ OperatorAsks = (*natsLink)(nil)
)
internal/link/messages.go
+8 -52
View File
@@ -313,58 +313,14 @@ const KindRunner = "runner"
// ReasonUsedAsFound starts the reason of a directory used as found.
const ReasonUsedAsFound = "used as found:"
// UnitsHealth is the machine's service managers in one statement (issue 315).
type UnitsHealth struct {
// State is running, degraded — a unit failed, the modules' or the machine's — or unknown when no
// manager could be read.
State string `json:"state"`
// Failed is every unit failed on two looks in a row that no module places: the machine's own.
Failed []FailedUnit `json:"failed"`
// Unread names a manager that could not be read, with why.
Unread []string `json:"unread,omitempty"`
// Away names an account's own manager not running, so not read: nothing in it is failed (issue 764).
Away []string `json:"away,omitempty"`
}
// FailedUnit is one failed unit no module places.
type FailedUnit struct {
Unit string `json:"unit"`
// Scope is system, or user: an account's own manager.
Scope string `json:"scope"`
// Load is loaded, not-found — a unit whose file is gone and whose failure its manager still holds…
Load string `json:"load,omitempty"`
// Result is how it failed: exit-code, timeout, start-limit-hit…
Result string `json:"result,omitempty"`
// Resource is the mesh's own resource that names it, when the mesh placed it in its own right.
Resource string `json:"resource,omitempty"`
Since time.Time `json:"since"`
}
// NetworkHealth is the machine's networking in one statement (ADR 0241): the worst of its parts, since
// when, and each part.
type NetworkHealth struct {
State string `json:"state"`
Since time.Time `json:"since"`
Parts []NetworkPart `json:"parts"`
}
// NetworkPart is one part of a machine's networking, as its engine judged it on its second look.
type NetworkPart struct {
// Part is resolv-conf, names, tunnel, bus or route.
Part string `json:"part"`
State string `json:"state"`
// Reason is why it is not healthy, in words with no address, path or domain; Said the detail.
Reason string `json:"reason,omitempty"`
Said string `json:"said,omitempty"`
// Writer is the program that rewrote the resolver file, when it can be named; Owner the module whose
// file it is — the uplink's holder.
Writer string `json:"writer,omitempty"`
Owner string `json:"owner,omitempty"`
// Toward is what the failure points at: "hub", or each resolver's address that failed.
Toward []string `json:"toward,omitempty"`
Since time.Time `json:"since"`
Streak int `json:"streak,omitempty"`
}
// The machine's networking and its service managers in a statement are the node's health report's parts (mesh-sdk
// delivery, issue 774): one layout for the message, the report and the watchman that reads it.
type (
NetworkHealth = delivery.NetworkHealth
NetworkPart = delivery.NetworkPart
UnitsHealth = delivery.UnitsHealth
FailedUnit = delivery.FailedUnit
)
// The states a long-running resource is said in (ADR 0240 §4).
const (
internal/link/nodehealth.go
+208
View File
@@ -0,0 +1,208 @@
package link
import (
"context"
"errors"
"fmt"
"slices"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/declaration"
"git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/nats-io/nats.go"
)
// The node-engine's own health report in the bus's own storage (issue 774, the operator's alert table, 2026-10-29): one
// key per node, `<node>`, of the engine's declared state `health` (HealthBucket, created by the grants part from the
// engine's manifest as its state `acts` is), written by this node's engine alone and only when what it says changed.
// The watchman (seat mesh-watchdogs) reads it for six alerts the controller raised from the passing health message:
//
// - machine-network: Network, the machine's route, tunnel, name servers, resolver file and bus, each a part;
// - network-rewritten: the resolver file's part, unhealthy with Writer, the program that rewrote it, and Owner, the
// module whose file it is;
// - network-unreachable: Peers, each peer of the mesh's tunnel and whether this machine saw it within five minutes,
// read across every node's report;
// - machine-units: Units, the failed system units no module owns;
// - relogin-needed: Relogin, each account whose running session lacks a group it was given;
// - directory-used-as-found: Found, each directory a module declares that the engine uses as found.
//
// The report's layout is mesh-sdk's (delivery.NodeHealth): the bucket, its version and its parts.
const (
HealthBucket = delivery.NodeHealthBucket
NodeHealthVersion = delivery.NodeHealthVersion
)
type (
NodeHealth = delivery.NodeHealth
PeerSeen = delivery.PeerSeen
NodeAccount = delivery.NodeAccount
NodeDirectory = delivery.NodeDirectory
)
// ReasonReloginNeeded starts an account's reason when its session needs a new login (accounts.ReasonRelogin).
const ReasonReloginNeeded = "relogin needed"
// Machines names the mesh's machines for a report's targets: each node by its address on the private network, and the
// hub's node. The zero value names none, and every part is then the machine's own.
type Machines struct {
ByAddress map[string]string
Hub string
}
// targets is the machines a part points at, as the controller holds it (mesh-controller machine_network.go): the hub's
// node for TowardHub, each resolver's node by its address; none when it points at nothing, or at this machine or an
// address no other mesh machine has, as one of them makes the part the machine's own.
func (m Machines) targets(node string, toward []string) []string {
if len(toward) == 0 {
return nil
}
var out []string
for _, t := range toward {
x := m.ByAddress[t]
if t == TowardHub {
x = m.Hub
}
if x == "" || x == node {
return nil
}
if !slices.Contains(out, x) {
out = append(out, x)
}
}
return out
}
// TowardHub is what a part failing toward the hub names in Toward (network.TowardHub).
const TowardHub = "hub"
// NodeHealthOf is the report a health statement and the tunnel's peers make: the statement's network without its
// streaks, each unhealthy part naming the machines it points at (machines), its machine units, and each account and
// directory of its resources that the six alerts read.
func NodeHealthOf(node string, h Health, peers []PeerSeen, machines Machines) NodeHealth {
out := NodeHealth{V: NodeHealthVersion, Node: node, Units: h.Units, Contract: h.Contract}
if len(peers) > 0 {
out.Peers = append([]PeerSeen(nil), peers...)
sort.Slice(out.Peers, func(i, k int) bool { return out.Peers[i].Key < out.Peers[k].Key })
}
if h.Network != nil {
n := NetworkHealth{State: h.Network.State, Since: h.Network.Since, Parts: make([]NetworkPart, 0, len(h.Network.Parts))}
for _, p := range h.Network.Parts {
p.Streak = 0 // a look's count moves every look; the report says a part's state, not how often it was seen
p.Targets = nil
if p.State == StateUnhealthy {
p.Targets = machines.targets(node, p.Toward)
}
n.Parts = append(n.Parts, p)
}
out.Network = &n
}
for _, r := range h.Resources {
// Each alert reads its own: an account may need a new login and be declared never to become root at once.
if r.Kind == KindAccount && r.State == StateUnhealthy && strings.HasPrefix(r.Reason, ReasonReloginNeeded) {
out.Relogin = append(out.Relogin, NodeAccount{Account: r.Account, Module: r.Module, Resource: r.Resource,
Reason: r.Reason, Since: r.Since})
}
if r.Kind == KindAccount && r.Root == declaration.RootNever {
// The account declared never to become root without a person, judged (ADR 0266): what DA and D-root read.
out.Root = append(out.Root, delivery.RootVerdict{Account: r.Account, Module: r.Module, Resource: r.Resource,
State: r.State, Reason: r.Reason, Since: r.Since})
}
if r.Kind == KindDirectory && strings.HasPrefix(r.Reason, ReasonUsedAsFound) {
out.Found = append(out.Found, NodeDirectory{Module: r.Module, Resource: r.Resource, Path: r.Target,
Reason: r.Reason, Since: r.Since})
}
}
sort.Slice(out.Relogin, func(i, k int) bool {
return out.Relogin[i].Account+out.Relogin[i].Module < out.Relogin[k].Account+out.Relogin[k].Module
})
sort.Slice(out.Found, func(i, k int) bool { return out.Found[i].Path < out.Found[k].Path })
sort.Slice(out.Root, func(i, k int) bool {
return out.Root[i].Module+out.Root[i].Resource < out.Root[k].Module+out.Root[k].Resource
})
return out
}
// HealthStore is where a node's report is kept: its own key of HealthBucket.
type HealthStore interface {
// Get is the key's value, nil when it has none.
Get(ctx context.Context, key string) ([]byte, error)
Put(ctx context.Context, key string, value []byte) error
}
// healthStore binds HealthBucket over the link open now: bound, never created.
func (l *natsLink) healthStore() (HealthStore, error) {
kv, err := l.js.KeyValue(HealthBucket)
if err != nil {
return nil, fmt.Errorf("the node-engine's state %s cannot be bound (it is created from the node-engine's "+
"manifest; written again once it is): %w", HealthBucket, err)
}
return natsHealth{kv}, nil
}
type natsHealth struct{ kv nats.KeyValue }
func (n natsHealth) Get(_ context.Context, key string) ([]byte, error) {
e, err := n.kv.Get(key)
if errors.Is(err, nats.ErrKeyNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
return e.Value(), nil
}
func (n natsHealth) Put(_ context.Context, key string, value []byte) error {
_, err := n.kv.Put(key, value)
return err
}
// WriteNodeHealth writes this node's report to its key when it says something other than the key holds, and says
// whether it wrote. The key is read before the first write of a run, so a restart that finds the same report writes
// nothing; a store that cannot be bound or written is an error, and the next look tries again.
func (q *Session) WriteNodeHealth(ctx context.Context, h NodeHealth) (bool, error) {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
l, ok := bus.(interface{ healthStore() (HealthStore, error) })
if bus == nil || !ok {
return false, ErrNoLink
}
store, err := l.healthStore()
if err != nil {
return false, err
}
return writeNodeHealth(ctx, store, &q.healthWritten, h, time.Now)
}
// writeNodeHealth is WriteNodeHealth over a store: last is the report last written or read, nil before the first.
func writeNodeHealth(ctx context.Context, store HealthStore, last **NodeHealth, h NodeHealth, now func() time.Time) (bool, error) {
if *last == nil {
raw, err := store.Get(ctx, h.Node)
if err != nil {
return false, fmt.Errorf("the health report of %s cannot be read: %w", h.Node, err)
}
if raw != nil {
if was, err := delivery.DecodeNodeHealth(raw); err == nil {
*last = &was
}
}
}
if *last != nil && delivery.SameNodeHealth(**last, h) {
return false, nil
}
h.At = now().UTC()
raw, err := delivery.EncodeNodeHealth(h)
if err != nil {
return false, err
}
if err := store.Put(ctx, h.Node, raw); err != nil {
return false, fmt.Errorf("the health report of %s cannot be written: %w", h.Node, err)
}
*last = &h
return true, nil
}
internal/link/record.go
+3 -1
View File
@@ -23,6 +23,8 @@ type RecordKeys struct {
Certificate string
// Authority is the mesh's own certificate as this machine last applied it (public).
Authority string
// Trusts is what this machine trusts a mesh signature from now (delivery.Trusts, ADR 0550 §2).
Trusts []string
}
// recordOf is the node's record from one facts report and its keys; nil for a report that carries no profile (one that
@@ -45,7 +47,7 @@ func recordOf(node string, r Report, keys RecordKeys) *delivery.NodeRecord {
return nil
}
rec := &delivery.NodeRecord{V: delivery.Version, Node: node, TunnelKey: keys.TunnelKey, SealingKey: keys.SealingKey,
ServingKey: keys.ServingKey, Certificate: keys.Certificate, Authority: keys.Authority}
ServingKey: keys.ServingKey, Certificate: keys.Certificate, Authority: keys.Authority, Trusts: keys.Trusts}
for _, c := range p.Capabilities {
if c.Present && c.Name != "" {
rec.Capabilities = append(rec.Capabilities, c.Name)
internal/link/run.go
+21
View File
@@ -7,6 +7,8 @@ import (
"errors"
"fmt"
"time"
meshasks "git.novox.be/novox/mesh-sdk/go/asks"
)
// Membership is what a node needs to reach its mesh again, held by the caller.
@@ -185,6 +187,12 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann
}
// And the controller asking this node to hand a directory used as found to the mesh (novox/hq issue
// 356), answered on the link the question came over.
// Every signed ask is verified with the key trusted at that ask, none after the lease write (ADR 0563 §7).
if f, ok := link.(interface {
FollowSigner(func() ed25519.PublicKey)
}); ok && queue.ControllerKey != nil {
f.FollowSigner(queue.ControllerKey)
}
if h, ok := link.(HandOvers); ok {
h.HearHandOvers(queue.HandOver)
}
@@ -200,6 +208,19 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann
if s, ok := link.(AccessActs); ok {
s.HearAccessActs(queue.AccessAct)
}
// And its node seat, node-engine, with the operator's answers to its own asks (novox/hq ADR 0563): served on this
// link, and what was answered while no link was open caught up from the router's records.
if queue.Acts != nil {
if s, ok := link.(EngineSeats); ok {
s.ServeEngineSeat(queue.Acts.Call)
}
if a, ok := link.(OperatorAsks); ok {
if err := a.HearWarrants(meshasks.EngineAsker(m.Node), queue.Acts.Heard); err != nil {
say("cannot hear the operator's answers to this engine's asks: " + err.Error())
}
go queue.Acts.Reconcile()
}
}
// And asking to move a value sealed to this machine into the vault (design 57's V3, issue 676).
if s, ok := link.(MoveSecrets); ok {
s.HearMoveSecrets(queue.MoveSecret)
internal/link/session.go
+57
View File
@@ -2,12 +2,14 @@ package link
import (
"context"
"crypto/ed25519"
"encoding/json"
"errors"
"fmt"
"sync"
"time"
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/health"
)
@@ -47,6 +49,12 @@ type Session struct {
// MoveSecret moves one value sealed to this machine into the vault, asked by the controller during the move of
// the secrets (design 57's V3, issue 676). Nil answers every such ask with a refusal.
MoveSecret MoveSecretFunc
// ControllerKey is the controller's key as the engine trusts it now: nil once its lease is handed away (novox/hq
// ADR 0563 §7). Each link verifies every signed ask with it, read per ask; nil ControllerKey keeps the membership's.
ControllerKey func() ed25519.PublicKey
// Acts is the node seat node-engine this engine holds (novox/hq ADR 0563): served on every link, and its asks to the
// operator asked and heard over it. Nil serves no seat.
Acts *HandActs
// Keys reads the node's public keys and its certificate for its record (ADR 0555 §3), at each facts report; nil writes
// no record.
Keys func() RecordKeys
@@ -78,6 +86,9 @@ type Session struct {
// haves is what this node-engine writes beside its facts (novox/hq ADR 0429 §4, issue 1132).
haves haveState
// healthWritten is this node's health report as last written or read (WriteNodeHealth): the health loop alone
// writes it, one look at a time.
healthWritten *NodeHealth
}
func (q *Session) init() {
@@ -337,3 +348,49 @@ func (q *Session) AskSeat(ctx context.Context, seat, verb string, args any, with
}
return env.Result, nil
}
// PublishAsk publishes an ask or a cancel to the operator's router over the link open now (novox/hq ADR 0563 §3).
func (q *Session) PublishAsk(subject string, body any, id string) error {
asking, err := q.operatorAsks()
if err != nil {
return err
}
return asking.PublishAsk(subject, body, id)
}
// RouterRecord reads one key of the router's records over the link open now.
func (q *Session) RouterRecord(key string) (*stdio.StateEntry, error) {
asking, err := q.operatorAsks()
if err != nil {
return nil, err
}
return asking.RouterRecord(key)
}
// SayEngine says one of the node seat's events over the link open now; with none open it is not said.
func (q *Session) SayEngine(event string, body any) {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
if sayer, ok := bus.(interface{ SayEngine(string, any) error }); ok {
if err := sayer.SayEngine(event, body); err != nil {
q.Say("could not say the node seat's " + event + ": " + err.Error())
}
}
}
func (q *Session) operatorAsks() (OperatorAsks, error) {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
if bus == nil {
return nil, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked)
}
asking, ok := bus.(OperatorAsks)
if !ok {
return nil, errNoAsking
}
return asking, nil
}
internal/link/setuid_search_test.go
+78
View File
@@ -57,4 +57,82 @@ func TestASetuidSearchAskIsBelievedOnlySignedForThisNodeAndOnce(t *testing.T) {
if a := AnswerSetuidSearch(check, nil, body); a.Refused == "" {
t.Fatalf("no handler: %+v", a)
}
// 774 (ADR 0563 §2): the node seat's setuid-search starts none while one runs, at most one in any hour, naming when the
// next may start, and records who asked.
t.Run("the node seat's search is bounded", func(t *testing.T) {
at, running, started := now, false, 0
h := &HandActs{Node: "n1", State: &MemState{}, Now: func() time.Time { return at },
Searching: func() bool { return running },
SetuidSearch: func(by string) (string, error) { started++; return "a new search starts", nil }}
for _, row := range []struct {
name string
after time.Duration
running bool
refused string
started int
}{
{name: "one runs", running: true, refused: "runs on this machine now", started: 0},
{name: "the first", started: 1},
{name: "within the hour", after: 59 * time.Minute, refused: "the next may start at 2026-10-10T03:00:00Z", started: 1},
{name: "after the hour", after: time.Hour + time.Second, started: 2},
} {
at, running = now.Add(row.after), row.running
_, err := h.Call(EngineCall{Verb: ActSetuidSearch, Caller: "ace.node-tools"})
if (row.refused == "") != (err == nil) || (err != nil && !strings.Contains(err.Error(), row.refused)) || started != row.started {
t.Errorf("%s: %v, %d started", row.name, err, started)
}
}
acts, _ := h.Acts(1)
if got := acts.(map[string]any)["acts"].([]HandAct); len(got) != 1 || got[0].Caller != "ace.node-tools" || got[0].State != HandDone {
t.Errorf("the last search is recorded as %+v", got)
}
// The bound is kept in a key of its own (rev-l3 round 1): two hundred judgements before a search, or the acts
// gone from the record, never let a second search start within the hour.
h.JudgeDirectories = func(string) (string, error) { return "judged", nil }
for range 200 {
if _, err := h.Call(EngineCall{Verb: ActJudgeDirectories, Caller: "ace.node-tools"}); err != nil {
t.Fatal(err)
}
}
at = now.Add(time.Hour + 2*time.Minute)
if _, err := h.Call(EngineCall{Verb: ActSetuidSearch, Caller: "ace.node-tools"}); err == nil || !strings.Contains(err.Error(), "the next may start") {
t.Errorf("a search after two hundred judgements: %v", err)
}
keys, _ := h.State.Keys("act.")
for _, k := range keys {
_ = h.State.Delete(k)
}
if _, err := h.Call(EngineCall{Verb: ActSetuidSearch, Caller: "ace.node-tools"}); err == nil || !strings.Contains(err.Error(), "the next may start") || started != 2 {
t.Errorf("a search with the acts gone: %v, %d started", err, started)
}
// rev-l3 round 2 (ADR 0563 §2, a wrong caller gains nothing): judgements and refused searches, which any caller
// may ask, keep one rolling key of the node's, so ten thousand of them write a bounded number of bytes and add no
// key to the bucket every node shares.
keysNow := func() (int, int) {
all, _ := h.State.Keys("")
size := 0
for _, k := range all {
if e, _ := h.State.Get(k); e != nil {
size += len(e.Value)
}
}
return len(all), size
}
_, _ = h.Call(EngineCall{Verb: ActJudgeDirectories, Caller: "unproven"})
keysBefore, _ := keysNow()
for i := range 10000 {
verb := ActJudgeDirectories
if i%10 == 0 {
verb = ActSetuidSearch
}
_, _ = h.Call(EngineCall{Verb: verb, Caller: "unproven"})
}
if n, size := keysNow(); n != keysBefore || size > 64<<10 {
t.Errorf("ten thousand judgements and refused searches left %d keys (was %d) of %d bytes", n, keysBefore, size)
}
listed, _ := h.Acts(0)
if got := listed.(map[string]any)["acts"].([]HandAct); len(got) == 0 || got[0].Caller != "unproven" {
t.Errorf("acts does not show the latest judgement first: %d", len(got))
}
})
}
internal/network/network.go
+46 -7
View File
@@ -32,6 +32,7 @@ import (
"fmt"
"os"
"path/filepath"
"slices"
"sort"
"strconv"
"strings"
@@ -94,6 +95,18 @@ type Finding struct {
Writer string
// Toward is what the failure points at: TowardHub, or each resolver's address that failed.
Toward []string
// Peers is, for the tunnel, each peer of the mesh's interface and whether it was seen (Peer).
Peers []Peer
}
// Peer is one peer of the mesh's tunnel as this machine sees it: its public key, the addresses it is allowed, and
// whether it handshook within StaleHandshake. What the watchman reads to tell a machine nobody reaches from one whose
// tunnel alone is down (network-unreachable). Seen, not the handshake's time: the time moves every two minutes, the
// fact only when a peer comes or goes.
type Peer struct {
Key string `json:"key"`
Addresses []string `json:"addresses,omitempty"`
Seen bool `json:"seen"`
}
// Part is one part's state, as the statement says it.
@@ -115,6 +128,8 @@ type Statement struct {
Since time.Time `json:"since"`
At time.Time `json:"at"`
Parts []Part `json:"parts"`
// Peers are the tunnel's peers on the last look that read them, sorted by key; nil where the tunnel is not read.
Peers []Peer `json:"peers,omitempty"`
}
// Machine is what a look reads, replaced in tests.
@@ -266,6 +281,9 @@ func (j *Judge) Look(ctx context.Context) (Statement, bool) {
changed = true
}
st.State, st.Since = worst, j.overall.since
// The tunnel's peers as this look read them; a peer seen or lost is a change of its own (network-unreachable).
st.Peers = findings[PartTunnel].Peers
changed = changed || !samePeers(j.said.Peers, st.Peers)
j.said = st
return st, changed
}
@@ -435,14 +453,22 @@ func (j *Judge) lookTunnel(ctx context.Context, now time.Time) Finding {
}
}
hub := ""
allowed := map[string][]string{}
for _, line := range strings.Split(ips, "\n") {
f := strings.Fields(line)
for _, prefix := range f[min(1, len(f)):] {
allowed[f[0]] = append(allowed[f[0]], prefix)
if _, bits, ok := strings.Cut(prefix, "/"); ok && bits != "32" && bits != "128" {
hub = f[0]
}
}
}
peers := make([]Peer, 0, len(handshakes))
for key, at := range handshakes {
peers = append(peers, Peer{Key: key, Addresses: allowed[key], Seen: at > 0 && now.Sub(time.Unix(at, 0)) <= StaleHandshake})
}
sort.Slice(peers, func(i, k int) bool { return peers[i].Key < peers[k].Key })
found := func(f Finding) Finding { f.Peers = peers; return f }
age := func(at int64) string {
if at == 0 {
return "never"
@@ -452,23 +478,36 @@ func (j *Judge) lookTunnel(ctx context.Context, now time.Time) Finding {
if hub != "" {
at := handshakes[hub]
if at > 0 && now.Sub(time.Unix(at, 0)) <= StaleHandshake {
return Finding{OK: true}
return found(Finding{OK: true})
}
return Finding{Reason: "the tunnel to the hub has not handshaken for over five minutes",
Said: fmt.Sprintf("%s's newest handshake with the hub was %s", Interface, age(at)), Toward: []string{TowardHub}}
return found(Finding{Reason: "the tunnel to the hub has not handshaken for over five minutes",
Said: fmt.Sprintf("%s's newest handshake with the hub was %s", Interface, age(at)), Toward: []string{TowardHub}})
}
if len(handshakes) == 0 {
return Finding{OK: true}
return found(Finding{OK: true})
}
var newest int64
for _, at := range handshakes {
newest = max(newest, at)
}
if newest > 0 && now.Sub(time.Unix(newest, 0)) <= StaleHandshake {
return Finding{OK: true}
return found(Finding{OK: true})
}
return Finding{Reason: "no machine has handshaken with the hub's tunnel for over five minutes",
Said: fmt.Sprintf("%s's newest handshake with any of its %d peers was %s", Interface, len(handshakes), age(newest))}
return found(Finding{Reason: "no machine has handshaken with the hub's tunnel for over five minutes",
Said: fmt.Sprintf("%s's newest handshake with any of its %d peers was %s", Interface, len(handshakes), age(newest))})
}
// samePeers says two looks saw the same peers, each seen or not alike.
func samePeers(a, b []Peer) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i].Key != b[i].Key || a[i].Seen != b[i].Seen || !slices.Equal(a[i].Addresses, b[i].Addresses) {
return false
}
}
return true
}
func (j *Judge) lookBus() Finding {
internal/network/network_test.go
+19 -1
View File
@@ -302,9 +302,27 @@ func TestOnTheHubAnyFreshPeerIsAHealthyTunnel(t *testing.T) {
f.hs = "A=\t" + itoa(f.now.Add(-time.Hour).Unix()) + "\nB=\t" + itoa(f.now.Unix()) + "\nC=\t0\n"
f.ips = "A=\t10.10.0.2/32\nB=\t10.10.0.3/32\nC=\t10.10.0.4/32\n"
j := f.judge(t)
if st := f.look(t, j); st.State != Healthy {
st := f.look(t, j)
if st.State != Healthy {
t.Fatalf("the hub with one fresh peer is %+v", st.Parts)
}
// Each peer and whether it was seen within five minutes, for the watchman's network-unreachable (issue 774); a
// peer seen again is a change, a handshake renewed alone is not.
want := []Peer{{Key: "A=", Addresses: []string{"10.10.0.2/32"}}, {Key: "B=", Addresses: []string{"10.10.0.3/32"}, Seen: true},
{Key: "C=", Addresses: []string{"10.10.0.4/32"}}}
if !samePeers(st.Peers, want) {
t.Fatalf("the peers are %+v", st.Peers)
}
f.hs = "A=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nB=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nC=\t0\n"
f.now = f.now.Add(LookEvery)
if st, changed := j.Look(t.Context()); !changed || !st.Peers[0].Seen {
t.Fatalf("a peer seen again is no change: changed %v, %+v", changed, st.Peers)
}
f.hs = "A=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nB=\t" + itoa(f.now.Add(LookEvery).Unix()) + "\nC=\t0\n"
f.now = f.now.Add(LookEvery)
if _, changed := j.Look(t.Context()); changed {
t.Fatal("handshakes renewed alone are a change")
}
}
func TestAnUnreadableTunnelIsSaidAndAMissingToolSkipsIt(t *testing.T) {
internal/selfcompose/selfcompose.go
+116
View File
@@ -12,7 +12,9 @@ package selfcompose
import (
"context"
"crypto/ed25519"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
@@ -28,6 +30,7 @@ import (
"git.novox.be/novox/mesh-sdk/go/compose"
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/overlay"
"github.com/nats-io/nats.go"
@@ -75,6 +78,13 @@ type Env struct {
// Fetch reads an archive at a URL in the artifact store: a moved module's templates archive, whose texts its built
// manifest carries (issue 1126). Read once per digest. Nil reads none, and a moved module does not compose.
Fetch func(ctx context.Context, url string) ([]byte, error)
// VaultAuthority is the vault's authority as this node read it from the vault's `signers` (ADR 0550 §2), composed
// beside the controller's as the mesh's authority (frombus.SignersReader); "" or nil composes the controller's alone.
VaultAuthority func() string
// Certify asks the vault's `sign` for this node's own certificate, keeps it and writes the node's record again (ADR
// 0550 §2): asked only when a module placed here asks for a certificate and the record holds none from an authority
// the node trusts. Nil asks nothing, and such a module stands in, as before.
Certify func(ctx context.Context) error
}
// VaultSecretChanged is the vault's event that a secret changed: a name and its generation, never a value.
@@ -99,6 +109,8 @@ type Composer struct {
// sealedOwn is each own secret's sealed value as first handed, by module/name and generation: a value sealed again
// is other bytes, so the one first had is kept while the generation stands, and the composition stays the same.
sealedOwn map[string]ownSealed
// certified is the serving key a certificate was last asked for successfully, so it is asked once per key.
certified string
said string
refusal string
// held is what the composer's last pass held: each module refused alone, and the node's own ("node") when it could
@@ -411,6 +423,24 @@ func (c *Composer) takeModuleHave(u agent.Update) bool {
}
// inOwnOf is, of the modules p places, those the node runs inside its own composition (c.inOwn), sorted; c.mu held.
// Machines is, from the placements this composer holds, each node by its address on the private network, and the hub's
// node: what names the machine a failing network part points at (delivery.NetworkPart.Targets, issue 774). Empty
// before any placement is read.
func (c *Composer) Machines() (byAddress map[string]string, hub string) {
c.mu.Lock()
defer c.mu.Unlock()
byAddress = map[string]string{}
for node, p := range c.placements {
if a, _, _ := strings.Cut(p.Decided.Address, "/"); a != "" {
byAddress[a] = node
}
if p.Decided.Hub {
hub = node
}
}
return byAddress, hub
}
func (c *Composer) inOwnOf(p delivery.Placement) []string {
var out []string
for _, m := range p.Modules {
@@ -501,6 +531,8 @@ func (c *Composer) compose(ctx context.Context) []agent.Update {
if c.env.Running != nil {
running = c.env.Running()
}
// Its own certificate, asked of the vault when a module here asks for one and the record holds none it trusts.
c.certifyIfWanted(ctx, r, own)
scope := frombus.Scope{Node: c.env.Node, KeptAt: kept, InOwn: inOwn, Running: running}
missed := c.missed.Load()
compositions, _, held, err := frombus.Compose(ctx, r, c.env.Mesh, scope, fps)
@@ -743,9 +775,93 @@ func (c *Composer) Policy(ctx context.Context, module string) string {
return policy
}
// HoldsData says whether a module's manifest declares data (catalogue.Manifest.DataItems), as a composition's Data says
// it, and whether the manifest could be read: the agent's Data for what runs of a module from the kept declaration
// (agent.Env.HoldsData, issue 924).
func (c *Composer) HoldsData(ctx context.Context, module string) (data, known bool) {
c.mu.Lock()
m, ok := c.modules[module]
c.mu.Unlock()
if !ok || m.Manifest == "" {
return false, false
}
raw, err := reader{c: c}.Manifest(ctx, m.Manifest)
if err != nil {
return false, false
}
parsed, err := catalogue.ParseManifestForInputs(raw)
if err != nil {
return false, false
}
return len(parsed.DataItems()) > 0, true
}
// reader is the composer's view of the bus as frombus reads it.
type reader struct{ c *Composer }
// VaultAuthority is the vault's authority the node follows (frombus.SignersReader).
func (r reader) VaultAuthority(context.Context) string {
if r.c.env.VaultAuthority == nil {
return ""
}
return r.c.env.VaultAuthority()
}
var _ frombus.SignersReader = reader{}
// certificateWanted says whether a module the placement p puts on this node asks for a certificate for the machine's
// internal name (its manifest's `certificate`), read from the indexer's manifests; a manifest that cannot be read is
// taken as asking none, and the composition says what it misses.
func (c *Composer) certificateWanted(ctx context.Context, r reader, p delivery.Placement) bool {
for _, m := range p.Modules {
c.mu.Lock()
digest := c.modules[m.Module].Manifest
c.mu.Unlock()
if digest == "" {
continue
}
raw, err := r.Manifest(ctx, digest)
if err != nil {
continue
}
if parsed, err := catalogue.ParseManifestForInputs(raw); err == nil && parsed.Certificate != nil {
return true
}
}
return false
}
// certifyIfWanted asks Certify once per serving key when a module placed here asks for a certificate and this node's
// record holds none for its internal name and serving key from an authority it trusts (the one it holds, or the vault's).
func (c *Composer) certifyIfWanted(ctx context.Context, r reader, p delivery.Placement) {
if c.env.Certify == nil || !c.certificateWanted(ctx, r, p) {
return
}
c.mu.Lock()
rec, ok := c.records[c.env.Node]
asked := c.certified
c.mu.Unlock()
if !ok || rec.ServingKey == "" || asked == rec.ServingKey {
return
}
key, err := base64.StdEncoding.DecodeString(rec.ServingKey)
if err != nil || len(key) != ed25519.PublicKeySize {
return
}
trusted := []string{rec.Authority, r.VaultAuthority(ctx)}
if rec.Certificate != "" && delivery.CheckNodeCertificate(rec.Certificate, trusted, overlay.InternalName(c.env.Node),
ed25519.PublicKey(key), time.Now()) == nil {
return
}
if err := c.env.Certify(ctx); err != nil {
c.sayOnce(fmt.Sprintf("a module here asks for a certificate, and the vault's sign gave none: %v", err))
return
}
c.mu.Lock()
c.certified = rec.ServingKey
c.mu.Unlock()
}
func (r reader) Placements(context.Context) (map[string]delivery.Placement, error) {
r.c.mu.Lock()
defer r.c.mu.Unlock()
internal/selfcompose/selfcompose_test.go
+27
View File
@@ -3,6 +3,7 @@ package selfcompose
import (
"context"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
@@ -1221,6 +1222,32 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
}
}
b.mu.Unlock()
// L3 s7 (ADR 0550 §2): the node asks the vault for its own certificate only when a module placed here asks for one
// and its record holds none from an authority it trusts; once per serving key.
certBus := newFakeBus()
certAsks := 0
certEnv := certBus.env()
certEnv.Certify = func(context.Context) error { certAsks++; return nil }
certEnv.VaultAuthority = func() string { return "" }
cc := New(certEnv)
withCert := `{"module":"web","version":"1","certificate":{"into":"/etc/web/tls.crt"}}`
cdigest := delivery.ManifestDigest([]byte(withCert))
cc.manifests[digest] = []byte(lemurs)
cc.manifests[cdigest] = []byte(withCert)
cc.modules = map[string]delivery.Module{"lemurs": {Manifest: digest}, "web": {Manifest: cdigest}}
cc.records["n1"] = delivery.NodeRecord{Node: "n1", ServingKey: base64.StdEncoding.EncodeToString(make([]byte, 32))}
onlyLemurs := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "lemurs"}}}
cc.certifyIfWanted(context.Background(), reader{c: cc}, onlyLemurs)
if certAsks != 0 {
t.Errorf("no module asks for a certificate, and the vault was asked %d times", certAsks)
}
withWeb := delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "web"}}}
cc.certifyIfWanted(context.Background(), reader{c: cc}, withWeb)
cc.certifyIfWanted(context.Background(), reader{c: cc}, withWeb)
if certAsks != 1 {
t.Errorf("a module asks for a certificate: the vault was asked %d times, want once", certAsks)
}
}
// patternBus is a bus whose every watch gets the updates of its bucket matching its pattern, as the bus's own watches
module.json
+9
View File
@@ -17,6 +17,15 @@
"consumes": [
"restic.backed-up"
],
"state": [
{
"name": "acts",
"ttl-seconds": 7776000
},
{
"name": "health"
}
],
"config": {
"operator-signers": {
"type": "string",
packaging/operator_keys_test.go
+12
View File
@@ -68,4 +68,16 @@ func TestTheManifestConsumesTheBackupEvent(t *testing.T) {
if len(m.Consumes) != 1 || m.Consumes[0] != "restic.backed-up" {
t.Fatalf("consumes %v", m.Consumes)
}
// 774 (ADR 0563 §2, §3): the acts of the node seat and the engine's own asks are its declared state acts, the bucket
// mesh-host_acts the grants part creates and each node's engine writes under its own keys.
parsed, err := catalogue.ParseManifest(raw)
if err != nil {
t.Fatal(err)
}
// 774 (the alert table): beside it the state health, one key per node, the node's own health report, kept until
// written again (no TTL: a node that stops writing keeps its last word, which the watchman reads with its time).
if len(parsed.State) != 2 || parsed.State[0].Name != "acts" || parsed.State[0].TTLSeconds != 90*24*60*60 ||
parsed.State[1].Name != "health" || parsed.State[1].TTLSeconds != 0 {
t.Fatalf("the manifest declares the state %+v", parsed.State)
}
}