Compare commits
48
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0184e2a5b8 | ||
|
|
50c229f905 | ||
|
|
69b845c0f7 | ||
|
|
caafd9306e | ||
|
|
0d22776b20 | ||
|
|
1803d25c3a | ||
|
|
d56c576e9a | ||
|
|
fbc6d6330f | ||
|
|
fb0cdabe25 | ||
|
|
0474a2a3cb | ||
|
|
180a7e7b80 | ||
|
|
be94d2689e | ||
|
|
b225c7bb4c | ||
|
|
1d44288514 | ||
|
|
af2d2a2c14 | ||
|
|
59d77dd457 | ||
|
|
23d66187a8 | ||
|
|
4476e58080 | ||
|
|
58de6aa10a | ||
|
|
cdd3f3fd4f | ||
|
|
21d6583b5e | ||
|
|
bb46a6ad0f | ||
|
|
f8afaefba0 | ||
|
|
1e38c42646 | ||
|
|
1241ee7041 | ||
|
|
2dc384d641 | ||
|
|
e4dbb49d2f | ||
|
|
02d9d7b053 | ||
|
|
7fc0ea5478 | ||
|
|
8be1c2ab0d | ||
|
|
ff9386dccc | ||
|
|
bba9081ca1 | ||
|
|
65cd7b4787 | ||
|
|
a78239b2aa | ||
|
|
2ac0c4686a | ||
|
|
5233f7584d | ||
|
|
8858dba25e | ||
|
|
9254180cab | ||
|
|
94f9aa298c | ||
|
|
ee9fcaba20 | ||
|
|
43f2d6020e | ||
|
|
6250b1274b | ||
|
|
7f04d12047 | ||
|
|
e88ddab241 | ||
|
|
06f70e11ee | ||
|
|
61cec66ba8 | ||
|
|
5e2fadce15 | ||
|
|
882e75c954 |
@@ -0,0 +1,39 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// authorityPath is where this engine keeps the mesh's own certificate as it last applied it (public): beside the
|
||||
// certificate issued for the machine's name, read into its record (novox/hq ADR 0555 §3).
|
||||
func authorityPath(state string) string { return filepath.Join(filepath.Dir(state), "authority.crt") }
|
||||
|
||||
// keepAuthority keeps the mesh's own certificate a composition carries (the resource a module's certificate.authority
|
||||
// asks for, catalogue.AuthorityID), so the record says the authority this machine holds. A composition carrying none
|
||||
// changes nothing.
|
||||
func keepAuthority(state string, c delivery.Composed) error {
|
||||
for _, raw := range c.Resources {
|
||||
var r struct {
|
||||
ID string `json:"id"`
|
||||
Content string `json:"content"`
|
||||
}
|
||||
if json.Unmarshal(raw, &r) != nil || r.Content == "" || !strings.HasSuffix(r.ID, "."+catalogue.AuthorityID()) {
|
||||
continue
|
||||
}
|
||||
path := authorityPath(state)
|
||||
if was, err := os.ReadFile(path); err == nil && string(was) == r.Content {
|
||||
return nil
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, []byte(r.Content), 0o644)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// Issue 1126 (ADR 0555 §3): the mesh's own certificate a composition carries is kept for the machine's record, and the
|
||||
// record's keys read it; a composition carrying none changes nothing.
|
||||
func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) {
|
||||
state := filepath.Join(t.TempDir(), "state", "state.json")
|
||||
res := func(id, content string) json.RawMessage {
|
||||
raw, _ := json.Marshal(map[string]any{"id": id, "type": "file", "path": "/etc/x", "content": content})
|
||||
return raw
|
||||
}
|
||||
if err := keepAuthority(state, delivery.Composed{Resources: []json.RawMessage{res("web.conf", "x")}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(authorityPath(state)); !os.IsNotExist(err) {
|
||||
t.Fatalf("a composition with no authority kept one: %v", err)
|
||||
}
|
||||
if err := keepAuthority(state, delivery.Composed{Resources: []json.RawMessage{res("ca-trust.certificate-authority", "CA-1")}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := recordKeys(state)(); got.Authority != "CA-1" {
|
||||
t.Errorf("the record's keys read the authority %q", got.Authority)
|
||||
}
|
||||
}
|
||||
@@ -1115,6 +1115,8 @@ func runLink(ctx context.Context, opts options) error {
|
||||
proof := &proving{statePath: opts.state, version: runningVersion(), say: say}
|
||||
queue := &link.Session{
|
||||
Membership: membership,
|
||||
// And the public halves and certificate its record says (novox/hq ADR 0555 §3), read at each facts report.
|
||||
Keys: recordKeys(opts.state),
|
||||
// And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8).
|
||||
HaveWritten: proof.haveWritten,
|
||||
// And the operator's hand-over of a directory used as found, carried by the controller from the
|
||||
@@ -2131,3 +2133,28 @@ func newlyMissing(last map[string]bool, st liveness.Statement) ([]string, map[st
|
||||
}
|
||||
return missingModules(fresh), now
|
||||
}
|
||||
|
||||
// recordKeys reads the public halves of the node's keys and the certificate the mesh issued for its internal name from
|
||||
// its state directory (novox/hq ADR 0555 §3): what the node's record says of them. A key that cannot be read is left
|
||||
// empty, never invented; the certificate is there only after the first declaration that carried it.
|
||||
func recordKeys(state string) func() link.RecordKeys {
|
||||
return func() link.RecordKeys {
|
||||
var k link.RecordKeys
|
||||
if id, err := identity.Load(identity.Path(state)); err == nil {
|
||||
k.TunnelKey = id.Overlay.Public
|
||||
}
|
||||
if sk, err := identity.LoadSealingKey(identity.SealingKeyPath(state)); err == nil {
|
||||
k.SealingKey = sk.Public
|
||||
}
|
||||
if sv, err := identity.LoadServingKey(identity.ServingKeyPath(state)); err == nil {
|
||||
k.ServingKey = sv.Public
|
||||
}
|
||||
if cert, err := os.ReadFile(identity.CertificatePath(state)); err == nil {
|
||||
k.Certificate = string(cert)
|
||||
}
|
||||
if authority, err := os.ReadFile(authorityPath(state)); err == nil {
|
||||
k.Authority = string(authority)
|
||||
}
|
||||
return k
|
||||
}
|
||||
}
|
||||
|
||||
+165
-5
@@ -13,11 +13,14 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/overlay"
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/selfcompose"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
)
|
||||
|
||||
@@ -53,18 +56,27 @@ func (w bucketWatch) Watch(ctx context.Context, pattern string) (<-chan agent.Up
|
||||
type sessionHaves struct{ session *link.Session }
|
||||
|
||||
func (h sessionHaves) Read(ctx context.Context, module string) (agent.Have, bool) {
|
||||
if module == agent.NodeModule {
|
||||
// The node's own key: the digest of its own composition it runs, and what that carries sealed.
|
||||
got, ok := h.session.ReadNodeHave(ctx)
|
||||
if !ok || got.Composed == "" {
|
||||
return agent.Have{}, false
|
||||
}
|
||||
return agent.Have{Module: module, Composed: got.Composed, Sealed: got.Sealed}, true
|
||||
}
|
||||
got, ok := h.session.ReadHave(ctx, module)
|
||||
if !ok {
|
||||
return agent.Have{}, false
|
||||
}
|
||||
return agent.Have{Module: module, FP: got.FP, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail}, true
|
||||
return agent.Have{Module: module, FP: got.FP, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail,
|
||||
Sealed: got.Sealed}, true
|
||||
}
|
||||
|
||||
func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
|
||||
if x.Module == agent.NodeModule {
|
||||
// The node's own key holds its facts and the digest of the composition it runs.
|
||||
if x.Phase == "" || x.Phase == string(delivery.PhaseRunning) {
|
||||
h.session.NoteNode(ctx, x.Composed)
|
||||
h.session.NoteNode(ctx, x.Composed, x.Sealed)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -73,7 +85,8 @@ func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
|
||||
phase = ""
|
||||
}
|
||||
h.session.NoteModule(ctx, link.ModuleFact{Module: x.Module, FP: x.FP, Composed: x.Composed, Phase: phase,
|
||||
Failed: phase == string(delivery.PhaseFailed), Detail: shorten(x.Detail)})
|
||||
Failed: phase == string(delivery.PhaseFailed), Detail: shorten(x.Detail), Judgement: x.Judgement, Waits: x.Waits,
|
||||
Sealed: x.Sealed})
|
||||
}
|
||||
|
||||
func (h sessionHaves) Removed(ctx context.Context, module string) {
|
||||
@@ -401,6 +414,15 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
||||
}
|
||||
var ag *agent.Agent
|
||||
kept := keptOnce(opts, signer)
|
||||
// The mesh's own certificate the declaration kept from before carries, kept for the record before this machine
|
||||
// composes its own (ADR 0555 §3): without it a module that asks for a certificate would be stood in for.
|
||||
if byModule, _ := kept(); byModule != nil {
|
||||
for module, resources := range byModule {
|
||||
if err := keepAuthority(opts.state, delivery.Composed{Module: module, Resources: resources}); err != nil {
|
||||
say("the mesh's certificate the kept declaration carries could not be kept for this machine's record: " + err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
// The bus's user list, written alone from the newest composition of the bus module (ADR 0547 §8): on every
|
||||
// node, and a node whose compositions carry no list writes nothing.
|
||||
users := userListFollower(opts, session, composedBucket, trustedKeys(opts, signer), say, func() {
|
||||
@@ -449,8 +471,22 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
||||
Backoff: link.Backoff,
|
||||
Apply: func(ctx context.Context, c delivery.Composed, others []delivery.Composed) agent.Result {
|
||||
actx, done := applyContext(session.WithLink(ctx))
|
||||
res := applyComposition(actx, opts, c, others, sched, say)
|
||||
// While the grants part's list is followed, the controller's list in the bus module's composition is not
|
||||
// written: the apply keeps the list on the machine (ADR 0547 clause 8, never both).
|
||||
held, herr := users.Hold(c)
|
||||
var res agent.Result
|
||||
if herr != nil {
|
||||
res = failedWith(herr.Error())
|
||||
} else {
|
||||
res = applyComposition(actx, opts, held, others, sched, say)
|
||||
}
|
||||
done()
|
||||
// The mesh's own certificate it carries is kept for this machine's record (ADR 0555 §3).
|
||||
if !res.Failed {
|
||||
if err := keepAuthority(opts.state, c); err != nil {
|
||||
say("the mesh's certificate in " + c.Module + "'s composition could not be kept for this machine's record: " + err.Error())
|
||||
}
|
||||
}
|
||||
// An apply of the bus module may write an older list than the newest composition holds: looked at again.
|
||||
if module, _ := users.State(); module != "" && module == c.Module {
|
||||
users.Again()
|
||||
@@ -468,9 +504,57 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
||||
return res
|
||||
},
|
||||
}
|
||||
// The node composes its own modules once the catalogue places it (novox/hq ADR 0547 §2, ADR 0549, issue 1126): its
|
||||
// placement key is read at the agent's start; until then it applies the controller's compositions as before.
|
||||
run, stopRun := context.WithCancel(linked)
|
||||
// What the node runs and how it opens a sealed value, for the composer to keep the boxes it runs; and the agent
|
||||
// told when the node cannot compose, so each wanted module's have says it.
|
||||
hooks := composerHooks{
|
||||
running: func() []delivery.Composed {
|
||||
if now := current.get(); now != nil {
|
||||
return now.Everything()
|
||||
}
|
||||
return nil
|
||||
},
|
||||
unseal: sealOpener(opts.state),
|
||||
// And what each module's have says it runs sealed: after a restart the agent has applied nothing yet.
|
||||
held: func(module string) []string {
|
||||
if module == agent.NodeModule {
|
||||
got, _ := session.ReadNodeHave(linked)
|
||||
return got.Sealed
|
||||
}
|
||||
got, ok := session.ReadHave(linked, module)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return got.Sealed
|
||||
},
|
||||
refused: func(why string) {
|
||||
if now := current.get(); now != nil {
|
||||
now.NotComposed(linked, why)
|
||||
}
|
||||
},
|
||||
}
|
||||
// And its answer to every placement candidate, placed or not, applying nothing (novox/hq ADR 0559 §4): written into
|
||||
// its own key, `<node>.node`, under candidates.
|
||||
shadowHooks := hooks
|
||||
shadowHooks.refused = nil
|
||||
shadow := selfcompose.NewShadow(composerEnv(session, node, say, shadowHooks),
|
||||
func(ctx context.Context, answers map[string]delivery.CandidateAnswer) {
|
||||
session.NoteCandidates(ctx, answers)
|
||||
})
|
||||
go shadow.Keep(linked, link.Backoff)
|
||||
if composer := selfComposer(linked, session, node, say, hooks); composer != nil {
|
||||
env.Composed, env.SelfComposed, env.Trusted, env.Policy = composer, true, nil, composer.Policy
|
||||
} else {
|
||||
// And once the catalogue places it, the agent begins again composing its own: the placement key appearing ends
|
||||
// this run, so a node does not wait for a restart to follow the seed of its placement (ADR 0549 §4).
|
||||
go endWhenPlaced(run, session, node, stopRun, say)
|
||||
}
|
||||
ag = agent.New(env)
|
||||
current.set(ag)
|
||||
err = ag.Run(linked)
|
||||
err = ag.Run(run)
|
||||
stopRun()
|
||||
cancel()
|
||||
current.set(nil)
|
||||
if ctx.Err() != nil {
|
||||
@@ -748,3 +832,79 @@ const applyBound = 30 * time.Minute
|
||||
func applyContext(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||
return context.WithTimeout(context.WithoutCancel(ctx), applyBound)
|
||||
}
|
||||
|
||||
// selfComposer is the node's own composer when the catalogue places the node (its key `placement.<node>` in the
|
||||
// indexer's state), and nil, said once, while it does not or the key cannot be read.
|
||||
// composerHooks are what the composer reads of the agent and the machine (selfcompose.Env Running, Unseal, Refused).
|
||||
type composerHooks struct {
|
||||
running func() []delivery.Composed
|
||||
unseal func(string) ([]byte, error)
|
||||
refused func(string)
|
||||
held func(module string) []string
|
||||
}
|
||||
|
||||
func selfComposer(ctx context.Context, session *link.Session, node string, say func(string), hooks composerHooks) *selfcompose.Composer {
|
||||
key, err := delivery.PlacementKey(node)
|
||||
if err != nil {
|
||||
say("this node applies the controller's compositions: " + err.Error())
|
||||
return nil
|
||||
}
|
||||
got, err := session.KVHistory(ctx, delivery.PlacementBucket, key)
|
||||
if err != nil || len(got) == 0 || got[len(got)-1].Deleted {
|
||||
why := "the catalogue does not place it yet"
|
||||
if err != nil {
|
||||
why = "its placement cannot be read: " + err.Error()
|
||||
}
|
||||
say("this node applies the controller's compositions: " + why)
|
||||
return nil
|
||||
}
|
||||
say("this node composes its own modules: the catalogue places it (" + key + ")")
|
||||
return selfcompose.New(composerEnv(session, node, say, hooks))
|
||||
}
|
||||
|
||||
// composerEnv is what the node's composers read of the bus, the vault and the agent: its own composer and its answer
|
||||
// to placement candidates (selfcompose.Shadow) alike.
|
||||
func composerEnv(session *link.Session, node string, say func(string), hooks composerHooks) selfcompose.Env {
|
||||
return selfcompose.Env{Node: node,
|
||||
Watch: func(ctx context.Context, bucket, pattern string) (<-chan agent.Update, error) {
|
||||
return bucketWatch{session, bucket}.Watch(ctx, pattern)
|
||||
},
|
||||
Get: func(ctx context.Context, bucket, key string) ([]byte, error) {
|
||||
h, err := session.KVHistory(ctx, bucket, key)
|
||||
if err != nil || len(h) == 0 || h[len(h)-1].Deleted {
|
||||
return nil, err
|
||||
}
|
||||
return h[len(h)-1].Value, nil
|
||||
},
|
||||
Mesh: frombus.Mesh{InternalSuffix: overlay.Suffix()},
|
||||
Say: say,
|
||||
// Its modules' own secrets, sealed by the vault to this machine's key (ADR 0550 §5).
|
||||
Own: func(ctx context.Context, args map[string]any) (json.RawMessage, error) {
|
||||
return session.AskVault(ctx, "own", args)
|
||||
},
|
||||
// And the vault's word that a secret changed, on which it composes again (ADR 0547 §2).
|
||||
Hear: session.Hear,
|
||||
Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held}
|
||||
}
|
||||
|
||||
// endWhenPlaced calls stop once this node's placement key appears in the indexer's placement (a value after the watch
|
||||
// caught up, or one already there), so the agent begins again and composes its own. A watch that cannot be made is said;
|
||||
// the node then switches at its next start.
|
||||
func endWhenPlaced(ctx context.Context, session *link.Session, node string, stop func(), say link.Announce) {
|
||||
key, err := delivery.PlacementKey(node)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
updates, err := bucketWatch{session, delivery.PlacementBucket}.Watch(ctx, key)
|
||||
if err != nil {
|
||||
say("whether the catalogue places this node is not followed (" + err.Error() + "): it composes its own from its next start")
|
||||
return
|
||||
}
|
||||
for u := range updates {
|
||||
if !u.Caught && !u.Deleted && len(u.Value) > 0 {
|
||||
say("the catalogue places this node now (" + key + "): the node agent begins again, composing its own modules")
|
||||
stop()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,11 +14,13 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/selfcompose"
|
||||
)
|
||||
|
||||
// The node agent whole, against a real bus with JetStream started in the test: the wants bucket, the node's composition
|
||||
@@ -28,9 +30,14 @@ import (
|
||||
const itNode = "it1"
|
||||
|
||||
type itBed struct {
|
||||
t *testing.T
|
||||
conn *nats.Conn
|
||||
js nats.JetStreamContext
|
||||
t *testing.T
|
||||
conn *nats.Conn
|
||||
js nats.JetStreamContext
|
||||
// nodeConn and nodeJS are the node's own, under its real grant (buscompose.PermissionsFor, KindNode): the agent runs
|
||||
// on them, so a watch or a read the grant does not allow fails here as it fails on the bus (issue 1126, 2026-10-27
|
||||
// 09:21: a watch of every have key stopped every node agent). conn is the test's, unrestricted, for setting up.
|
||||
nodeConn *nats.Conn
|
||||
nodeJS nats.JetStreamContext
|
||||
wants nats.KeyValue
|
||||
composed nats.KeyValue
|
||||
haves nats.KeyValue
|
||||
@@ -43,9 +50,22 @@ type itBed struct {
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func aServer(t *testing.T) *nats.Conn {
|
||||
// aServer is an embedded bus with JetStream and two users: the test's, unrestricted, and the node's, granted exactly what
|
||||
// buscompose.PermissionsFor gives a node (KindNode); it answers a connection of each.
|
||||
func aServer(t *testing.T) (admin, node *nats.Conn) {
|
||||
t.Helper()
|
||||
srv, err := server.NewServer(&server.Options{Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true})
|
||||
perms, err := buscompose.PermissionsFor(buscompose.Principal{Kind: buscompose.KindNode, Node: itNode})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
granted := &server.Permissions{Publish: &server.SubjectPermission{Allow: perms.Publish, Deny: perms.PublishDeny},
|
||||
Subscribe: &server.SubjectPermission{Allow: perms.Subscribe}}
|
||||
if perms.AllowResponses {
|
||||
granted.Response = &server.ResponsePermission{MaxMsgs: 1, Expires: buscompose.ResponseTTL}
|
||||
}
|
||||
nodeUser := buscompose.Principal{Kind: buscompose.KindNode, Node: itNode}.Username()
|
||||
srv, err := server.NewServer(&server.Options{Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true,
|
||||
Users: []*server.User{{Username: "test", Password: "test"}, {Username: nodeUser, Password: "node", Permissions: granted}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -54,12 +74,15 @@ func aServer(t *testing.T) *nats.Conn {
|
||||
t.Fatal("the embedded server did not start")
|
||||
}
|
||||
t.Cleanup(srv.Shutdown)
|
||||
conn, err := nats.Connect(srv.ClientURL())
|
||||
if err != nil {
|
||||
if admin, err = nats.Connect(srv.ClientURL(), nats.UserInfo("test", "test")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(conn.Close)
|
||||
return conn
|
||||
t.Cleanup(admin.Close)
|
||||
if node, err = nats.Connect(srv.ClientURL(), nats.UserInfo(nodeUser, "node"), nats.CustomInboxPrefix("_INBOX."+nodeUser)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(node.Close)
|
||||
return admin, node
|
||||
}
|
||||
|
||||
func newItBed(t *testing.T) *itBed {
|
||||
@@ -74,11 +97,15 @@ func newItBed(t *testing.T) *itBed {
|
||||
builtFor = "arch"
|
||||
t.Cleanup(func() { builtFor = was })
|
||||
|
||||
b := &itBed{t: t, conn: aServer(t), dir: t.TempDir()}
|
||||
b := &itBed{t: t, dir: t.TempDir()}
|
||||
b.conn, b.nodeConn = aServer(t)
|
||||
var err error
|
||||
if b.js, err = b.conn.JetStream(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b.nodeJS, err = b.nodeConn.JetStream(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
composedBucket, _ := delivery.ComposedBucketFor(itNode)
|
||||
for _, c := range []struct {
|
||||
name string
|
||||
@@ -112,8 +139,8 @@ func (b *itBed) start(trustOperator bool) (stop func()) {
|
||||
}
|
||||
}
|
||||
b.session = &link.Session{Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second,
|
||||
Say: b.say}
|
||||
b.session.Attach(context.Background(), link.OverNATS{Conn: b.conn, JS: b.js, Node: itNode})
|
||||
Say: b.say, Keys: func() link.RecordKeys { return link.RecordKeys{} }}
|
||||
b.session.Attach(context.Background(), link.OverNATS{Conn: b.nodeConn, JS: b.nodeJS, Node: itNode})
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
@@ -311,3 +338,116 @@ func TestNatsTheHandPathAppliesACompositionTheOperatorSigned(t *testing.T) {
|
||||
return h.FP == fp && h.Composed == dig && h.Phase == delivery.PhaseRunning
|
||||
})
|
||||
}
|
||||
|
||||
// Issue 1126 (ADR 0547 §2, the milestone's H8): the node agent started whole against a real bus, once the catalogue
|
||||
// places its node (the placement written after it started), composes a module from its want, the indexer's manifest, its record and its mesh-config value, and applies it,
|
||||
// with no key in the controller's compositions; a changed config value is composed again on its event and applied as a
|
||||
// revision of what runs. And a catalogue pull request's placement candidate equal to its placement is answered "same"
|
||||
// in the node's own key, and nothing is applied for it (ADR 0559 §4, subtask 10 of the milestone).
|
||||
func TestNatsANodeThePlacementNamesComposesAndAppliesItsOwn(t *testing.T) {
|
||||
b := newItBed(t)
|
||||
kvOf := func(bucket string) nats.KeyValue {
|
||||
kv, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: bucket, History: 5})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return kv
|
||||
}
|
||||
placement, modules, manifests := kvOf(delivery.PlacementBucket), kvOf(delivery.ModulesBucket), kvOf(delivery.ManifestsBucket)
|
||||
resolved := kvOf(selfcompose.ResolvedBucket)
|
||||
kvOf(delivery.BuilderBuildsBucket)
|
||||
candidates := kvOf(delivery.PlacementCandidatesBucket)
|
||||
// The controller's stream the node's facts go into, so the node's key carries the facts it says.
|
||||
if _, err := b.js.AddStream(&nats.StreamConfig{Name: "CONTROL", Subjects: []string{link.ReportSubject(itNode)}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
path := filepath.Join(b.dir, "web.conf")
|
||||
man := `{"module":"web","version":"1","config":{"greeting":{"type":"string","description":"what the file says","default":"hello"}},` +
|
||||
`"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0644","content":"greeting=${config:greeting}\n"}]}`
|
||||
digest := delivery.ManifestDigest([]byte(man))
|
||||
mraw, _ := delivery.EncodeManifest(delivery.Manifest{Module: "web", JSON: man})
|
||||
mkey, _ := delivery.ManifestsKey(digest)
|
||||
fp := fpIt("a")
|
||||
modv, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40)})
|
||||
place, _ := delivery.EncodePlacement(delivery.Placement{Node: itNode, Modules: []delivery.PlacedModule{{Module: "web"}}})
|
||||
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: itNode, Capabilities: []string{"service-manager"}}})
|
||||
pkey, _ := delivery.PlacementKey(itNode)
|
||||
hkey, _ := delivery.HaveNodeKey(itNode)
|
||||
for _, w := range []struct {
|
||||
kv nats.KeyValue
|
||||
key string
|
||||
value []byte
|
||||
}{{manifests, mkey, mraw}, {modules, "web", modv}, {b.haves, hkey, rec},
|
||||
{resolved, itNode + ".web.greeting", []byte(`{"value":"bonjour"}`)}} {
|
||||
if _, err := w.kv.Put(w.key, w.value); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fp, Self: true})
|
||||
wkey, _ := delivery.WantKey(itNode, "web")
|
||||
if _, err := b.wants.Put(wkey, want); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
stop := b.start(false)
|
||||
defer stop()
|
||||
// Not placed yet: the agent applies the controller's compositions, of which there are none. The catalogue's seed then
|
||||
// places the node, and the agent begins again composing its own (ADR 0549 §4), with no restart of the engine.
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if b.file("web") != "" {
|
||||
t.Fatal("a node the catalogue does not place composed its own")
|
||||
}
|
||||
if _, err := placement.Put(pkey, place); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
within(t, 5*time.Second, "the node did not compose and apply its own module once placed", func() bool { return b.file("web") == "greeting=bonjour\n" })
|
||||
within(t, 2*time.Second, "the have does not name the build", func() bool {
|
||||
h := b.have("web")
|
||||
return h.FP == fp && h.Composed != "" && h.Phase == delivery.PhaseRunning
|
||||
})
|
||||
if keys, _ := b.composed.Keys(); len(keys) != 0 {
|
||||
t.Errorf("the controller's compositions hold %v; the node composed its own", keys)
|
||||
}
|
||||
first := b.have("web").Composed
|
||||
if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"salut"}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
within(t, 3*time.Second, "a changed config value was not composed and applied again", func() bool { return b.file("web") == "greeting=salut\n" })
|
||||
within(t, 2*time.Second, "the have does not name the revision", func() bool {
|
||||
h := b.have("web")
|
||||
return h.FP == fp && h.Composed != first && h.Phase == delivery.PhaseRunning
|
||||
})
|
||||
|
||||
// The node says its facts (its record, as the test wrote it), then a candidate equal to its placement is written: the
|
||||
// node answers "same" at the candidate's head and revision in its own key, and applies nothing.
|
||||
b.session.SayFacts(context.Background(), link.Report{Profile: map[string]any{"capabilities": []any{
|
||||
map[string]any{"name": "service-manager", "present": true}}}})
|
||||
second := b.have("web").Composed
|
||||
same, _ := delivery.DecodePlacement(place)
|
||||
craw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Group: "feat/x",
|
||||
Pull: delivery.Head{Repository: delivery.CatalogueRepository, Number: 1, Head: "h1"}, Placement: &same})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ckey, _ := delivery.PlacementCandidateKey("feat/x", itNode)
|
||||
crev, err := candidates.Put(ckey, craw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var answer delivery.CandidateAnswer
|
||||
within(t, 10*time.Second, "the node did not answer its candidate", func() bool {
|
||||
e, err := b.haves.Get(hkey)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
n, err := delivery.DecodeHaveNode(e.Value())
|
||||
answer = n.Candidates["feat/x"]
|
||||
return err == nil && answer.Head == "h1" && n.Record != nil
|
||||
})
|
||||
if answer.Verdict != delivery.CandidateSame || answer.Revision != crev || len(answer.Same) != 1 || answer.Same[0] != "web" {
|
||||
t.Errorf("the answer to a candidate equal to the placement: %+v", answer)
|
||||
}
|
||||
if h := b.have("web"); h.Composed != second || b.file("web") != "greeting=salut\n" {
|
||||
t.Errorf("a candidate moved what runs: %+v, %q", h, b.file("web"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,9 +7,13 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/seal"
|
||||
"github.com/novox/mesh-host/internal/apply"
|
||||
"github.com/novox/mesh-host/internal/identity"
|
||||
"github.com/novox/mesh-host/internal/link"
|
||||
"github.com/novox/mesh-host/internal/store"
|
||||
"github.com/novox/mesh-host/internal/system"
|
||||
@@ -42,6 +46,35 @@ func userListFollower(opts options, session *link.Session, composedBucket string
|
||||
Report: func(ctx context.Context, module string, w userlist.Written) {
|
||||
session.NoteUserList(ctx, module, w)
|
||||
},
|
||||
// From take-over, the grants part's list (ADR 0547 clauses 7, 8): its one key watched, its object read by name and
|
||||
// opened with this machine's sealing key, the key's public half said in the have so the grants part seals to it.
|
||||
Grants: bucketWatch{session, delivery.GrantsUserListBucket},
|
||||
Fetch: func(ctx context.Context, name string) ([]byte, error) {
|
||||
return session.GrantsObject(ctx, name, grantsListLimit)
|
||||
},
|
||||
Open: func(b seal.Box) ([]byte, error) {
|
||||
key, err := identity.LoadSealingKey(identity.SealingKeyPath(opts.state))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return key.OpenVaultBox(b)
|
||||
},
|
||||
Key: func() string {
|
||||
key, err := identity.LoadSealingKey(identity.SealingKeyPath(opts.state))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
pem, err := key.PublicPEM()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return pem
|
||||
},
|
||||
// The grants part's key as last heard, kept beside the node's state, followed while it cannot be watched; its
|
||||
// watch tried again as a passing failure of the bus is (ADR 0547 clause 8).
|
||||
Kept: func() ([]byte, uint64) { return keptGrantsKey(opts) },
|
||||
Keep: func(value []byte, rev uint64) { keepGrantsKey(opts, value, rev, say) },
|
||||
Backoff: link.Backoff,
|
||||
// The vault's key from its answer to node-key: the loaded list's hashes are handed sealed to it (ADR 0547
|
||||
// clause 9).
|
||||
VaultKey: vaultKey.get,
|
||||
@@ -49,6 +82,9 @@ func userListFollower(opts options, session *link.Session, composedBucket string
|
||||
})
|
||||
}
|
||||
|
||||
// grantsListLimit is the largest sealed list the bus node reads: the live list is about a hundred kilobytes.
|
||||
const grantsListLimit = 16 << 20
|
||||
|
||||
// userListFile is a user-list resource as the file it declares.
|
||||
func userListFile(raw []byte) (*declaration.File, error) {
|
||||
r, err := declaration.ParseResource(raw)
|
||||
@@ -125,3 +161,44 @@ func writeUserListAlone(ctx context.Context, opts options, raw []byte, body []by
|
||||
}
|
||||
return digest, writeErr == nil && out.Action != "unchanged", writeErr
|
||||
}
|
||||
|
||||
// grantsKeyPath is where the grants part's user-list key last heard is kept, beside the node's state.
|
||||
func grantsKeyPath(opts options) string {
|
||||
return filepath.Join(filepath.Dir(opts.state), "grants-user-list.json")
|
||||
}
|
||||
|
||||
type keptKey struct {
|
||||
Revision uint64 `json:"revision"`
|
||||
Value json.RawMessage `json:"value"`
|
||||
}
|
||||
|
||||
func keptGrantsKey(opts options) ([]byte, uint64) {
|
||||
raw, err := os.ReadFile(grantsKeyPath(opts))
|
||||
if err != nil {
|
||||
return nil, 0
|
||||
}
|
||||
var k keptKey
|
||||
if json.Unmarshal(raw, &k) != nil || len(k.Value) == 0 {
|
||||
return nil, 0
|
||||
}
|
||||
return k.Value, k.Revision
|
||||
}
|
||||
|
||||
func keepGrantsKey(opts options, value []byte, rev uint64, say link.Announce) {
|
||||
path := grantsKeyPath(opts)
|
||||
if value == nil {
|
||||
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
announceOr(say)("the grants part's user-list key kept on this machine could not be taken away: " + err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
raw, _ := json.Marshal(keptKey{Revision: rev, Value: value})
|
||||
tmp := path + ".new"
|
||||
err := os.WriteFile(tmp, raw, 0o600)
|
||||
if err == nil {
|
||||
err = os.Rename(tmp, path)
|
||||
}
|
||||
if err != nil {
|
||||
announceOr(say)("the grants part's user-list key could not be kept on this machine: " + err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ module github.com/novox/mesh-host
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026102838-174789941936
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027075026-05b678310366
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
golang.org/x/crypto v0.57.0
|
||||
|
||||
@@ -1,11 +1,17 @@
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f h1:lUEyNPPF3w4Q0LIgcUUD6ofkKVhCF0n7dVjMoixQmXo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024234636-10b6e202320f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204213-f63fa86c7456 h1:He2o9vSKjgKbWOCOgOJY0WlaDh1n8RcGhvGY/dwXsBQ=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204213-f63fa86c7456/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204730-0be0ffd6159e h1:k/1NjjW5HuuuHW2ldyWTvjp+1u3fN0RO18Q57Iup6Dk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261025204730-0be0ffd6159e/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026102838-174789941936 h1:HGBoWimZXYZp+Qn9VCstYDitOP84t9EUQYd1yM9j2eg=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026102838-174789941936/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117 h1:88CNZOoQgu7YPYKW16qkWT8Gw9Ucg07aTrIImA2nYUk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261026210326-979644420117/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa h1:tT6sl3vY0iVAXw+mqy3iGKUBUjVBRWvQvKUOTmTyWwk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef h1:uMqybf/sTAjapyllVjy8mjCi8evpcOmgnjmqRdJRsFo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012259-a37d2622ed56 h1:PUc5s3oaWEg0uyVmXhJSSK3x0IXZBXWGqq30U/CSx30=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012259-a37d2622ed56/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012704-062c29c3a22f h1:AKzV973UTWe0OYFoIjgGu0f+B26xh8QsAHzCpsTIjM8=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012704-062c29c3a22f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f h1:mimAGERguQmCGYVNdRTrFcZ1dhpU6GWo+Q3EDhHOrVk=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027014310-cadd7b1b2f8f/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027075026-05b678310366 h1:B7oQ82ar2McqZ9OrbJfyvG+50Ol6fBUoK3eYE2uTPvo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027075026-05b678310366/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
+251
-14
@@ -23,6 +23,7 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -49,9 +50,16 @@ type Have struct {
|
||||
Module string
|
||||
FP string
|
||||
Composed string
|
||||
// Phase is "running", "failed", "held" or "removed"; empty reads as running.
|
||||
// Phase is "running", "failed", "held", "waiting-yes" or "removed"; empty reads as running.
|
||||
Phase string
|
||||
Detail string
|
||||
// Judgement and Waits are, in phase waiting-yes, the judgement of the move and the digest of the composition that
|
||||
// waits for a person's yes (ADR 0547 §3).
|
||||
Judgement string
|
||||
Waits string
|
||||
// Sealed is, on a node composing its own, each value the composition it names carries sealed to the node's key
|
||||
// (delivery.Have.Sealed): kept in the have so the composer reuses them after a restart (ADR 0547 §3).
|
||||
Sealed []string
|
||||
}
|
||||
|
||||
// Haves is the node's have bucket as the agent uses it.
|
||||
@@ -120,6 +128,14 @@ type Env struct {
|
||||
// transient failure only on RetryTransient or a change of either key.
|
||||
Backoff func(try int) time.Duration
|
||||
Say func(string)
|
||||
// SelfComposed says the compositions are the node's own (novox/hq ADR 0547 §2, issue 1126): Composed hands what the
|
||||
// node composed itself, unsigned, and a want names the build alone. The composition composed for the want's build is
|
||||
// the one applied, when it differs from what runs.
|
||||
SelfComposed bool
|
||||
// Policy is a module's upgrade policy as its manifest derives it (catalogue.DerivedUpgrade); with SelfComposed, a
|
||||
// move of a module of policy record that recreates a container or moves its data waits for a person's yes in its
|
||||
// want (ADR 0547 §3). Nil judges no move.
|
||||
Policy func(ctx context.Context, module string) string
|
||||
}
|
||||
|
||||
// transientFor is how many transient retries say the phase applying before the have says failed: past them the bus has
|
||||
@@ -143,7 +159,9 @@ type Agent struct {
|
||||
// prior is a module the node ran before this agent started, so it has something to back up.
|
||||
prior map[string]bool
|
||||
said map[string]string
|
||||
retry chan []string
|
||||
// notComposed is the modules whose have says the node could not compose them: why, and the have before (NotComposed).
|
||||
notComposed map[string]notComposedMark
|
||||
retry chan []string
|
||||
// tries counts the transient failures in a row of each module; again carries a retry whose wait ran out, and
|
||||
// transient asks for every transiently failed module at once. stopped is closed when Run returns.
|
||||
tries map[string]int
|
||||
@@ -189,7 +207,7 @@ func New(env Env) *Agent {
|
||||
env.Say = func(string) {}
|
||||
}
|
||||
return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{},
|
||||
attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{},
|
||||
attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, notComposed: map[string]notComposedMark{},
|
||||
retry: make(chan []string, 8), tries: map[string]int{}, again: make(chan againAsk, 64),
|
||||
transient: make(chan struct{}, 1), stopped: make(chan struct{})}
|
||||
}
|
||||
@@ -290,6 +308,143 @@ func (a *Agent) RetryTransient() {
|
||||
}
|
||||
}
|
||||
|
||||
// notComposedMark is why a module's have says the node could not compose it, and the have it said before.
|
||||
type notComposedMark struct {
|
||||
why string
|
||||
prior Have
|
||||
}
|
||||
|
||||
// NotComposed says the node composing its own could not compose its modules (ADR 0547 §2: a stand-in refused, an
|
||||
// input that cannot be read): each module with a want gets its have failed, naming why, while it keeps running what it
|
||||
// ran, so the failure is seen where every have is read and not only in the journal. The have keeps what runs: the
|
||||
// fingerprint, the digest and the sealed values the agent runs, or, before it ran anything since its start, those its
|
||||
// last have said (a refusal at the first compose after a start). Called with "" once the node composes again: each
|
||||
// module so marked has its have back as it was, unless an apply wrote it since.
|
||||
func (a *Agent) NotComposed(ctx context.Context, why string) {
|
||||
a.mu.Lock()
|
||||
var modules []string
|
||||
if why != "" {
|
||||
for module, w := range a.wants {
|
||||
if w.Present && !w.Want.Removed && module != NodeModule && a.notComposed[module].why != why {
|
||||
modules = append(modules, module)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for module := range a.notComposed {
|
||||
modules = append(modules, module)
|
||||
}
|
||||
}
|
||||
a.mu.Unlock()
|
||||
sort.Strings(modules)
|
||||
for _, module := range modules {
|
||||
a.mu.Lock()
|
||||
mark, marked := a.notComposed[module]
|
||||
r, ran := a.running[module]
|
||||
a.mu.Unlock()
|
||||
if why == "" {
|
||||
a.mu.Lock()
|
||||
delete(a.notComposed, module)
|
||||
a.mu.Unlock()
|
||||
back := mark.prior
|
||||
if ran {
|
||||
back = Have{Module: module, FP: r.FP, Composed: r.Composed, Phase: string(delivery.PhaseRunning)}
|
||||
if r.Phase != "" {
|
||||
back.Phase = string(r.Phase)
|
||||
}
|
||||
}
|
||||
if back.Composed == "" {
|
||||
continue // nothing said what ran: nothing to say back
|
||||
}
|
||||
a.writeHave(ctx, back)
|
||||
continue
|
||||
}
|
||||
if !marked {
|
||||
if ran {
|
||||
mark.prior = Have{Module: module, FP: r.FP, Composed: r.Composed, Phase: string(r.Phase)}
|
||||
if mark.prior.Phase == "" {
|
||||
mark.prior.Phase = string(delivery.PhaseRunning)
|
||||
}
|
||||
} else if h, ok := a.env.Haves.Read(ctx, module); ok {
|
||||
mark.prior = h // what its last have said runs: this agent has run nothing of it since its start
|
||||
}
|
||||
}
|
||||
mark.why = why
|
||||
a.mu.Lock()
|
||||
a.notComposed[module] = mark
|
||||
a.mu.Unlock()
|
||||
a.writeHave(ctx, Have{Module: module, FP: mark.prior.FP, Composed: mark.prior.Composed, Sealed: mark.prior.Sealed,
|
||||
Phase: string(delivery.PhaseFailed), Detail: "the node cannot compose it, and keeps running what it ran: " + why})
|
||||
}
|
||||
}
|
||||
|
||||
// writeHave writes a module's have (the node's own key for NodeModule); on a node composing its own, with the sealed
|
||||
// values of the composition it names when the caller gave none (Have.Sealed): those of the composition the agent applied or holds at that digest, else
|
||||
// those the module's last have said at that digest.
|
||||
//
|
||||
// A writer that names no composition (one written from the agent's running entry before this agent ran anything of the
|
||||
// module since its start: held by a failed backup, waiting for a yes, not composed) keeps what the module's last have
|
||||
// said runs, its fingerprint, digest and sealed values, rather than a have naming nothing: what runs is not lost, and
|
||||
// the boxes the composer reuses after a restart stay (review of L2, rounds 2 and 3).
|
||||
func (a *Agent) writeHave(ctx context.Context, h Have) {
|
||||
if h.Composed == "" && h.Module != NodeModule {
|
||||
if last, ok := a.env.Haves.Read(ctx, h.Module); ok && last.Composed != "" {
|
||||
h.FP, h.Composed = last.FP, last.Composed
|
||||
if h.Sealed == nil {
|
||||
h.Sealed = last.Sealed
|
||||
}
|
||||
}
|
||||
}
|
||||
// The node's own composition too: its core boxes are sealed again by the vault at a restart, and kept in its key the
|
||||
// composer reuses them, so the restart applies nothing again (review of L2, round 3, finding 1).
|
||||
if a.env.SelfComposed && h.Sealed == nil && h.Composed != "" {
|
||||
h.Sealed = a.sealedOf(ctx, h.Module, h.FP, h.Composed)
|
||||
}
|
||||
a.env.Haves.Write(ctx, h)
|
||||
}
|
||||
|
||||
func (a *Agent) sealedOf(ctx context.Context, module, fp, composed string) []string {
|
||||
a.mu.Lock()
|
||||
if c, ok := a.applied[module]; ok && c.Err == nil && c.Dig == composed {
|
||||
a.mu.Unlock()
|
||||
return SealedIn(c.C)
|
||||
}
|
||||
if c, ok := a.compositionFor(module, fp); ok && c.Err == nil && c.Dig == composed {
|
||||
a.mu.Unlock()
|
||||
return SealedIn(c.C)
|
||||
}
|
||||
a.mu.Unlock()
|
||||
if h, ok := a.env.Haves.Read(ctx, module); ok && h.Composed == composed {
|
||||
return h.Sealed
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SealedIn is each value a composition carries sealed to its node's key: a file's `sealed` and a vault box
|
||||
// (catalogue.VaultSealedField), as their JSON text, in the order of its resources.
|
||||
func SealedIn(c delivery.Composed) []string {
|
||||
var out []string
|
||||
for _, raw := range c.Resources {
|
||||
var r map[string]json.RawMessage
|
||||
if json.Unmarshal(raw, &r) != nil {
|
||||
continue
|
||||
}
|
||||
for _, field := range []string{"sealed", catalogue.VaultSealedField} {
|
||||
f, ok := r[field]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
var text string
|
||||
if json.Unmarshal(f, &text) != nil {
|
||||
text = string(f) // a vault box is carried as its JSON
|
||||
}
|
||||
if text != "" {
|
||||
out = append(out, text)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Running is the compositions the node runs now: each module's that the agent applied or found already running, and
|
||||
// the node's own. What the node judges its health by and says of itself is built from them.
|
||||
func (a *Agent) Running() []delivery.Composed {
|
||||
@@ -366,7 +521,14 @@ func (a *Agent) takeComposition(u Update) string {
|
||||
}
|
||||
now := compNow{Raw: u.Value, Rev: u.Revision}
|
||||
c, err := delivery.DecodeComposed(u.Value)
|
||||
if err == nil {
|
||||
if err == nil && a.env.SelfComposed {
|
||||
// The node composed it itself: nothing crossed the bus, so nothing is signed (ADR 0547 §2).
|
||||
now.C = c
|
||||
now.Dig, err = delivery.ComposedDigestOf(c)
|
||||
if err != nil {
|
||||
now.C = delivery.Composed{}
|
||||
}
|
||||
} else if err == nil {
|
||||
var trusted []ed25519.PublicKey
|
||||
if a.env.Trusted != nil {
|
||||
trusted = a.env.Trusted()
|
||||
@@ -636,7 +798,16 @@ func (a *Agent) settleOne(ctx context.Context, module string) {
|
||||
a.mu.Lock()
|
||||
comp, have := a.compositionFor(module, w.Want.FP)
|
||||
a.mu.Unlock()
|
||||
if (!have || comp.Err != nil || comp.Dig != w.Want.Composed) && w.Want.Composed != "" {
|
||||
if a.env.SelfComposed {
|
||||
if !have || comp.Err != nil {
|
||||
return // waits for the node's own composition of the want's build
|
||||
}
|
||||
// The composition the node composed for the want's build is the one it applies (ADR 0547 §2).
|
||||
w.Want.Composed = comp.Dig
|
||||
if a.waitsForYes(ctx, module, w, comp) {
|
||||
return
|
||||
}
|
||||
} else if (!have || comp.Err != nil || comp.Dig != w.Want.Composed) && w.Want.Composed != "" {
|
||||
// The composition the want names may be an older revision of the key, replaced by a newer one at the same
|
||||
// fingerprint (a want held there keeps its digest): taken from the revisions the bucket keeps (ADR 0539 clause 1).
|
||||
if old, found := a.revisionNamed(ctx, module, w.Want.FP, w.Want.Composed); found {
|
||||
@@ -736,8 +907,8 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
|
||||
a.mu.Unlock()
|
||||
// Said again, so the session holds the module and its key carries the health this engine judges,
|
||||
// not the one an older engine left there (xorg's unknown on g14 and shanks, 2026-10-25).
|
||||
a.env.Haves.Write(ctx, Have{Module: module, FP: h.FP, Composed: h.Composed,
|
||||
Phase: string(delivery.PhaseRunning), Detail: h.Detail})
|
||||
a.writeHave(ctx, Have{Module: module, FP: h.FP, Composed: h.Composed,
|
||||
Phase: string(delivery.PhaseRunning), Detail: h.Detail, Sealed: h.Sealed})
|
||||
return true
|
||||
}
|
||||
return false
|
||||
@@ -759,7 +930,7 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com
|
||||
a.mu.Lock()
|
||||
a.running[module] = delivery.Have{FP: w.Want.FP, Composed: w.Want.Composed, Phase: delivery.PhaseRunning}
|
||||
a.mu.Unlock()
|
||||
a.env.Haves.Write(ctx, Have{Module: module, FP: w.Want.FP, Composed: w.Want.Composed, Phase: string(delivery.PhaseRunning)})
|
||||
a.writeHave(ctx, Have{Module: module, FP: w.Want.FP, Composed: w.Want.Composed, Phase: string(delivery.PhaseRunning)})
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -773,7 +944,7 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
|
||||
// A data module is backed up before a new composition of it moves (ADR 0429 §14), unless the want says to go on
|
||||
// without one (ADR 0539 §16).
|
||||
if comp.C.Data && ranBefore && a.env.Backup != nil && w.Want.Backup != delivery.BackupSkip {
|
||||
a.env.Haves.Write(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseBackingUp)})
|
||||
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseBackingUp)})
|
||||
if got := a.env.Backup(ctx, module); !got.Done {
|
||||
a.env.Say(fmt.Sprintf("%s is held: its backup before %s failed: %s", module, fp, got.Words))
|
||||
a.mu.Lock()
|
||||
@@ -781,12 +952,12 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
|
||||
prev := a.running[module]
|
||||
a.mu.Unlock()
|
||||
// It keeps running what it has: the have says what it holds and why it waits.
|
||||
a.env.Haves.Write(ctx, Have{Module: module, FP: prev.FP, Composed: prev.Composed, Phase: string(delivery.PhaseHeld),
|
||||
a.writeHave(ctx, Have{Module: module, FP: prev.FP, Composed: prev.Composed, Phase: string(delivery.PhaseHeld),
|
||||
Detail: got.Words})
|
||||
return
|
||||
}
|
||||
}
|
||||
a.env.Haves.Write(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseApplying)})
|
||||
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseApplying)})
|
||||
res := a.env.Apply(ctx, comp.C, others)
|
||||
a.mu.Lock()
|
||||
what := fmt.Sprintf("%s at %s", module, fp)
|
||||
@@ -816,7 +987,7 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
|
||||
phase = delivery.PhaseFailed // still retried, and said as failed: no longer a moment
|
||||
}
|
||||
a.env.Say(fmt.Sprintf("applying %s failed because the bus did not answer (try %d), retried %s: %s", what, try, when, res.Detail))
|
||||
a.env.Haves.Write(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(phase),
|
||||
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(phase),
|
||||
Detail: fmt.Sprintf("retrying %s (try %d): %s", when, try, res.Detail)})
|
||||
return
|
||||
}
|
||||
@@ -825,17 +996,18 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
|
||||
a.attempt[module] = attempt{FP: fp, Composed: dig, WantRev: w.Rev, CompRev: comp.Rev}
|
||||
a.mu.Unlock()
|
||||
a.env.Say(fmt.Sprintf("applying %s failed: %s", what, res.Detail))
|
||||
a.env.Haves.Write(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed), Detail: res.Detail})
|
||||
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed), Detail: res.Detail})
|
||||
return
|
||||
}
|
||||
delete(a.attempt, module)
|
||||
delete(a.notComposed, module) // the apply says what it runs now
|
||||
a.running[module] = delivery.Have{FP: fp, Composed: dig, Phase: delivery.PhaseRunning}
|
||||
a.applied[module] = comp
|
||||
a.mu.Unlock()
|
||||
if a.env.Settled != nil {
|
||||
a.env.Settled(module)
|
||||
}
|
||||
a.env.Haves.Write(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseRunning), Detail: res.Detail})
|
||||
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseRunning), Detail: res.Detail})
|
||||
}
|
||||
|
||||
func (a *Agent) remove(ctx context.Context, module string, w wantNow) {
|
||||
@@ -925,3 +1097,68 @@ func short(s string) string {
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// waitsForYes judges a move the node composed for a module of policy record (ADR 0547 §3, ADR 0377 §3): one that
|
||||
// recreates a container or moves the module's data, or that cannot be judged, is not applied while the want names no yes
|
||||
// for this composition; the have says phase waiting-yes with the judgement, and the assigner asks a person. A composition
|
||||
// equal to what runs is no move. True when the module waits.
|
||||
func (a *Agent) waitsForYes(ctx context.Context, module string, w wantNow, comp compNow) bool {
|
||||
if module == NodeModule || a.env.Policy == nil || a.env.Policy(ctx, module) != catalogue.PolicyRecord || w.Want.Yes == comp.Dig {
|
||||
return false
|
||||
}
|
||||
a.mu.Lock()
|
||||
r, ran := a.running[module]
|
||||
if ran && r.Composed == comp.Dig && r.Phase != delivery.PhaseRemoved {
|
||||
a.mu.Unlock()
|
||||
return false
|
||||
}
|
||||
from, known := a.runningComposition(module)
|
||||
prior := ran || a.prior[module]
|
||||
a.mu.Unlock()
|
||||
if !ran && !known {
|
||||
// Nothing of it run since this agent's start, and no kept declaration: its last have says what runs (review of L2,
|
||||
// round 3). The same composition is no move; another one is a move whose source cannot be read, so it waits for
|
||||
// a yes rather than being taken as the module's first composition.
|
||||
if h, ok := a.env.Haves.Read(ctx, module); ok && h.Composed != "" {
|
||||
if h.Composed == comp.Dig {
|
||||
return false
|
||||
}
|
||||
prior = true
|
||||
}
|
||||
}
|
||||
var j delivery.Judged
|
||||
switch {
|
||||
case known:
|
||||
j = delivery.JudgeMove(&from, comp.C)
|
||||
case prior:
|
||||
j = delivery.Judged{Judgement: delivery.JudgedCannot, Words: "what " + module + " runs on this node cannot be read"}
|
||||
default:
|
||||
j = delivery.JudgeMove(nil, comp.C)
|
||||
}
|
||||
switch j.Judgement {
|
||||
case delivery.JudgedRecreates, delivery.JudgedMovesData, delivery.JudgedCannot:
|
||||
default:
|
||||
return false
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.sayOnce("yes:"+module+":"+comp.Dig, fmt.Sprintf("%s at %s waits for a person's yes: %s", module, w.Want.FP, j.Words))
|
||||
prev := a.running[module]
|
||||
a.mu.Unlock()
|
||||
a.writeHave(ctx, Have{Module: module, FP: prev.FP, Composed: prev.Composed, Phase: string(delivery.PhaseWaitingYes),
|
||||
Detail: j.Words, Judgement: string(j.Judgement), Waits: comp.Dig})
|
||||
return true
|
||||
}
|
||||
|
||||
// runningComposition is what the module runs now: the composition last applied, else its resources in the declaration
|
||||
// the node kept from before the switch. Called with a.mu held.
|
||||
func (a *Agent) runningComposition(module string) (delivery.Composed, bool) {
|
||||
if c, ok := a.applied[module]; ok && c.Err == nil {
|
||||
return c.C, true
|
||||
}
|
||||
if a.env.Kept != nil {
|
||||
if rs, ok := a.env.Kept()[module]; ok {
|
||||
return delivery.Composed{Node: a.env.Node, Module: module, Resources: rs}, true
|
||||
}
|
||||
}
|
||||
return delivery.Composed{}, false
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
@@ -326,6 +327,7 @@ func TestStartingOnWhatTheKeptDeclarationHoldsAppliesNothing(t *testing.T) {
|
||||
|
||||
// What the node-engine itself last applied is what runs after a restart, not the kept declaration.
|
||||
func TestStartingOnItsOwnHaveAppliesNothing(t *testing.T) {
|
||||
t.Run("says the have again", testStartingOnItsOwnHaveSaysTheHaveAgainSoItsHealthIsThisEngines)
|
||||
b := newBed(t)
|
||||
fp := fpOf("a")
|
||||
ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1"))
|
||||
@@ -345,7 +347,7 @@ func TestStartingOnItsOwnHaveAppliesNothing(t *testing.T) {
|
||||
// it and writes the health this engine judges. Not said, its key kept the health an older engine wrote: xorg's have on
|
||||
// g14 and shanks said unknown from before the engine said a module with nothing to judge ok, and the assigner held every
|
||||
// module requiring x11-display back for it (2026-10-25).
|
||||
func TestStartingOnItsOwnHaveSaysTheHaveAgainSoItsHealthIsThisEngines(t *testing.T) {
|
||||
func testStartingOnItsOwnHaveSaysTheHaveAgainSoItsHealthIsThisEngines(t *testing.T) {
|
||||
b := newBed(t)
|
||||
fp := fpOf("a")
|
||||
ck, craw, dig := composition(t, "xorg", fp, false, file("xorg", "x", "1"))
|
||||
@@ -545,6 +547,7 @@ func TestABackupDoneIsFollowedByTheApplyAndANewModuleIsNotBackedUp(t *testing.T)
|
||||
|
||||
// R6-A AC4 at the agent: a removed want removes that module only, and its have is written removed.
|
||||
func TestARemovedWantRemovesThatModuleOnly(t *testing.T) {
|
||||
t.Run("its composition holds nothing back", testARemovedModulesCompositionHoldsNothingBack)
|
||||
b := newBed(t)
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
@@ -598,6 +601,7 @@ func TestTheNodesOwnCompositionComesBeforeAnyModules(t *testing.T) {
|
||||
// R6-A AC9: a composition signed by either trusted key is applied; one signed by another key, one changed after it was
|
||||
// signed and one unsigned are refused and nothing of them is applied (ADR 0539 §3).
|
||||
func TestOnlyACompositionSignedByATrustedKeyIsApplied(t *testing.T) {
|
||||
t.Run("with no trusted key", testWithNoTrustedKeyNothingIsApplied)
|
||||
b := newBed(t)
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
@@ -648,7 +652,7 @@ func TestOnlyACompositionSignedByATrustedKeyIsApplied(t *testing.T) {
|
||||
}
|
||||
|
||||
// With no key trusted, nothing is applied: a node that does not know its signers fails closed.
|
||||
func TestWithNoTrustedKeyNothingIsApplied(t *testing.T) {
|
||||
func testWithNoTrustedKeyNothingIsApplied(t *testing.T) {
|
||||
b := newBed(t)
|
||||
b.env.Trusted = nil
|
||||
stop := b.run()
|
||||
@@ -787,7 +791,7 @@ func TestTheNodesOwnApplyHoldsOutAModuleThatMovedToItsOwnComposition(t *testing.
|
||||
|
||||
// A module the assigner removed holds nothing back, its compositions still in the bucket or not: its resources are its
|
||||
// removal's to take (issue 1945).
|
||||
func TestARemovedModulesCompositionHoldsNothingBack(t *testing.T) {
|
||||
func testARemovedModulesCompositionHoldsNothingBack(t *testing.T) {
|
||||
b := newBed(t)
|
||||
fp := fpOf("b")
|
||||
rk, rraw, _ := composition(t, "gone", fp, false, file("gone", "x", "1"))
|
||||
@@ -952,3 +956,148 @@ func TestAModuleRunningFromTheKeptDeclarationKeepsItsFilledPaths(t *testing.T) {
|
||||
t.Fatalf("what runs from the kept declaration: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1126 (ADR 0547 §2, §3): with the node's own compositions, unsigned, and wants naming the build alone, the
|
||||
// composition composed for the want's build is applied, and only when it differs from what runs; for a module of
|
||||
// policy record a move that recreates a container waits in phase waiting-yes with its judgement until the want names
|
||||
// the yes for that composition, and is then applied.
|
||||
func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(t *testing.T) {
|
||||
b := newBed(t)
|
||||
b.env.SelfComposed, b.env.Trusted = true, nil
|
||||
b.env.Policy = func(_ context.Context, m string) string {
|
||||
if m == "db" || m == "rec" {
|
||||
return catalogue.PolicyRecord
|
||||
}
|
||||
return catalogue.PolicyRoll
|
||||
}
|
||||
stop := b.run()
|
||||
defer stop()
|
||||
unsigned := func(module, fp string, rs ...json.RawMessage) (string, []byte, string) {
|
||||
c := delivery.Composed{Node: node, Module: module, FP: fp, Resources: rs}
|
||||
key, _ := delivery.ComposedKey(node, module, fp)
|
||||
raw, err := delivery.EncodeComposed(c)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dig, _ := delivery.ComposedDigestOf(c)
|
||||
return key, raw, dig
|
||||
}
|
||||
want := func(module, fp, yes string) {
|
||||
key, _ := delivery.WantKey(node, module)
|
||||
raw, err := delivery.EncodeWant(delivery.Want{FP: fp, Yes: yes, Self: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b.wants.put(key, raw)
|
||||
}
|
||||
|
||||
fp := fpOf("a")
|
||||
pw, _ := json.Marshal(map[string]any{"id": "web.pw", "type": "file", "path": "/tmp/r6/web/pw", "sealed": "1", "module": "web"})
|
||||
ck, craw, dig := unsigned("web", fp, file("web", "x", "1"), pw)
|
||||
b.comps.put(ck, craw)
|
||||
want("web", fp, "")
|
||||
eventually(t, "the node's own composition was not applied", func() bool { return len(b.appliedList()) == 1 })
|
||||
eventually(t, "the have does not name the composed digest", func() bool { return b.haves.last("web").Composed == dig })
|
||||
// The same composition handed again is no move.
|
||||
b.comps.put(ck, craw)
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
// The node's own composition carries its sealed values in its key, as a module's have does, so a restart reuses its
|
||||
// core boxes and applies nothing again (review of L2, round 3, finding 1).
|
||||
box, _ := json.Marshal(map[string]any{"id": "core-box", "type": "file", "path": "/tmp/r6/core-box", "sealed": "core"})
|
||||
own := delivery.Composed{Node: node, Resources: []json.RawMessage{box}}
|
||||
nk, _ := delivery.ComposedNodeKey(node)
|
||||
nraw, _ := delivery.EncodeComposed(own)
|
||||
ndig, _ := delivery.ComposedDigestOf(own)
|
||||
b.comps.put(nk, nraw)
|
||||
nwk, _ := delivery.WantNodeKey(node)
|
||||
nwraw, _ := delivery.EncodeNodeWant(delivery.NodeWant{Self: true})
|
||||
b.wants.put(nwk, nwraw)
|
||||
eventually(t, "the node's own key does not carry its sealed values", func() bool {
|
||||
h := b.haves.last(NodeModule)
|
||||
return h.Composed == ndig && len(h.Sealed) == 1 && h.Sealed[0] == "core"
|
||||
})
|
||||
// The node cannot compose (a stand-in refused): each wanted module's have says so, failed, naming why, and still
|
||||
// names what runs; composed again, the have is what it runs. web was applied by this agent; idle was not since the
|
||||
// agent's start (the switch and every delivery of the node-engine restart it), so what runs is what its have said.
|
||||
idleFP, idleDig := fpOf("c"), "sha256:"+strings.Repeat("9", 64)
|
||||
b.haves.Write(context.Background(), Have{Module: "idle", FP: idleFP, Composed: idleDig, Phase: string(delivery.PhaseRunning),
|
||||
Sealed: []string{`{"box":"idle"}`}})
|
||||
want("idle", idleFP, "")
|
||||
eventually(t, "idle's want was not taken", func() bool {
|
||||
b.ag.mu.Lock()
|
||||
defer b.ag.mu.Unlock()
|
||||
return b.ag.wants["idle"].Present
|
||||
})
|
||||
if h := b.haves.last("web"); len(h.Sealed) != 1 || h.Sealed[0] != "1" {
|
||||
t.Errorf("web's running have does not carry its sealed value: %+v", h)
|
||||
}
|
||||
b.ag.NotComposed(context.Background(), "a stand-in for web's password")
|
||||
for _, row := range []struct {
|
||||
module, fp, composed, sealed string
|
||||
}{{"web", fp, dig, "1"}, {"idle", idleFP, idleDig, `{"box":"idle"}`}} {
|
||||
h := b.haves.last(row.module)
|
||||
if h.Phase != string(delivery.PhaseFailed) || h.FP != row.fp || h.Composed != row.composed ||
|
||||
len(h.Sealed) != 1 || h.Sealed[0] != row.sealed || !strings.Contains(h.Detail, "web's password") {
|
||||
t.Errorf("%s's have while the node cannot compose: %+v", row.module, h)
|
||||
}
|
||||
}
|
||||
b.ag.NotComposed(context.Background(), "")
|
||||
for _, row := range []struct{ module, composed string }{{"web", dig}, {"idle", idleDig}} {
|
||||
if h := b.haves.last(row.module); h.Phase != string(delivery.PhaseRunning) || h.Composed != row.composed || len(h.Sealed) != 1 {
|
||||
t.Errorf("%s's have once the node composes again: %+v", row.module, h)
|
||||
}
|
||||
}
|
||||
// A revision at the same build (a config value changed) is applied.
|
||||
_, craw2, dig2 := unsigned("web", fp, file("web", "x", "2"))
|
||||
b.comps.put(ck, craw2)
|
||||
eventually(t, "the revision was not applied", func() bool { return len(b.appliedList()) == 3 && b.haves.last("web").Composed == dig2 })
|
||||
|
||||
// A record module: its first composition is no move; a new image recreates its container and waits for a yes.
|
||||
container := func(image string) json.RawMessage {
|
||||
module := strings.SplitN(image, "@", 2)[0]
|
||||
raw, _ := json.Marshal(map[string]any{"id": module + ".server", "type": "container", "module": module, "name": module, "image": image})
|
||||
return raw
|
||||
}
|
||||
dk, draw, _ := unsigned("db", fp, container("db@sha256:"+strings.Repeat("1", 64)))
|
||||
b.comps.put(dk, draw)
|
||||
want("db", fp, "")
|
||||
eventually(t, "the record module's first composition was not applied", func() bool { return len(b.appliedList()) == 4 })
|
||||
fpB := fpOf("b")
|
||||
dk2, draw2, ddig2 := unsigned("db", fpB, container("db@sha256:"+strings.Repeat("2", 64)))
|
||||
// rec is a record module this agent has run nothing of since its start (the switch and every delivery of the
|
||||
// node-engine restart it): its have alone says what runs. Its move waits for a yes as db's does, and its have keeps
|
||||
// what runs (fingerprint, digest, sealed values) rather than naming nothing (review of L2, round 3).
|
||||
recFP, recDig := fpOf("d"), "sha256:"+strings.Repeat("8", 64)
|
||||
b.haves.Write(context.Background(), Have{Module: "rec", FP: recFP, Composed: recDig, Phase: string(delivery.PhaseRunning),
|
||||
Sealed: []string{`{"box":"rec"}`}})
|
||||
rk, rraw, rdig := unsigned("rec", fpB, container("rec@sha256:"+strings.Repeat("3", 64)))
|
||||
b.comps.put(dk2, draw2)
|
||||
b.comps.put(rk, rraw)
|
||||
want("db", fpB, "")
|
||||
want("rec", fpB, "")
|
||||
applied := len(b.appliedList())
|
||||
for _, row := range []struct {
|
||||
module, fp, composed, waits string
|
||||
sealed []string
|
||||
}{{"db", fp, "", ddig2, nil}, {"rec", recFP, recDig, rdig, []string{`{"box":"rec"}`}}} {
|
||||
eventually(t, row.module+"'s move did not wait for a yes", func() bool {
|
||||
h := b.haves.last(row.module)
|
||||
return h.Phase == string(delivery.PhaseWaitingYes) && h.Waits == row.waits
|
||||
})
|
||||
h := b.haves.last(row.module)
|
||||
if h.FP != row.fp || (row.composed != "" && h.Composed != row.composed) || h.Composed == "" ||
|
||||
(row.sealed != nil && !slices.Equal(h.Sealed, row.sealed)) {
|
||||
t.Errorf("%s's have waiting for a yes does not keep what runs: %+v", row.module, h)
|
||||
}
|
||||
}
|
||||
if got := b.appliedList(); len(got) != applied {
|
||||
t.Fatalf("a move waiting for a yes was applied: %v", got)
|
||||
}
|
||||
want("db", fpB, ddig2)
|
||||
want("rec", fpB, rdig)
|
||||
eventually(t, "the yes did not apply the move", func() bool {
|
||||
h, r := b.haves.last("db"), b.haves.last("rec")
|
||||
return len(b.appliedList()) == applied+2 && h.Phase == string(delivery.PhaseRunning) && h.FP == fpB &&
|
||||
r.Phase == string(delivery.PhaseRunning) && r.FP == fpB && r.Composed == rdig
|
||||
})
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -94,6 +95,14 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
|
||||
if r.Schedule != "" {
|
||||
unit = r.Name + ".timer"
|
||||
}
|
||||
// The yielding slice is checked even then: a slice file removed by hand would leave the build unbound.
|
||||
if wrote, err := placeYieldSlice(r); err != nil {
|
||||
return out, err
|
||||
} else if wrote {
|
||||
if _, err := run(ctx, "systemctl", "daemon-reload"); err != nil {
|
||||
return out, err
|
||||
}
|
||||
}
|
||||
if active, err := run(ctx, "systemctl", "is-active", "--quiet", unit); err == nil {
|
||||
_ = active
|
||||
return out, nil
|
||||
@@ -121,7 +130,12 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
|
||||
}
|
||||
// Replaced rather than merged: the bundle is the whole of what it runs, and files left from a
|
||||
// previous version would be loaded by a runtime that walks a directory.
|
||||
if err := os.RemoveAll(at); err != nil {
|
||||
// **Retired, never deleted under its process** (novox/hq issue 734): the old process runs until
|
||||
// the restart below, from this directory and in it. Deleted here, every command it started
|
||||
// before the restart failed with "getwd: no such file or directory" — a builder's toolchain probe
|
||||
// refused three checks on 2026-10-26. Renamed aside, its process keeps a directory that exists,
|
||||
// and the build is deleted once nothing runs from it, as a judged build's is.
|
||||
if err := witness.Retire(daemonRoot, r.Name, at, time.Now()); err != nil {
|
||||
return out, err
|
||||
}
|
||||
} else {
|
||||
@@ -189,6 +203,9 @@ func applyProcess(ctx context.Context, r *declaration.Process, run Runner,
|
||||
if err := os.WriteFile(unit, []byte(unitFor(r)), 0o644); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if _, err := placeYieldSlice(r); err != nil {
|
||||
return out, err
|
||||
}
|
||||
if _, err := run(ctx, "systemctl", "daemon-reload"); err != nil {
|
||||
return out, err
|
||||
}
|
||||
@@ -277,6 +294,7 @@ func unitFor(r *declaration.Process) string {
|
||||
if r.User != "" {
|
||||
fmt.Fprintf(&b, "User=%s\n", r.User)
|
||||
}
|
||||
writeYields(&b, r.Yields)
|
||||
fmt.Fprintf(&b, "ExecStart=%s\n", strings.Join(runFrom(r), " "))
|
||||
if r.Schedule != "" || r.RunOnce {
|
||||
// Started by its timer, or once by the host, and expected to finish. Restarting it would have it run
|
||||
@@ -293,6 +311,78 @@ func unitFor(r *declaration.Process) string {
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// machineCPUs is how many processors this machine has, for a share of them in a quota. A variable only so a test can
|
||||
// fix it.
|
||||
var machineCPUs = runtime.NumCPU
|
||||
|
||||
// writeYields writes how a process gives way to everything else on its machine (novox/hq ADR 0553, issue 1806): a
|
||||
// build seat's holder, whose builds must never slow the operator's desktop.
|
||||
//
|
||||
// **Into the node's one yielding slice**, where the CPU weight and the quota are (yieldSlice): every yielding process
|
||||
// of the machine shares them, so three build modules take one share together, never one each. **Per process**, what a
|
||||
// slice cannot say: Nice=19 for every process it starts, and the lowest best-effort IO level. The IO level binds only
|
||||
// where the disk's scheduler honours it (bfq); under `none`, the usual one for NVMe, it does nothing. Not the idle IO
|
||||
// class: a disk that is never idle would starve a build for good.
|
||||
//
|
||||
// The share is written into the unit as a comment, so a changed share changes the unit, the daemon's identity, and the
|
||||
// slice is written again.
|
||||
func writeYields(b *strings.Builder, y *declaration.Yields) {
|
||||
if y == nil {
|
||||
return
|
||||
}
|
||||
fmt.Fprintf(b, "# Yields: %s, %d%% of the machine shared in %s\n", y.Priority, y.CPUShare, declaration.YieldSlice)
|
||||
fmt.Fprintf(b, "Slice=%s\n", declaration.YieldSlice)
|
||||
if y.Priority == declaration.YieldIdle {
|
||||
b.WriteString("Nice=19\nIOSchedulingClass=best-effort\nIOSchedulingPriority=7\n")
|
||||
}
|
||||
}
|
||||
|
||||
// yieldSlice is the node's one yielding slice (declaration.YieldSlice) for a process that yields: idle CPU weight and
|
||||
// IO weight 1 for an idle priority, and CPUQuota for its share of the processors this machine has (systemd counts a
|
||||
// quota in percent of one processor, so half of 24 is 1200%), at least one processor. At the top of the hierarchy, so
|
||||
// its weight and quota bind against the operator's user slice and the system's.
|
||||
//
|
||||
// The IO weight binds only where the disk's scheduler honours it (bfq, or the io.cost controller); under `none` it does
|
||||
// nothing, and the CPU bound is what holds.
|
||||
//
|
||||
// Every yielding process of a machine carries the node's one share (the composer's, ADR 0553 §3), so whichever writes
|
||||
// it writes the same file. Left in place when nothing yields any more: an empty slice binds nothing.
|
||||
func yieldSlice(y *declaration.Yields) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("# Generated by the mesh. Do not edit — this file is replaced whenever a yielding\n")
|
||||
b.WriteString("# process is declared again (novox/hq ADR 0553).\n")
|
||||
b.WriteString("[Unit]\nDescription=The mesh's yielding processes: builds, at idle priority under one share\n\n")
|
||||
b.WriteString("[Slice]\n")
|
||||
if y.Priority == declaration.YieldIdle {
|
||||
b.WriteString("CPUWeight=idle\nIOWeight=1\n")
|
||||
}
|
||||
if y.CPUShare > 0 {
|
||||
quota := y.CPUShare * machineCPUs()
|
||||
if quota < 100 {
|
||||
quota = 100
|
||||
}
|
||||
fmt.Fprintf(&b, "CPUQuota=%d%%\n", quota)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// placeYieldSlice writes the yielding slice for a process that yields, when what is there differs, and says whether it
|
||||
// wrote: the caller reloads the service manager before it starts anything.
|
||||
func placeYieldSlice(r *declaration.Process) (bool, error) {
|
||||
if r.Yields == nil {
|
||||
return false, nil
|
||||
}
|
||||
path := filepath.Join(unitDir, declaration.YieldSlice)
|
||||
want := yieldSlice(r.Yields)
|
||||
if have, err := os.ReadFile(path); err == nil && string(have) == want {
|
||||
return false, nil
|
||||
}
|
||||
if err := os.WriteFile(path, []byte(want), 0o644); err != nil {
|
||||
return false, fmt.Errorf("the yielding slice %s: %w", path, err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// timerFor is the cadence a scheduled process runs on.
|
||||
//
|
||||
// **The mesh's cron expression, handed to the machine's own timer.** The host already parses and
|
||||
|
||||
@@ -21,10 +21,16 @@ func aProcess() *declaration.Process {
|
||||
}
|
||||
}
|
||||
|
||||
// The unit the mesh writes says what it runs, where, and that it comes back.
|
||||
// The unit the mesh writes says what it runs, where, and that it comes back — and that it is the
|
||||
// mesh's: **generated whole and saying so**, because an edit that survives until the next declaration
|
||||
// and then vanishes is worse than one that is refused. The long-running mode stays up and is not a step.
|
||||
func TestTheUnitRunsWhatTheDaemonSaid(t *testing.T) {
|
||||
unit := unitFor(aProcess())
|
||||
if !strings.HasPrefix(unit, "#") {
|
||||
t.Fatalf("the unit does not open saying it is generated:\n%s", unit)
|
||||
}
|
||||
for _, want := range []string{
|
||||
"Do not edit",
|
||||
"ExecStart=node index.js",
|
||||
"WorkingDirectory=/var/lib/mesh/daemons/greeter",
|
||||
"Restart=always",
|
||||
@@ -34,14 +40,8 @@ func TestTheUnitRunsWhatTheDaemonSaid(t *testing.T) {
|
||||
t.Fatalf("the unit does not say %q:\n%s", want, unit)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// **Generated whole and saying so.** Every managed file on a machine carries this, because an edit
|
||||
// that survives until the next declaration and then vanishes is worse than one that is refused.
|
||||
func TestTheUnitSaysItIsTheMeshs(t *testing.T) {
|
||||
unit := unitFor(aProcess())
|
||||
if !strings.HasPrefix(unit, "#") || !strings.Contains(unit, "Do not edit") {
|
||||
t.Fatalf("the unit does not say it is generated:\n%s", unit)
|
||||
if strings.Contains(unit, "Type=oneshot") {
|
||||
t.Fatalf("a process that should stay up is declared a step:\n%s", unit)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,6 +83,60 @@ func TestAProcessRunsAsWhoItSaid(t *testing.T) {
|
||||
if strings.Contains(unitFor(aProcess()), "User=") {
|
||||
t.Fatalf("a process that named no user had one written for it:\n%s", unitFor(aProcess()))
|
||||
}
|
||||
|
||||
// A process that yields (ADR 0553) runs in the node's one yielding slice at nice 19; the slice holds the idle weight
|
||||
// and the quota for its share of the machine's processors; one that does not yield has none of it.
|
||||
defer func(n func() int) { machineCPUs = n }(machineCPUs)
|
||||
for _, c := range []struct {
|
||||
cpus int
|
||||
y *declaration.Yields
|
||||
unit, slice, not []string
|
||||
}{
|
||||
{24, &declaration.Yields{Priority: declaration.YieldIdle, CPUShare: 50},
|
||||
[]string{"Slice=meshyield.slice\n", "Nice=19\n", "IOSchedulingPriority=7\n", "50%"},
|
||||
[]string{"CPUWeight=idle\n", "IOWeight=1\n", "CPUQuota=1200%\n"}, []string{"CPUQuota=", "CPUWeight="}},
|
||||
{1, &declaration.Yields{CPUShare: 25}, []string{"Slice=meshyield.slice\n"}, []string{"CPUQuota=100%\n"},
|
||||
[]string{"Nice="}},
|
||||
{24, nil, nil, nil, []string{"Slice=", "Nice=", "CPUQuota="}},
|
||||
} {
|
||||
machineCPUs = func() int { return c.cpus }
|
||||
p := aProcess()
|
||||
p.Yields = c.y
|
||||
unit := unitFor(p)
|
||||
for _, w := range c.unit {
|
||||
if !strings.Contains(unit, w) {
|
||||
t.Errorf("yields %+v: the unit does not say %q:\n%s", c.y, w, unit)
|
||||
}
|
||||
}
|
||||
for _, n := range c.not {
|
||||
if strings.Contains(unit, n) {
|
||||
t.Errorf("yields %+v: the unit says %q:\n%s", c.y, n, unit)
|
||||
}
|
||||
}
|
||||
if c.y == nil {
|
||||
continue
|
||||
}
|
||||
slice := yieldSlice(c.y)
|
||||
for _, w := range c.slice {
|
||||
if !strings.Contains(slice, w) {
|
||||
t.Errorf("yields %+v on %d processors: the slice does not say %q:\n%s", c.y, c.cpus, w, slice)
|
||||
}
|
||||
}
|
||||
}
|
||||
// The slice is placed beside the units, once, and again only when it differs.
|
||||
was := unitDir
|
||||
unitDir = t.TempDir()
|
||||
defer func() { unitDir = was }()
|
||||
p := aProcess()
|
||||
p.Yields = &declaration.Yields{Priority: declaration.YieldIdle, CPUShare: 50}
|
||||
for i, want := range []bool{true, false} {
|
||||
if wrote, err := placeYieldSlice(p); err != nil || wrote != want {
|
||||
t.Errorf("placing %d: wrote %v (%v), want %v", i, wrote, err, want)
|
||||
}
|
||||
}
|
||||
if b, err := os.ReadFile(filepath.Join(unitDir, "meshyield.slice")); err != nil || !strings.Contains(string(b), "[Slice]") {
|
||||
t.Errorf("the slice file: %q %v", b, err)
|
||||
}
|
||||
}
|
||||
|
||||
// **Three modes, one kind.** A scheduled process is a timer plus a unit that finishes, not a unit
|
||||
@@ -124,17 +178,6 @@ func TestACronBecomesATimersCalendar(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// And the long-running mode is unchanged by any of it: it stays up and comes back.
|
||||
func TestAProcessThatStaysUpIsStillRestartedWhenItExits(t *testing.T) {
|
||||
unit := unitFor(aProcess())
|
||||
if !strings.Contains(unit, "Restart=always") {
|
||||
t.Fatalf("a process that should stay up is not restarted when it exits:\n%s", unit)
|
||||
}
|
||||
if strings.Contains(unit, "Type=oneshot") {
|
||||
t.Fatalf("a process that should stay up is declared a step:\n%s", unit)
|
||||
}
|
||||
}
|
||||
|
||||
// **A unit file reinterprets a value in three ways nothing else does**, and a module's environment
|
||||
// routinely contains all three — a generated password is arbitrary bytes.
|
||||
func TestAnEnvironmentValueMeansWhatTheDeclarationSaid(t *testing.T) {
|
||||
|
||||
@@ -3,7 +3,9 @@ package apply
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -103,3 +105,79 @@ func TestARolledBackBuildIsNotPlacedAgain(t *testing.T) {
|
||||
t.Fatalf("the apply does not say why it kept the build before: %+v", o)
|
||||
}
|
||||
}
|
||||
|
||||
// **A process nobody judges keeps the directory it works in until its restart** (novox/hq issue 734).
|
||||
// On 2026-10-26 three checks on one builder were refused in the second a new build of it arrived: the
|
||||
// applier deleted the running build's directory before the restart, and the `go version` the old
|
||||
// process started in it failed with "getwd: no such file or directory". Here a process works in the
|
||||
// build's directory without running from it, as that `go version` did: the apply of a new build leaves
|
||||
// its directory in place, and the retired build goes once nothing works in it.
|
||||
func TestAProcessNobodyJudgesKeepsItsDirectoryUntilItsRestart(t *testing.T) {
|
||||
if _, err := os.Stat("/proc/self/cwd"); err != nil {
|
||||
t.Skip("which process works in a build is read from /proc")
|
||||
}
|
||||
units, bundles := t.TempDir(), t.TempDir()
|
||||
wasUnits, wasBundles := unitDir, daemonRoot
|
||||
unitDir, daemonRoot = units, bundles
|
||||
t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
|
||||
run := func(ctx context.Context, name string, args ...string) (string, error) { return "", nil }
|
||||
|
||||
oldBody, oldDigest := anArchive(t, map[string]string{"mesh-build-go": "old\n"})
|
||||
newBody, newDigest := anArchive(t, map[string]string{"mesh-build-go": "new\n"})
|
||||
builder := func(body []byte, digest string) string {
|
||||
return `{"id":"mesh-build-go.builder","type":"process","name":"mesh-build-go","source":"` + serving(t, body) +
|
||||
`","digest":"` + digest + `","run":["./mesh-build-go"]}`
|
||||
}
|
||||
_, state, err := Apply(context.Background(), archHost(t), declare(t, builder(oldBody, oldDigest)), store.State{},
|
||||
store.OriginDeclared, run, nil, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sleep, err := exec.LookPath("sleep")
|
||||
if err != nil {
|
||||
t.Skip("no sleep to run")
|
||||
}
|
||||
working := exec.Command(sleep, "60")
|
||||
working.Dir = filepath.Join(bundles, "mesh-build-go")
|
||||
if err := working.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = working.Process.Kill(); _ = working.Wait() })
|
||||
cwd := filepath.Join("/proc", strconv.Itoa(working.Process.Pid), "cwd")
|
||||
for deadline := time.Now().Add(5 * time.Second); ; {
|
||||
if at, err := os.Readlink(cwd); err == nil && at == working.Dir {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("PID %d does not work in %s within 5s", working.Process.Pid, working.Dir)
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
if _, _, err := Apply(context.Background(), archHost(t), declare(t, builder(newBody, newDigest)), state,
|
||||
store.OriginDeclared, run, nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := os.ReadFile(filepath.Join(bundles, "mesh-build-go", "mesh-build-go")); string(got) != "new\n" {
|
||||
t.Fatalf("the new build is not where the unit runs it from: %q", got)
|
||||
}
|
||||
at, err := os.Readlink(cwd)
|
||||
if err != nil || strings.HasSuffix(at, " (deleted)") {
|
||||
t.Fatalf("the directory a running process works in was deleted before its restart: %q (%v)", at, err)
|
||||
}
|
||||
if got, _ := os.ReadFile(filepath.Join(at, "mesh-build-go")); string(got) != "old\n" {
|
||||
t.Fatalf("the process's directory no longer holds its own build: %q", got)
|
||||
}
|
||||
if len(witness.Trials(bundles)) != 0 {
|
||||
t.Fatal("a process nobody judges went on trial")
|
||||
}
|
||||
|
||||
// Its process gone, the next look deletes the retired build and keeps nothing about the process.
|
||||
_ = working.Process.Kill()
|
||||
_ = working.Wait()
|
||||
witness.SweepAll(bundles)
|
||||
if _, err := os.Stat(witness.Dir(bundles, "mesh-build-go")); !os.IsNotExist(err) {
|
||||
t.Fatalf("a retired build nothing works in was kept: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
+50
-2
@@ -10,6 +10,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
@@ -92,7 +93,32 @@ func (b OverNATS) FileObject(ctx context.Context, name string, limit int64) ([]b
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stream := declaration.FileStreamPrefix + bucket
|
||||
return objectByName(ctx, js, bucket, declaration.FileStreamPrefix+bucket, name, limit)
|
||||
}
|
||||
|
||||
// GrantsObject reads one object of the grants part's bucket of sealed user lists by its name (novox/hq ADR 0547 clause 7),
|
||||
// with the same subjects and nothing more as FileObject: the bus node is granted exactly those (mesh-sdk
|
||||
// buscompose.GrantsUserListFollows).
|
||||
func (b OverNATS) GrantsObject(ctx context.Context, name string, limit int64) ([]byte, error) {
|
||||
js := b.JS
|
||||
wait := 10 * time.Second
|
||||
if at, ok := ctx.Deadline(); ok {
|
||||
if wait = time.Until(at); wait <= 0 {
|
||||
return nil, context.DeadlineExceeded
|
||||
}
|
||||
}
|
||||
if b.Conn != nil {
|
||||
bounded, err := b.Conn.JetStream(nats.MaxWait(wait))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
js = bounded
|
||||
}
|
||||
return objectByName(ctx, js, delivery.GrantsUserListObjects, delivery.GrantsUserListObjectStream, name, limit)
|
||||
}
|
||||
|
||||
// objectByName is one object of bucket (its stream stream) by its name, no larger than limit, verified against its digest.
|
||||
func objectByName(ctx context.Context, js nats.JetStreamContext, bucket, stream, name string, limit int64) ([]byte, error) {
|
||||
meta, err := js.GetLastMsg(stream, "$O."+bucket+".M."+base64.URLEncoding.EncodeToString([]byte(name)),
|
||||
nats.DirectGet(), nats.Context(ctx))
|
||||
if err != nil {
|
||||
@@ -109,7 +135,7 @@ func (b OverNATS) FileObject(ctx context.Context, name string, limit int64) ([]b
|
||||
return nil, fmt.Errorf("the object %s was deleted from the bucket %s", name, bucket)
|
||||
}
|
||||
if info.Size > uint64(limit) {
|
||||
return nil, fmt.Errorf("the object is larger than the %d bytes the declaration names", limit)
|
||||
return nil, fmt.Errorf("the object is larger than the %d bytes allowed", limit)
|
||||
}
|
||||
if info.Size == 0 {
|
||||
return []byte{}, nil
|
||||
@@ -142,6 +168,28 @@ func (l *natsLink) FileObject(ctx context.Context, name string, limit int64) ([]
|
||||
return OverNATS{Conn: l.conn, JS: l.js, Node: l.node}.FileObject(ctx, name, limit)
|
||||
}
|
||||
|
||||
func (l *natsLink) GrantsObject(ctx context.Context, name string, limit int64) ([]byte, error) {
|
||||
return OverNATS{Conn: l.conn, JS: l.js, Node: l.node}.GrantsObject(ctx, name, limit)
|
||||
}
|
||||
|
||||
// GrantsObjects is a link that can read an object of the grants part's bucket of sealed user lists.
|
||||
type GrantsObjects interface {
|
||||
GrantsObject(ctx context.Context, name string, limit int64) ([]byte, error)
|
||||
}
|
||||
|
||||
// GrantsObject reads an object of the grants part's bucket by its name on the link open now.
|
||||
func (q *Session) GrantsObject(ctx context.Context, name string, limit int64) ([]byte, error) {
|
||||
q.init()
|
||||
q.mu.Lock()
|
||||
bus := q.bus
|
||||
q.mu.Unlock()
|
||||
g, ok := bus.(GrantsObjects)
|
||||
if bus == nil || !ok {
|
||||
return nil, ErrNoLink
|
||||
}
|
||||
return g.GrantsObject(ctx, name, limit)
|
||||
}
|
||||
|
||||
var (
|
||||
_ FilesBus = OverNATS{}
|
||||
_ FilesBus = (*natsLink)(nil)
|
||||
|
||||
+224
-18
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -37,12 +38,20 @@ type haveState struct {
|
||||
mu sync.Mutex
|
||||
modules map[string]ModuleFact
|
||||
health map[string]delivery.Health
|
||||
// words is, by module, what the last look found of the resource that made its health failed or waits: the have says
|
||||
// it, so a failed health is never without words (issue 1496: "the node gave no words" hid a directory used as found).
|
||||
words map[string]string
|
||||
// looked says a look's health was taken: a running module it judges no resource of then has nothing that could be
|
||||
// down, and is ok (issue 1945).
|
||||
looked bool
|
||||
node *delivery.HaveNode
|
||||
// nodeComposed is the digest of the node's own composition it runs (ADR 0539 §9), kept beside the node's facts.
|
||||
nodeComposed string
|
||||
// nodeSealed is, on a node composing its own, each value its own composition carries sealed to its key (ADR 0547 §3,
|
||||
// issue 1126): kept in its key so a restart reuses them and applies nothing again.
|
||||
nodeSealed []string
|
||||
// nodeCandidates is the node's answer to each placement candidate group (novox/hq ADR 0559 §4), written into its key.
|
||||
nodeCandidates map[string]delivery.CandidateAnswer
|
||||
// written is the build of each module this node-engine wrote a have key for and that the declaration it applied
|
||||
// still holds; removed is each such module the latest declaration no longer holds, with the build its key last
|
||||
// said, until the key is written once with the phase removed (issue 1369, novox/hq ADR 0408 §4: never deleted).
|
||||
@@ -51,6 +60,8 @@ type haveState struct {
|
||||
// userList is what the bus node's engine last wrote of the bus's user list, by the module whose have says it (novox/hq
|
||||
// ADR 0547 §8); kept apart from the module's fact, which every apply replaces.
|
||||
userList map[string]delivery.UserListWritten
|
||||
// userListNoKey is each module whose user list was said to have no have key to ride on, so it is said once.
|
||||
userListNoKey map[string]bool
|
||||
// unjudged is each module whose composition moved (an apply began, or the fact names another build) and that no look
|
||||
// has judged since it stopped moving: its health is not yet judged, never the last look's of what ran before.
|
||||
unjudged map[string]bool
|
||||
@@ -106,10 +117,11 @@ func (s *haveState) owed() bool {
|
||||
// takeLook keeps a look's health: of a module still in motion nothing, since the look judged what is being replaced;
|
||||
// of every other module what the look found, which is then judged. The caller holds the lock.
|
||||
func (s *haveState) takeLook(h *Health) {
|
||||
next := moduleHealths(h)
|
||||
next, words := moduleHealths(h)
|
||||
for module, f := range s.modules {
|
||||
if inMotion(f.Phase) {
|
||||
delete(next, module)
|
||||
delete(words, module)
|
||||
if s.unjudged == nil {
|
||||
s.unjudged = map[string]bool{}
|
||||
}
|
||||
@@ -118,19 +130,32 @@ func (s *haveState) takeLook(h *Health) {
|
||||
}
|
||||
delete(s.unjudged, module)
|
||||
}
|
||||
s.health, s.looked = next, true
|
||||
s.health, s.words, s.looked = next, words, true
|
||||
}
|
||||
|
||||
// wordsOf is what a have says of its health when the apply said nothing: the words of the resource that made it failed
|
||||
// or waits, while it is so. The caller holds the lock.
|
||||
func (s *haveState) wordsOf(module string, health delivery.Health) string {
|
||||
if health != delivery.HealthFailed && health != delivery.HealthWaits {
|
||||
return ""
|
||||
}
|
||||
return s.words[module]
|
||||
}
|
||||
|
||||
// moduleHealths is the health of each module a look names, from the states of its resources: failed when any is
|
||||
// unhealthy, waits when any waits (waiting, starting, held), ok when all are healthy, unknown otherwise.
|
||||
func moduleHealths(h *Health) map[string]delivery.Health {
|
||||
out := map[string]delivery.Health{}
|
||||
// unhealthy, waits when any waits (waiting, starting, held), ok when all are healthy, unknown otherwise; and, for a
|
||||
// module failed or waiting, the words of the first resource that made it so. A directory used as found is not the
|
||||
// module's health: it says nothing of how the module runs, waits for the operator's hand-over and keeps its own
|
||||
// condition, raised from the same statement (issue 1496: on ace mesh-build-node's cache directory, there before the
|
||||
// mesh, made its have failed while its check answered healthy, and the assigner held its version on every node).
|
||||
func moduleHealths(h *Health) (map[string]delivery.Health, map[string]string) {
|
||||
out, words := map[string]delivery.Health{}, map[string]string{}
|
||||
if h == nil {
|
||||
return out
|
||||
return out, words
|
||||
}
|
||||
rank := map[string]int{}
|
||||
for _, r := range h.Resources {
|
||||
if r.Module == "" {
|
||||
if r.Module == "" || usedAsFound(r) {
|
||||
continue
|
||||
}
|
||||
var this int
|
||||
@@ -146,12 +171,32 @@ func moduleHealths(h *Health) map[string]delivery.Health {
|
||||
}
|
||||
if was, seen := rank[r.Module]; !seen || this > was {
|
||||
rank[r.Module] = this
|
||||
if this >= 2 {
|
||||
words[r.Module] = resourceWords(r)
|
||||
}
|
||||
}
|
||||
}
|
||||
for module, r := range rank {
|
||||
out[module] = [...]delivery.Health{delivery.HealthOK, delivery.HealthUnknown, delivery.HealthWaits, delivery.HealthFailed}[r]
|
||||
if r < 2 {
|
||||
delete(words, module)
|
||||
}
|
||||
}
|
||||
return out
|
||||
return out, words
|
||||
}
|
||||
|
||||
// usedAsFound says a resource of a statement is a directory used as found (issue 339).
|
||||
func usedAsFound(r ResourceHealth) bool {
|
||||
return r.Kind == KindDirectory && strings.HasPrefix(r.Reason, ReasonUsedAsFound)
|
||||
}
|
||||
|
||||
// resourceWords is one resource's state in the words a have carries: its id, its state and why.
|
||||
func resourceWords(r ResourceHealth) string {
|
||||
w := r.Resource + " " + r.State
|
||||
if why := strings.TrimSpace(r.Reason); why != "" {
|
||||
w += ": " + why
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// nodeFacts is the node's own facts of a report, each as the JSON the report carries it in.
|
||||
@@ -187,8 +232,7 @@ func (q *Session) noteApply(r Report) bool {
|
||||
if r.Health != nil {
|
||||
s.takeLook(r.Health)
|
||||
}
|
||||
n := nodeFacts(r)
|
||||
s.node = &n
|
||||
s.node = q.nodeFactsWithRecord(r, s.node)
|
||||
return true
|
||||
}
|
||||
s.mu.Lock()
|
||||
@@ -240,24 +284,45 @@ func (q *Session) noteApply(r Report) bool {
|
||||
if s.owed() {
|
||||
q.oweHealth()
|
||||
}
|
||||
n := nodeFacts(r)
|
||||
s.node = &n
|
||||
s.node = q.nodeFactsWithRecord(r, s.node)
|
||||
return true
|
||||
}
|
||||
|
||||
// nodeFactsWithRecord is the node's own facts of a report with its record (ADR 0555 §3): a report that says nothing of
|
||||
// the machine leaves the record last made standing, and one without a health statement keeps the contract last stated.
|
||||
func (q *Session) nodeFactsWithRecord(r Report, before *delivery.HaveNode) *delivery.HaveNode {
|
||||
n := nodeFacts(r)
|
||||
if q.Keys != nil {
|
||||
n.Record = recordOf(q.Membership.Node, r, q.Keys())
|
||||
}
|
||||
if before != nil && before.Record != nil {
|
||||
switch {
|
||||
case n.Record == nil:
|
||||
n.Record = before.Record
|
||||
case n.Record.Contract == 0:
|
||||
n.Record.Contract = before.Record.Contract
|
||||
}
|
||||
}
|
||||
return &n
|
||||
}
|
||||
|
||||
// noteHealth takes a look's health, and says whether any module's changed.
|
||||
func (q *Session) noteHealth(h Health) bool {
|
||||
s := &q.haves
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
before := map[string]delivery.Health{}
|
||||
said := func(module string, f ModuleFact) string {
|
||||
health := s.healthOf(module, f)
|
||||
return string(health) + "\x00" + s.wordsOf(module, health)
|
||||
}
|
||||
before := map[string]string{}
|
||||
for module, f := range s.modules {
|
||||
before[module] = s.healthOf(module, f)
|
||||
before[module] = said(module, f)
|
||||
}
|
||||
s.takeLook(&h)
|
||||
changed := false
|
||||
for module, f := range s.modules {
|
||||
if s.healthOf(module, f) != before[module] {
|
||||
if said(module, f) != before[module] {
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
@@ -283,7 +348,7 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
|
||||
s := &q.haves
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.modules == nil && s.node == nil && len(s.removed) == 0 {
|
||||
if s.modules == nil && s.node == nil && len(s.removed) == 0 && len(s.userList) == 0 {
|
||||
return
|
||||
}
|
||||
node := q.Membership.Node
|
||||
@@ -306,13 +371,20 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
|
||||
f := s.modules[module]
|
||||
h := delivery.Have{V: delivery.Version, FP: f.FP, Composed: f.Composed, Build: f.Build,
|
||||
Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown, Detail: f.Detail}
|
||||
if delivery.Phase(f.Phase) == delivery.PhaseWaitingYes {
|
||||
h.Judgement, h.Waits = delivery.Judgement(f.Judgement), f.Waits
|
||||
}
|
||||
switch {
|
||||
case f.Phase != "":
|
||||
h.Phase = delivery.Phase(f.Phase)
|
||||
case f.Failed:
|
||||
h.Phase = delivery.PhaseFailed
|
||||
}
|
||||
h.Sealed = f.Sealed
|
||||
h.Health = s.healthOf(module, f)
|
||||
if h.Detail == "" {
|
||||
h.Detail = s.wordsOf(module, h.Health)
|
||||
}
|
||||
if w, ok := s.userList[module]; ok {
|
||||
h.UserList = &w
|
||||
}
|
||||
@@ -340,12 +412,17 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
|
||||
if !q.writeRemoved(write, bucket, node) {
|
||||
return
|
||||
}
|
||||
if !q.writeUserListsAlone(write, bucket, node) {
|
||||
return
|
||||
}
|
||||
if s.node != nil {
|
||||
key, err := delivery.HaveNodeKey(node)
|
||||
if err == nil {
|
||||
var raw []byte
|
||||
facts := *s.node
|
||||
facts.Composed = s.nodeComposed
|
||||
facts.Sealed = s.nodeSealed
|
||||
facts.Candidates = s.nodeCandidates
|
||||
if raw, err = delivery.EncodeHaveNode(facts); err == nil {
|
||||
want := facts
|
||||
if !q.putHave(write, bucket, key, raw, func(current []byte) bool {
|
||||
@@ -455,6 +532,78 @@ func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string) b
|
||||
return true
|
||||
}
|
||||
|
||||
// writeUserListsAlone writes the user list of each module this node-engine holds no fact of (no apply here said its have)
|
||||
// into that module's key as the bucket holds it, keeping every other field the key says (novox/hq ADR 0547 §8). The have
|
||||
// of a module is one value with parts from more than one source: the apply's (fingerprint, digest, build, phase, health)
|
||||
// and the user-list follower's. Written only on the pass over the modules this engine applied, the follower's part was
|
||||
// lost whenever the bus module's apply was not this engine's to say (2026-10-26 on the bus node: the bus module had no
|
||||
// want and had left the declaration, so its key kept an older engine's value and the vault read no hashes in it). A key
|
||||
// that does not exist is not created: a have says a phase, and nothing here knows it; that is said once. False when the
|
||||
// pass should stop. The caller holds the lock of the haves.
|
||||
func (q *Session) writeUserListsAlone(ctx context.Context, b HaveBucket, node string) bool {
|
||||
s := &q.haves
|
||||
alone := make([]string, 0, len(s.userList))
|
||||
for module := range s.userList {
|
||||
_, held := s.modules[module]
|
||||
_, gone := s.removed[module]
|
||||
if !held && !gone {
|
||||
alone = append(alone, module)
|
||||
}
|
||||
}
|
||||
sort.Strings(alone)
|
||||
for _, module := range alone {
|
||||
key, err := delivery.HaveKey(node, module)
|
||||
if err != nil {
|
||||
q.sayHave(fmt.Sprintf("module %q has no key in the have bucket: %v", module, err))
|
||||
continue
|
||||
}
|
||||
if ctx.Err() != nil {
|
||||
q.sayHave(fmt.Sprintf("the time for writing the have bucket ran out before %s; the next apply or look writes it", key))
|
||||
return false
|
||||
}
|
||||
current, revision, err := b.Get(ctx, key)
|
||||
switch {
|
||||
case errors.Is(err, ErrHaveAbsent):
|
||||
if !s.userListNoKey[module] {
|
||||
if s.userListNoKey == nil {
|
||||
s.userListNoKey = map[string]bool{}
|
||||
}
|
||||
s.userListNoKey[module] = true
|
||||
q.sayHave(fmt.Sprintf("the bus's user list written for %s is in no have: %s does not exist and no apply on "+
|
||||
"this node says %s's phase, so it is written once one does", module, key, module))
|
||||
}
|
||||
continue
|
||||
case err != nil:
|
||||
q.sayHave(fmt.Sprintf("could not read %s in the have bucket, and wrote nothing more there: %v", key, err))
|
||||
q.haveFailed(err)
|
||||
return false
|
||||
}
|
||||
delete(s.userListNoKey, module)
|
||||
h, err := delivery.DecodeHave(current)
|
||||
if err != nil {
|
||||
q.sayHave(fmt.Sprintf("the bus's user list is not written in %s: its value cannot be read: %v", key, err))
|
||||
continue
|
||||
}
|
||||
w := s.userList[module]
|
||||
next := h
|
||||
next.UserList = &w
|
||||
if delivery.SameHave(h, next) {
|
||||
continue
|
||||
}
|
||||
raw, err := delivery.EncodeHave(next)
|
||||
if err != nil {
|
||||
q.sayHave(fmt.Sprintf("cannot encode %s: %v", key, err))
|
||||
continue
|
||||
}
|
||||
if err := b.Set(ctx, key, raw, revision); err != nil {
|
||||
q.sayHave(fmt.Sprintf("could not write %s in the have bucket; the next apply or look writes it again: %v", key, err))
|
||||
q.haveFailed(err)
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// putHave reads a key and writes it unless same says it already holds the value. False when the pass should stop: the
|
||||
// bucket failed, the error is said once, and the rest wait for the next pass.
|
||||
func (q *Session) putHave(ctx context.Context, b HaveBucket, key string, value []byte, same func([]byte) bool) bool {
|
||||
@@ -561,12 +710,14 @@ func (q *Session) NoteRemoved(ctx context.Context, module, build string) {
|
||||
q.writeHaves(ctx, bus)
|
||||
}
|
||||
|
||||
// NoteNode takes the digest of the node's own composition it runs, and writes the node's key where it differs.
|
||||
func (q *Session) NoteNode(ctx context.Context, composed string) {
|
||||
// NoteNode takes the digest of the node's own composition it runs and the values it carries sealed to the node's key
|
||||
// (nil for a composition the node did not make itself), and writes the node's key where it differs.
|
||||
func (q *Session) NoteNode(ctx context.Context, composed string, sealed []string) {
|
||||
q.init()
|
||||
s := &q.haves
|
||||
s.mu.Lock()
|
||||
s.nodeComposed = composed
|
||||
s.nodeSealed = sealed
|
||||
if s.node == nil {
|
||||
n := delivery.HaveNode{V: delivery.Version}
|
||||
s.node = &n
|
||||
@@ -578,6 +729,61 @@ func (q *Session) NoteNode(ctx context.Context, composed string) {
|
||||
q.writeHaves(ctx, bus)
|
||||
}
|
||||
|
||||
// NoteCandidates takes the node's answers to the placement candidates, by group (its shadow composer's, novox/hq ADR
|
||||
// 0559 §4), and writes the node's key where it differs; a group left out is no longer answered.
|
||||
func (q *Session) NoteCandidates(ctx context.Context, answers map[string]delivery.CandidateAnswer) {
|
||||
q.init()
|
||||
s := &q.haves
|
||||
s.mu.Lock()
|
||||
s.nodeCandidates = answers
|
||||
if len(answers) == 0 {
|
||||
s.nodeCandidates = nil
|
||||
}
|
||||
// Before the node's facts are first said, its key is not written for an answer alone: it would carry no record. The
|
||||
// answers ride with the first facts.
|
||||
known := s.node != nil
|
||||
s.mu.Unlock()
|
||||
if !known {
|
||||
return
|
||||
}
|
||||
q.mu.Lock()
|
||||
bus := q.bus
|
||||
q.mu.Unlock()
|
||||
q.writeHaves(ctx, bus)
|
||||
}
|
||||
|
||||
// ReadNodeHave is the node's own key as the bucket holds it, and whether it holds one: what the node's own composition
|
||||
// it last ran is and carries sealed, read after a restart (ADR 0547 §3).
|
||||
func (q *Session) ReadNodeHave(ctx context.Context) (delivery.HaveNode, bool) {
|
||||
q.init()
|
||||
q.mu.Lock()
|
||||
bus := q.bus
|
||||
q.mu.Unlock()
|
||||
hb, ok := bus.(HaveBus)
|
||||
if !ok || bus == nil {
|
||||
return delivery.HaveNode{}, false
|
||||
}
|
||||
key, err := delivery.HaveNodeKey(q.Membership.Node)
|
||||
if err != nil {
|
||||
return delivery.HaveNode{}, false
|
||||
}
|
||||
read, cancel := context.WithTimeout(ctx, q.timeout())
|
||||
defer cancel()
|
||||
bucket, err := hb.HaveBind(read)
|
||||
if err != nil {
|
||||
return delivery.HaveNode{}, false
|
||||
}
|
||||
value, _, err := bucket.Get(read, key)
|
||||
if err != nil {
|
||||
return delivery.HaveNode{}, false
|
||||
}
|
||||
n, err := delivery.DecodeHaveNode(value)
|
||||
if err != nil {
|
||||
return delivery.HaveNode{}, false
|
||||
}
|
||||
return n, true
|
||||
}
|
||||
|
||||
// ReadHave is the have this node-engine last wrote for a module, and whether the bucket holds one. A bucket that cannot
|
||||
// be read answers as none: the agent then compares with the declaration it kept, and applies only what differs.
|
||||
func (q *Session) ReadHave(ctx context.Context, module string) (delivery.Have, bool) {
|
||||
|
||||
+134
-9
@@ -10,6 +10,7 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
@@ -198,25 +199,35 @@ func TestAModulesHealthFollowsTheLook(t *testing.T) {
|
||||
l := newHaveLink()
|
||||
q, _ := aHaveQueue(l)
|
||||
rep := Report{Modules: []ModuleFact{{Module: "a", Build: "b"}, {Module: "b", Build: "b"},
|
||||
{Module: "c", Build: "b"}, {Module: "d", Build: "b"}}}
|
||||
{Module: "c", Build: "b"}, {Module: "d", Build: "b"}, {Module: "e", Build: "b"}}}
|
||||
q.SayFacts(context.Background(), rep)
|
||||
for _, m := range []string{"a", "b", "c", "d"} {
|
||||
for _, m := range []string{"a", "b", "c", "d", "e"} {
|
||||
if h := l.have(t, "n1."+m).Health; h != delivery.HealthUnknown {
|
||||
t.Errorf("%s with no look is %s, want unknown", m, h)
|
||||
}
|
||||
}
|
||||
h := healthOf(map[string]string{"a": StateHealthy, "b": StateUnhealthy, "c": StateStarting})
|
||||
h.Resources[slicesIndex(h.Resources, "b.server")].Reason = ReasonRestarting
|
||||
// 1496: a directory used as found is not the module's health (e runs healthy beside one); a failed or waiting module's
|
||||
// have says the words of the resource that made it so.
|
||||
h.Resources = append(h.Resources, ResourceHealth{Module: "a", Resource: "a.two", State: StateWaiting},
|
||||
ResourceHealth{Module: "b", Resource: "b.two", State: StateHealthy}, ResourceHealth{Module: "d", Resource: "d.x", State: StateUnknown})
|
||||
ResourceHealth{Module: "b", Resource: "b.two", State: StateHealthy}, ResourceHealth{Module: "d", Resource: "d.x", State: StateUnknown},
|
||||
ResourceHealth{Module: "e", Resource: "e.server", State: StateHealthy},
|
||||
ResourceHealth{Module: "e", Resource: "e.cache", Kind: KindDirectory, State: StateUnhealthy, Reason: ReasonUsedAsFound + " owned by 1:1"})
|
||||
if !q.SayHealth(context.Background(), *h) {
|
||||
t.Fatal("not said")
|
||||
}
|
||||
for m, want := range map[string]delivery.Health{"a": delivery.HealthWaits, "b": delivery.HealthFailed,
|
||||
"c": delivery.HealthWaits, "d": delivery.HealthUnknown} {
|
||||
"c": delivery.HealthWaits, "d": delivery.HealthUnknown, "e": delivery.HealthOK} {
|
||||
if got := l.have(t, "n1."+m).Health; got != want {
|
||||
t.Errorf("%s is %s, want %s", m, got, want)
|
||||
}
|
||||
}
|
||||
for m, want := range map[string]string{"a": "a.two waiting", "b": "b.server unhealthy: restarting", "d": "", "e": ""} {
|
||||
if got := l.have(t, "n1."+m).Detail; got != want {
|
||||
t.Errorf("%s says %q, want %q", m, got, want)
|
||||
}
|
||||
}
|
||||
before := l.setCount()
|
||||
h2 := *h
|
||||
q.SayHealth(context.Background(), h2)
|
||||
@@ -225,8 +236,8 @@ func TestAModulesHealthFollowsTheLook(t *testing.T) {
|
||||
}
|
||||
h2.Resources = []ResourceHealth{{Module: "a", Resource: "a.server", State: StateHealthy}}
|
||||
q.SayHealth(context.Background(), h2)
|
||||
if got := l.have(t, "n1.a").Health; got != delivery.HealthOK {
|
||||
t.Errorf("a after a healthy look is %s", got)
|
||||
if got := l.have(t, "n1.a"); got.Health != delivery.HealthOK || got.Detail != "" {
|
||||
t.Errorf("a after a healthy look is %s, saying %q", got.Health, got.Detail)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -259,6 +270,43 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
|
||||
if !delivery.SameHaveNode(got, want) {
|
||||
t.Errorf("the node key holds %+v, want %+v", got, want)
|
||||
}
|
||||
|
||||
// 1126 (ADR 0555 §3): with its keys, the node key carries the node's record: the profile's present capabilities and
|
||||
// networks, the outward links, the health's contract and the public keys; a report that changes none of it writes
|
||||
// nothing, a changed fact writes the key once, and a report without a profile or health leaves them standing.
|
||||
keys := RecordKeys{TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk", Certificate: "cert"}
|
||||
q.Keys = func() RecordKeys { return keys }
|
||||
rep.Profile = map[string]any{"capabilities": []any{map[string]any{"name": "systemd", "present": true},
|
||||
map[string]any{"name": "docker", "present": false}, map[string]any{"name": "apt", "present": true}},
|
||||
"machine": map[string]any{"networks": []any{map[string]any{"link": "eth0", "network": "192.0.2.0/24"}}}}
|
||||
rep.Outward = []string{"eth0"}
|
||||
rep.Health = &Health{Contract: NodeKeyContract}
|
||||
q.SayFacts(context.Background(), rep)
|
||||
if got, err = delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || got.Record == nil {
|
||||
t.Fatalf("the node key holds no record: %+v, %v", got, err)
|
||||
}
|
||||
wantRecord := delivery.NodeRecord{V: delivery.Version, Node: "n1", TunnelKey: "tk", SealingKey: "sk", ServingKey: "pk",
|
||||
Certificate: "cert", Capabilities: []string{"apt", "systemd"}, OutwardLinks: []string{"eth0"}, NetworksSaid: true,
|
||||
Networks: []catalogue.AttachedNetwork{{Link: "eth0", Network: "192.0.2.0/24"}}, Contract: NodeKeyContract}
|
||||
if !delivery.SameNodeRecord(*got.Record, wantRecord) {
|
||||
t.Errorf("record = %+v, want %+v", *got.Record, wantRecord)
|
||||
}
|
||||
at := l.kv["n1.node"].rev
|
||||
q.SayFacts(context.Background(), rep)
|
||||
if l.kv["n1.node"].rev != at {
|
||||
t.Errorf("a report changing nothing wrote the node key again")
|
||||
}
|
||||
keys.Certificate = "cert-2"
|
||||
q.SayFacts(context.Background(), rep)
|
||||
if got, _ = delivery.DecodeHaveNode(l.kv["n1.node"].value); l.kv["n1.node"].rev != at+1 || got.Record.Certificate != "cert-2" {
|
||||
t.Errorf("a new certificate: revision %d (was %d), record %+v", l.kv["n1.node"].rev, at, got.Record)
|
||||
}
|
||||
bare := rep
|
||||
bare.Health = nil
|
||||
q.SayFacts(context.Background(), bare)
|
||||
if got, _ = delivery.DecodeHaveNode(l.kv["n1.node"].value); got.Record.Contract != NodeKeyContract {
|
||||
t.Errorf("a report without a health statement dropped the contract: %+v", got.Record)
|
||||
}
|
||||
}
|
||||
|
||||
// 1132/AC7: a write the bucket refuses is said in the log; the report went out, and the next apply writes again.
|
||||
@@ -534,9 +582,22 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth(
|
||||
if got := l.have(t, "n1.web"); got.Phase != delivery.PhaseRemoved {
|
||||
t.Fatalf("removed: %+v", got)
|
||||
}
|
||||
q.NoteNode(context.Background(), dig)
|
||||
if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Composed != dig {
|
||||
t.Fatalf("the node key: %+v %v", n, err)
|
||||
q.NoteNode(context.Background(), dig, []string{"core"})
|
||||
if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Composed != dig || len(n.Sealed) != 1 || n.Sealed[0] != "core" {
|
||||
t.Fatalf("the node key, with its own composition's sealed values: %+v %v", n, err)
|
||||
}
|
||||
if n, ok := q.ReadNodeHave(context.Background()); !ok || n.Composed != dig || len(n.Sealed) != 1 {
|
||||
t.Fatalf("the node key read back: %+v %v", n, ok)
|
||||
}
|
||||
// The node's answers to placement candidates ride in the same key (ADR 0559 §4), and go when no group is answered.
|
||||
answers := map[string]delivery.CandidateAnswer{"feat/x": {Head: "h", Verdict: delivery.CandidateSame, Same: []string{"web"}}}
|
||||
q.NoteCandidates(context.Background(), answers)
|
||||
if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Candidates["feat/x"].Head != "h" || n.Composed != dig {
|
||||
t.Fatalf("the node key with its answers: %+v %v", n, err)
|
||||
}
|
||||
q.NoteCandidates(context.Background(), nil)
|
||||
if n, _ := delivery.DecodeHaveNode(l.kv["n1.node"].value); n.Candidates != nil {
|
||||
t.Fatalf("an answer left after its group went: %+v", n.Candidates)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -660,3 +721,67 @@ func TestADeclarationReportMovesAHealthAndARemovalForgetsIt(t *testing.T) {
|
||||
t.Fatal("a removed module is still marked unjudged")
|
||||
}
|
||||
}
|
||||
|
||||
// ADR 0547 §8, 2026-10-26 on the bus node: the bus's user list of a module this engine says no have of (no want, and no
|
||||
// longer in the declaration) still reaches the module's key, merged into what the key says, every other field kept; a
|
||||
// key that does not exist is not made up and is said once; a writer that drops the list again is written over.
|
||||
func TestAUserListReachesTheHaveOfAModuleNoApplyHereSays(t *testing.T) {
|
||||
l := newHaveLink()
|
||||
older := delivery.Have{V: delivery.Version, Build: "build-1", Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||
l.put(t, "n1.nats", older)
|
||||
q, said := aHaveQueue(l)
|
||||
dig := "sha256:" + strings.Repeat("c", 64)
|
||||
w := delivery.UserListWritten{Source: "n1.nats", Revision: 7, Wanted: dig, Digest: dig, Users: 58,
|
||||
At: time.Date(2026, 10, 26, 12, 26, 51, 0, time.UTC)}
|
||||
q.NoteUserList(context.Background(), "nats", w)
|
||||
got := l.have(t, "n1.nats")
|
||||
if got.UserList == nil || got.UserList.Users != 58 || got.UserList.Digest != dig {
|
||||
t.Fatalf("the user list is not in the have: %+v", got)
|
||||
}
|
||||
if got.Build != "build-1" || got.Phase != delivery.PhaseRunning || got.Health != delivery.HealthOK {
|
||||
t.Fatalf("the merge lost the key's own fields: %+v", got)
|
||||
}
|
||||
rev := l.kv["n1.nats"].rev
|
||||
q.NoteUserList(context.Background(), "nats", w)
|
||||
if l.kv["n1.nats"].rev != rev {
|
||||
t.Errorf("the same list was written again, to revision %d", l.kv["n1.nats"].rev)
|
||||
}
|
||||
|
||||
// Another writer replaces the value without the list: the next pass merges it back.
|
||||
l.put(t, "n1.nats", older)
|
||||
q.SayFacts(context.Background(), Report{})
|
||||
if got := l.have(t, "n1.nats"); got.UserList == nil || got.Build != "build-1" {
|
||||
t.Fatalf("a value written without the list kept it out: %+v", got)
|
||||
}
|
||||
|
||||
// No key at all: nothing is created, said once; the module's first have carries the list.
|
||||
q.NoteUserList(context.Background(), "other", w)
|
||||
q.NoteUserList(context.Background(), "other", w)
|
||||
if _, ok := l.kv["n1.other"]; ok {
|
||||
t.Fatalf("a have was made up for a module no apply says")
|
||||
}
|
||||
n := 0
|
||||
for _, s := range *said {
|
||||
if strings.Contains(s, "is in no have") {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Errorf("the missing key was said %d times: %q", n, *said)
|
||||
}
|
||||
fp := "src2:" + strings.Repeat("a", 64)
|
||||
q.NoteModule(context.Background(), ModuleFact{Module: "other", FP: fp, Composed: dig})
|
||||
if got := l.have(t, "n1.other"); got.UserList == nil || got.FP != fp {
|
||||
t.Fatalf("the first have of the module lost the list: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// slicesIndex is the index of the resource with id in rs, or -1.
|
||||
func slicesIndex(rs []ResourceHealth, id string) int {
|
||||
for i, r := range rs {
|
||||
if r.Resource == id {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
@@ -743,4 +743,9 @@ type ModuleFact struct {
|
||||
Phase string
|
||||
Failed bool
|
||||
Detail string
|
||||
// Judgement and Waits are, in phase waiting-yes, the judgement of the move and the composition waiting (ADR 0547 §3).
|
||||
Judgement string
|
||||
Waits string
|
||||
// Sealed is the values the composition it names carries sealed to this node's key (delivery.Have.Sealed).
|
||||
Sealed []string
|
||||
}
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"sort"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
)
|
||||
|
||||
// The node's record (novox/hq ADR 0555 §3, issue 1126): what the machine observes of itself, written in its own have key
|
||||
// `<node>.node` (delivery.HaveNode.Record) with its facts, so every node and the parts that compose read it from the bus
|
||||
// and hear its change as the key's. Each value is the one the controller took from this node's enrolment and reports
|
||||
// until now (its profile's present capabilities and networks, its outward links, its health statement's contract, and
|
||||
// the public halves of the keys it enrolled with), read here from the same sources, never from the controller.
|
||||
|
||||
// RecordKeys are the public halves the node holds and the certificate the mesh issued for its internal name, as read
|
||||
// from its state directory at the time of a pass: the certificate arrives after enrolment, a rekey changes the tunnel key.
|
||||
type RecordKeys struct {
|
||||
TunnelKey string
|
||||
SealingKey string
|
||||
ServingKey string
|
||||
Certificate string
|
||||
// Authority is the mesh's own certificate as this machine last applied it (public).
|
||||
Authority string
|
||||
}
|
||||
|
||||
// recordOf is the node's record from one facts report and its keys; nil for a report that carries no profile (one that
|
||||
// says nothing of the machine, such as a rekey), so the record last written stands.
|
||||
func recordOf(node string, r Report, keys RecordKeys) *delivery.NodeRecord {
|
||||
if r.Profile == nil {
|
||||
return nil
|
||||
}
|
||||
var p struct {
|
||||
Capabilities []struct {
|
||||
Name string `json:"name"`
|
||||
Present bool `json:"present"`
|
||||
} `json:"capabilities"`
|
||||
Machine *struct {
|
||||
Networks []catalogue.AttachedNetwork `json:"networks"`
|
||||
} `json:"machine"`
|
||||
}
|
||||
raw, err := json.Marshal(r.Profile)
|
||||
if err != nil || json.Unmarshal(raw, &p) != nil {
|
||||
return nil
|
||||
}
|
||||
rec := &delivery.NodeRecord{V: delivery.Version, Node: node, TunnelKey: keys.TunnelKey, SealingKey: keys.SealingKey,
|
||||
ServingKey: keys.ServingKey, Certificate: keys.Certificate, Authority: keys.Authority}
|
||||
for _, c := range p.Capabilities {
|
||||
if c.Present && c.Name != "" {
|
||||
rec.Capabilities = append(rec.Capabilities, c.Name)
|
||||
}
|
||||
}
|
||||
sort.Strings(rec.Capabilities)
|
||||
if p.Machine != nil {
|
||||
rec.NetworksSaid = true
|
||||
rec.Networks = p.Machine.Networks
|
||||
}
|
||||
if len(r.Outward) > 0 {
|
||||
rec.OutwardLinks = append([]string(nil), r.Outward...)
|
||||
}
|
||||
if r.Health != nil {
|
||||
rec.Contract = r.Health.Contract
|
||||
}
|
||||
return rec
|
||||
}
|
||||
@@ -47,8 +47,11 @@ type Session struct {
|
||||
// MoveSecret moves one value sealed to this machine into the vault, asked by the controller during the move of
|
||||
// the secrets (design 57's V3, issue 676). Nil answers every such ask with a refusal.
|
||||
MoveSecret MoveSecretFunc
|
||||
Say Announce
|
||||
Timeout time.Duration
|
||||
// Keys reads the node's public keys and its certificate for its record (ADR 0555 §3), at each facts report; nil writes
|
||||
// no record.
|
||||
Keys func() RecordKeys
|
||||
Say Announce
|
||||
Timeout time.Duration
|
||||
|
||||
once sync.Once
|
||||
|
||||
|
||||
@@ -0,0 +1,805 @@
|
||||
// Package selfcompose is the node agent's own composer (novox/hq ADR 0547 §2, issue 1126): it composes this node's
|
||||
// modules and its own composition with mesh-sdk's composer, from what is on the bus, and hands the compositions to the
|
||||
// agent as the agent once read them from the controller's bucket `composed-<node>`.
|
||||
//
|
||||
// **It watches, it never polls** (ADR 0547 §2): its inputs are watches of the node's wants, the indexer's placement and
|
||||
// modules, every node's own have key (where a node's record rides, ADR 0555 §3) and mesh-config's resolved values of this
|
||||
// node. A change of any of them marks the node to compose again; one loop drains the mark, so a burst of changes composes
|
||||
// the node once (composing one node takes 0.8 to 4.4 s, measured 2026-10-26). It hands the agent only the compositions that
|
||||
// differ from the ones it handed before, and takes back one no want names any more. A composition that cannot be made is
|
||||
// said, once per reason, and the ones handed before stand: a node keeps running what it ran rather than nothing.
|
||||
package selfcompose
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
)
|
||||
|
||||
// ResolvedBucket is mesh-config's state of resolved values, keyed `<node>.<module>.<key>` (mesh-catalog modules/mesh-config
|
||||
// resolve.go ResolvedKey); a node reads its own slice alone (ADR 0547 §2).
|
||||
const ResolvedBucket = "mesh-config_resolved"
|
||||
|
||||
// Env is what the composer needs of the node.
|
||||
type Env struct {
|
||||
Node string
|
||||
// Watch follows one bucket's keys matching pattern, as the agent's watches do.
|
||||
Watch func(ctx context.Context, bucket, pattern string) (<-chan agent.Update, error)
|
||||
// Get reads one key's latest value; nil and no error when there is none.
|
||||
Get func(ctx context.Context, bucket, key string) ([]byte, error)
|
||||
Mesh frombus.Mesh
|
||||
Say func(string)
|
||||
// Own asks the vault's `own` in this node's engine's name, with the arguments given: its modules' own secrets and the
|
||||
// pair credentials asked, sealed to its key (ADR 0550 §5). Nil asks nothing, and a module with either does not compose.
|
||||
Own func(ctx context.Context, args map[string]any) (json.RawMessage, error)
|
||||
// Hear subscribes a subject: here the vault's word that a secret changed (VaultSecretChanged), on which the node
|
||||
// composes again, so a rotated secret or credential reaches it at once (ADR 0547 §2). Nil hears nothing.
|
||||
Hear func(subject string) (<-chan []byte, func(), error)
|
||||
// Refused is told why the node cannot compose its modules, and "" once it composes again, so each wanted module's
|
||||
// have says it (agent.NotComposed): a refusal is loud where haves are read, not one journal line. Nil tells nobody.
|
||||
Refused func(why string)
|
||||
// Running is what the node runs now (agent.Everything), and Unseal opens a value sealed to this machine: a secret
|
||||
// the vault seals again for each ask is kept as the box the node already runs when that box opens to the same
|
||||
// value, so neither the switch to composing its own nor a restart changes a composition (ADR 0547 §3). Either nil
|
||||
// keeps the vault's box.
|
||||
Running func() []delivery.Composed
|
||||
Unseal func(sealed string) ([]byte, error)
|
||||
// Held is the sealed values a module's have says it runs (delivery.Have.Sealed), kept by the agent on the bus: what
|
||||
// runs after a restart, before the agent applied anything, and after the switch once the kept declaration is old.
|
||||
Held func(module string) []string
|
||||
}
|
||||
|
||||
// VaultSecretChanged is the vault's event that a secret changed: a name and its generation, never a value.
|
||||
const VaultSecretChanged = "mesh.mod.mesh-vault.event.secret-changed"
|
||||
|
||||
// Composer composes the node and is the agent's watcher of its compositions (agent.Env.Composed).
|
||||
type Composer struct {
|
||||
env Env
|
||||
|
||||
mu sync.Mutex
|
||||
wants map[string]delivery.Want // by module, the node's own under agent.NodeModule
|
||||
placements map[string]delivery.Placement
|
||||
records map[string]delivery.NodeRecord
|
||||
// inOwn is, by module, whether this node's have of it names no fingerprint of the new form (mesh-wireguard, a module
|
||||
// at a build of the old form): the node runs it inside its own composition, and composes it there again (issue 1126).
|
||||
inOwn map[string]bool
|
||||
modules map[string]delivery.Module
|
||||
values map[string]map[string]any // by module, this node's
|
||||
manifests map[string][]byte // by digest, read once
|
||||
handed map[string]string // composed key to the digest handed
|
||||
// sealedOwn is each own secret's sealed value as first handed, by module/name and generation: a value sealed again
|
||||
// is other bytes, so the one first had is kept while the generation stands, and the composition stays the same.
|
||||
sealedOwn map[string]ownSealed
|
||||
said string
|
||||
refusal string
|
||||
caught bool
|
||||
rev uint64
|
||||
}
|
||||
|
||||
// New is a composer for one node.
|
||||
func New(env Env) *Composer {
|
||||
if env.Say == nil {
|
||||
env.Say = func(string) {}
|
||||
}
|
||||
return &Composer{env: env, wants: map[string]delivery.Want{}, placements: map[string]delivery.Placement{},
|
||||
records: map[string]delivery.NodeRecord{}, inOwn: map[string]bool{}, modules: map[string]delivery.Module{}, values: map[string]map[string]any{},
|
||||
manifests: map[string][]byte{}, handed: map[string]string{}, sealedOwn: map[string]ownSealed{}}
|
||||
}
|
||||
|
||||
// ownSealed is one own secret's sealed value at a generation and the key it is sealed to.
|
||||
type ownSealed struct {
|
||||
generation int
|
||||
key string
|
||||
sealed string
|
||||
}
|
||||
|
||||
// input is one watched bucket and how its updates are taken.
|
||||
type input struct {
|
||||
bucket, pattern string
|
||||
take func(c *Composer, u agent.Update)
|
||||
// changed, when set, takes the update in take's place and says whether it changed anything composed.
|
||||
changed func(c *Composer, u agent.Update) bool
|
||||
}
|
||||
|
||||
func (c *Composer) inputs() []input {
|
||||
node := c.env.Node
|
||||
return []input{
|
||||
{delivery.AssignerWantsBucket, "want." + node + ".>", (*Composer).takeWant, nil},
|
||||
{delivery.PlacementBucket, ">", (*Composer).takePlacement, nil},
|
||||
// Every node's record, and this node's own haves (what it runs inside its own composition): two watches, each
|
||||
// within the node's grant (buscompose: readGrants of the have bucket at `*.node`, NodeKeysGrants at `<node>.>`).
|
||||
// A watch of every key is refused by the bus (issue 1126, 2026-10-27 09:21: the fold's `>` stopped every agent).
|
||||
{delivery.HaveBucket, "*." + delivery.HaveNodeWord, nil, (*Composer).takeRecordChanged},
|
||||
{delivery.HaveBucket, node + ".>", nil, (*Composer).takeModuleHave},
|
||||
{delivery.ModulesBucket, ">", (*Composer).takeModule, nil},
|
||||
{ResolvedBucket, node + ".>", (*Composer).takeValue, nil},
|
||||
}
|
||||
}
|
||||
|
||||
// Watch starts the composer and answers the compositions it makes: each as a key of `composed-<node>` would be, then a
|
||||
// Caught update once every input was read and the node composed once, then every change. It ends when ctx ends or an
|
||||
// input's watch ends.
|
||||
func (c *Composer) Watch(ctx context.Context, _ string) (<-chan agent.Update, error) {
|
||||
ins := c.inputs()
|
||||
type tagged struct {
|
||||
in int
|
||||
u agent.Update
|
||||
ok bool
|
||||
}
|
||||
merged := make(chan tagged, 64)
|
||||
for i, in := range ins {
|
||||
ch, err := c.env.Watch(ctx, in.bucket, in.pattern)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the node composes from %s, which cannot be watched: %w", in.bucket, err)
|
||||
}
|
||||
go func(i int, ch <-chan agent.Update) {
|
||||
for u := range ch {
|
||||
select {
|
||||
case merged <- tagged{in: i, u: u, ok: true}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
select {
|
||||
case merged <- tagged{in: i}:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}(i, ch)
|
||||
}
|
||||
if c.env.Hear != nil {
|
||||
heard, stop, err := c.env.Hear(VaultSecretChanged)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the node composes again when a secret changes, and cannot hear the vault say so: %w", err)
|
||||
}
|
||||
go func() {
|
||||
defer stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case _, ok := <-heard:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case merged <- tagged{in: -1, ok: true}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
out := make(chan agent.Update, 64)
|
||||
go func() {
|
||||
defer close(out)
|
||||
caught := make([]bool, len(ins))
|
||||
ready := false
|
||||
dirty := false
|
||||
for {
|
||||
// Drain whatever came, then compose once for all of it.
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case t := <-merged:
|
||||
if !t.ok {
|
||||
c.env.Say(fmt.Sprintf("the watch of %s ended; the node composes no more until its agent starts again", ins[t.in].bucket))
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case t.in < 0:
|
||||
dirty = true // a secret changed: its new generation is asked of the vault at the compose
|
||||
case t.u.Caught:
|
||||
caught[t.in] = true
|
||||
case ins[t.in].changed != nil:
|
||||
dirty = ins[t.in].changed(c, t.u) || dirty
|
||||
default:
|
||||
ins[t.in].take(c, t.u)
|
||||
dirty = true
|
||||
}
|
||||
if len(merged) > 0 {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if !ready {
|
||||
ready = all(caught)
|
||||
if !ready {
|
||||
continue
|
||||
}
|
||||
dirty = true
|
||||
}
|
||||
if !dirty {
|
||||
continue
|
||||
}
|
||||
dirty = false
|
||||
for _, u := range c.compose(ctx) {
|
||||
select {
|
||||
case out <- u:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
if !c.caughtSent() {
|
||||
select {
|
||||
case out <- agent.Update{Caught: true}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func all(b []bool) bool {
|
||||
for _, v := range b {
|
||||
if !v {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// caughtSent says the Caught update went out, marking it sent the first time it is asked.
|
||||
func (c *Composer) caughtSent() bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
sent := c.caught
|
||||
c.caught = true
|
||||
return sent
|
||||
}
|
||||
|
||||
func (c *Composer) takeWant(u agent.Update) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if _, err := delivery.ParseWantNodeKey(u.Key); err == nil {
|
||||
if u.Deleted {
|
||||
delete(c.wants, agent.NodeModule)
|
||||
} else {
|
||||
c.wants[agent.NodeModule] = delivery.Want{}
|
||||
}
|
||||
return
|
||||
}
|
||||
node, module, err := delivery.ParseWantKey(u.Key)
|
||||
if err != nil || node != c.env.Node {
|
||||
return
|
||||
}
|
||||
w, err := delivery.DecodeWant(u.Value)
|
||||
if u.Deleted || err != nil || w.Removed || w.FP == "" {
|
||||
delete(c.wants, module)
|
||||
return
|
||||
}
|
||||
c.wants[module] = w
|
||||
}
|
||||
|
||||
func (c *Composer) takePlacement(u agent.Update) {
|
||||
node, err := delivery.ParsePlacementKey(u.Key)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
p, err := delivery.DecodePlacement(u.Value)
|
||||
if u.Deleted || err != nil || p.Node != node {
|
||||
delete(c.placements, node)
|
||||
return
|
||||
}
|
||||
c.placements[node] = p
|
||||
}
|
||||
|
||||
func (c *Composer) takeRecord(u agent.Update) {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(u.Key)
|
||||
if err != nil || !isNode {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
h, err := delivery.DecodeHaveNode(u.Value)
|
||||
if u.Deleted || err != nil || h.Record == nil {
|
||||
delete(c.records, node)
|
||||
return
|
||||
}
|
||||
c.records[node] = *h.Record
|
||||
}
|
||||
|
||||
// takeRecordChanged takes a node's record, which is always composed again.
|
||||
func (c *Composer) takeRecordChanged(u agent.Update) bool {
|
||||
c.takeRecord(u)
|
||||
return true
|
||||
}
|
||||
|
||||
// takeModuleHave takes one of this node's modules' haves, and says whether it changed which it runs inside its own
|
||||
// composition: one whose have names no fingerprint of the new form and is not removed (nats on novox: a build-...
|
||||
// id; mesh-wireguard: none). Read from the haves, which outlast a restart, never from what the agent has applied
|
||||
// this run: a fresh agent composes before it applies anything (review of the fold, round 1).
|
||||
func (c *Composer) takeModuleHave(u agent.Update) bool {
|
||||
node, module, isNode, err := delivery.ParseHaveKey(u.Key)
|
||||
if err != nil || isNode || node != c.env.Node {
|
||||
return false
|
||||
}
|
||||
in := false
|
||||
if !u.Deleted {
|
||||
if h, err := delivery.DecodeHave(u.Value); err == nil {
|
||||
in = h.Phase != delivery.PhaseRemoved && !strings.HasPrefix(h.FP, delivery.SourcePrefix2)
|
||||
}
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.inOwn[module] == in {
|
||||
return false
|
||||
}
|
||||
if in {
|
||||
c.inOwn[module] = true
|
||||
} else {
|
||||
delete(c.inOwn, module)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// inOwnOf is, of the modules p places, those the node runs inside its own composition (c.inOwn), sorted; c.mu held.
|
||||
func (c *Composer) inOwnOf(p delivery.Placement) []string {
|
||||
var out []string
|
||||
for _, m := range p.Modules {
|
||||
if c.inOwn[m.Module] {
|
||||
out = append(out, m.Module)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func (c *Composer) takeModule(u agent.Update) {
|
||||
name, err := delivery.ParseModulesKey(u.Key)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
m, err := delivery.DecodeModule(u.Value)
|
||||
if u.Deleted || err != nil {
|
||||
delete(c.modules, name)
|
||||
return
|
||||
}
|
||||
c.modules[name] = m
|
||||
}
|
||||
|
||||
func (c *Composer) takeValue(u agent.Update) {
|
||||
rest, ok := strings.CutPrefix(u.Key, c.env.Node+".")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
module, key, ok := strings.Cut(rest, ".")
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
var r struct {
|
||||
Value json.RawMessage `json:"value"`
|
||||
}
|
||||
var v any
|
||||
if u.Deleted || json.Unmarshal(u.Value, &r) != nil || len(r.Value) == 0 || json.Unmarshal(r.Value, &v) != nil {
|
||||
delete(c.values[module], key)
|
||||
return
|
||||
}
|
||||
if c.values[module] == nil {
|
||||
c.values[module] = map[string]any{}
|
||||
}
|
||||
c.values[module][key] = v
|
||||
}
|
||||
|
||||
// compose composes the node once and answers the updates to hand: each composition whose digest differs from the one
|
||||
// handed, and a deletion for each handed one no want names any more.
|
||||
func (c *Composer) compose(ctx context.Context) []agent.Update {
|
||||
c.mu.Lock()
|
||||
fps := map[string]string{}
|
||||
kept := map[string]string{}
|
||||
var behind []string // modules wanted at another build than the trunk's: composed with that build's manifest
|
||||
for module, w := range c.wants {
|
||||
if module == agent.NodeModule {
|
||||
continue
|
||||
}
|
||||
fps[module] = w.FP
|
||||
if m, ok := c.modules[module]; ok && m.FP2 != w.FP {
|
||||
behind = append(behind, module)
|
||||
}
|
||||
}
|
||||
_, nodeWanted := c.wants[agent.NodeModule]
|
||||
// What it runs inside its own composition and no want names at a build of the new form stays there, as the
|
||||
// controller's composed writer folds it (issue 1126); compose folds none a want names.
|
||||
inOwn := c.inOwnOf(c.placements[c.env.Node])
|
||||
r := reader{c: c}
|
||||
c.mu.Unlock()
|
||||
sort.Strings(behind)
|
||||
for _, module := range behind {
|
||||
digest, err := c.builtManifest(ctx, module, fps[module])
|
||||
if err != nil {
|
||||
why := fmt.Sprintf("%s is wanted at %s, whose manifest cannot be read, so the node does not compose: %v", module, fps[module], err)
|
||||
c.refused(why)
|
||||
c.sayOnce(why)
|
||||
return nil
|
||||
}
|
||||
kept[module] = digest
|
||||
}
|
||||
|
||||
compositions, _, err := frombus.Compose(ctx, r, c.env.Mesh, frombus.Scope{Node: c.env.Node, Kept: kept, InOwn: inOwn}, fps)
|
||||
if err != nil {
|
||||
c.refused(fmt.Sprintf("%v", err))
|
||||
c.sayOnce(fmt.Sprintf("the node cannot compose its modules, and keeps running what it runs: %v", err))
|
||||
return nil
|
||||
}
|
||||
c.refused("")
|
||||
c.sayOnce("")
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
var out []agent.Update
|
||||
now := map[string]bool{}
|
||||
keys := make([]string, 0, len(compositions))
|
||||
for key := range compositions {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
comp := compositions[key]
|
||||
if comp.Module == "" && !nodeWanted {
|
||||
continue
|
||||
}
|
||||
now[key] = true
|
||||
dig, err := delivery.ComposedDigestOf(comp)
|
||||
if err != nil {
|
||||
c.env.Say(fmt.Sprintf("the composition %s has no digest, and is not handed: %v", key, err))
|
||||
continue
|
||||
}
|
||||
if c.handed[key] == dig {
|
||||
continue
|
||||
}
|
||||
raw, err := delivery.EncodeComposed(comp)
|
||||
if err != nil {
|
||||
c.env.Say(fmt.Sprintf("the composition %s cannot be written, and is not handed: %v", key, err))
|
||||
continue
|
||||
}
|
||||
c.rev++
|
||||
c.handed[key] = dig
|
||||
out = append(out, agent.Update{Key: key, Value: raw, Revision: c.rev})
|
||||
}
|
||||
for key := range c.handed {
|
||||
if !now[key] {
|
||||
c.rev++
|
||||
delete(c.handed, key)
|
||||
out = append(out, agent.Update{Key: key, Deleted: true, Revision: c.rev})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// refused tells the agent why the node cannot compose, or "" that it composes again, once per change.
|
||||
func (c *Composer) refused(why string) {
|
||||
c.mu.Lock()
|
||||
same := c.refusal == why
|
||||
c.refusal = why
|
||||
c.mu.Unlock()
|
||||
if !same && c.env.Refused != nil {
|
||||
c.env.Refused(why)
|
||||
}
|
||||
}
|
||||
|
||||
// runningBox is the box the node already runs that opens to the same value as box, or box: what the agent runs, and
|
||||
// what each wanted module's have says it runs sealed (Held). Read before c.mu is taken, since the agent's lock is taken
|
||||
// to read what runs.
|
||||
func (c *Composer) runningBox(box string) string {
|
||||
if c.env.Unseal == nil || (c.env.Running == nil && c.env.Held == nil) {
|
||||
return box
|
||||
}
|
||||
value, err := c.env.Unseal(box)
|
||||
if err != nil {
|
||||
return box
|
||||
}
|
||||
var held []string
|
||||
if c.env.Running != nil {
|
||||
for _, comp := range c.env.Running() {
|
||||
held = append(held, agent.SealedIn(comp)...)
|
||||
}
|
||||
}
|
||||
if c.env.Held != nil {
|
||||
c.mu.Lock()
|
||||
// The node's own key among them (agent.NodeModule): its core boxes, as its last run said them.
|
||||
modules := make([]string, 0, len(c.wants))
|
||||
for m := range c.wants {
|
||||
modules = append(modules, m)
|
||||
}
|
||||
c.mu.Unlock()
|
||||
sort.Strings(modules)
|
||||
for _, m := range modules {
|
||||
held = append(held, c.env.Held(m)...)
|
||||
}
|
||||
}
|
||||
for _, h := range held {
|
||||
if h == box {
|
||||
continue
|
||||
}
|
||||
if v, err := c.env.Unseal(h); err == nil && bytes.Equal(v, value) {
|
||||
return h
|
||||
}
|
||||
}
|
||||
return box
|
||||
}
|
||||
|
||||
func (c *Composer) sayOnce(line string) {
|
||||
c.mu.Lock()
|
||||
if c.said == line {
|
||||
c.mu.Unlock()
|
||||
return
|
||||
}
|
||||
c.said = line
|
||||
c.mu.Unlock()
|
||||
if line != "" {
|
||||
c.env.Say(line)
|
||||
}
|
||||
}
|
||||
|
||||
// builtManifest is the digest of the manifest a module's build at a fingerprint was made from (the builder's build key):
|
||||
// what a node composes a module from when its want names another build than the trunk's.
|
||||
func (c *Composer) builtManifest(ctx context.Context, module, fp string) (string, error) {
|
||||
key, err := delivery.BuilderBuildKey(module, fp)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
raw, err := c.env.Get(ctx, delivery.BuilderBuildsBucket, key)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if raw == nil {
|
||||
return "", fmt.Errorf("the builder holds no build %s", key)
|
||||
}
|
||||
b, err := delivery.DecodeBuilderBuild(raw)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if b.Manifest == "" {
|
||||
return "", fmt.Errorf("the build %s names no manifest", key)
|
||||
}
|
||||
return b.Manifest, nil
|
||||
}
|
||||
|
||||
// Policy is a module's upgrade policy as its manifest at the indexer's trunk derives it (catalogue.DerivedUpgrade), for
|
||||
// the agent's judgement of a move (ADR 0547 §3); empty when the manifest is not known or cannot be read. The manifest is
|
||||
// read within ctx, the agent's (review of L2, round 1, finding 5), so a stopping agent stops the read.
|
||||
func (c *Composer) Policy(ctx context.Context, module string) string {
|
||||
c.mu.Lock()
|
||||
m, ok := c.modules[module]
|
||||
c.mu.Unlock()
|
||||
if !ok || m.Manifest == "" {
|
||||
return ""
|
||||
}
|
||||
raw, err := reader{c: c}.Manifest(ctx, m.Manifest)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
parsed, err := catalogue.ParseManifestForInputs(raw)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
policy, _, _ := catalogue.DerivedUpgrade(parsed)
|
||||
return policy
|
||||
}
|
||||
|
||||
// reader is the composer's view of the bus as frombus reads it.
|
||||
type reader struct{ c *Composer }
|
||||
|
||||
func (r reader) Placements(context.Context) (map[string]delivery.Placement, error) {
|
||||
r.c.mu.Lock()
|
||||
defer r.c.mu.Unlock()
|
||||
out := make(map[string]delivery.Placement, len(r.c.placements))
|
||||
for k, v := range r.c.placements {
|
||||
out[k] = v
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r reader) Records(context.Context) (map[string]delivery.NodeRecord, error) {
|
||||
r.c.mu.Lock()
|
||||
defer r.c.mu.Unlock()
|
||||
out := make(map[string]delivery.NodeRecord, len(r.c.records))
|
||||
for k, v := range r.c.records {
|
||||
out[k] = v
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r reader) Modules(context.Context) (map[string]delivery.Module, error) {
|
||||
r.c.mu.Lock()
|
||||
defer r.c.mu.Unlock()
|
||||
out := make(map[string]delivery.Module, len(r.c.modules))
|
||||
for k, v := range r.c.modules {
|
||||
out[k] = v
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r reader) Manifest(ctx context.Context, digest string) ([]byte, error) {
|
||||
r.c.mu.Lock()
|
||||
raw, ok := r.c.manifests[digest]
|
||||
r.c.mu.Unlock()
|
||||
if ok {
|
||||
return raw, nil
|
||||
}
|
||||
key, err := delivery.ManifestsKey(digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value, err := r.c.env.Get(ctx, delivery.ManifestsBucket, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if value == nil {
|
||||
return nil, fmt.Errorf("the indexer holds no manifest %s", digest)
|
||||
}
|
||||
m, err := delivery.DecodeManifest(value)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.c.mu.Lock()
|
||||
r.c.manifests[digest] = []byte(m.JSON)
|
||||
r.c.mu.Unlock()
|
||||
return []byte(m.JSON), nil
|
||||
}
|
||||
|
||||
func (r reader) Values(_ context.Context, node, module string) (map[string]any, error) {
|
||||
if node != r.c.env.Node {
|
||||
return nil, nil // a node reads its own slice alone (ADR 0547 §2)
|
||||
}
|
||||
r.c.mu.Lock()
|
||||
defer r.c.mu.Unlock()
|
||||
if len(r.c.values[module]) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
out := make(map[string]any, len(r.c.values[module]))
|
||||
for k, v := range r.c.values[module] {
|
||||
out[k] = v
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r reader) OwnSecrets(ctx context.Context, node string) ([]compose.OwnSecretRow, error) {
|
||||
if node != r.c.env.Node || r.c.env.Own == nil {
|
||||
return nil, nil
|
||||
}
|
||||
raw, err := r.c.env.Own(ctx, map[string]any{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the vault's own: %w", err)
|
||||
}
|
||||
var a struct {
|
||||
Node string `json:"node"`
|
||||
Key string `json:"key"`
|
||||
Own []struct {
|
||||
Module string `json:"module"`
|
||||
Name string `json:"name"`
|
||||
Origin string `json:"origin"`
|
||||
Generation int `json:"generation"`
|
||||
Sealed json.RawMessage `json:"sealed"`
|
||||
} `json:"own"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, fmt.Errorf("the vault's own answer is not one: %w", err)
|
||||
}
|
||||
if a.Node != node {
|
||||
return nil, fmt.Errorf("the vault answered %q's own secrets to %s", a.Node, node)
|
||||
}
|
||||
ids := make([]string, len(a.Own))
|
||||
fresh := make([]string, len(a.Own))
|
||||
for i, o := range a.Own {
|
||||
ids[i] = o.Module + "/" + o.Name
|
||||
if !r.c.keeps(ids[i], o.Generation, a.Key) {
|
||||
fresh[i] = r.c.runningBox(string(o.Sealed))
|
||||
}
|
||||
}
|
||||
r.c.mu.Lock()
|
||||
defer r.c.mu.Unlock()
|
||||
out := make([]compose.OwnSecretRow, 0, len(a.Own))
|
||||
for i, o := range a.Own {
|
||||
id := ids[i]
|
||||
kept, ok := r.c.sealedOwn[id]
|
||||
if !ok || kept.generation != o.Generation || kept.key != a.Key {
|
||||
kept = ownSealed{generation: o.Generation, key: a.Key, sealed: fresh[i]}
|
||||
if kept.sealed == "" {
|
||||
kept.sealed = string(o.Sealed)
|
||||
}
|
||||
r.c.sealedOwn[id] = kept
|
||||
}
|
||||
out = append(out, compose.OwnSecretRow{Node: node, Module: o.Module, Name: o.Name, Origin: o.Origin,
|
||||
SealedTo: a.Key, Sealed: kept.sealed})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (r reader) Credentials(ctx context.Context, node string, asConsumer, asProvider []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) {
|
||||
if node != r.c.env.Node || r.c.env.Own == nil {
|
||||
return nil, nil, nil
|
||||
}
|
||||
type asked struct {
|
||||
Role string `json:"role"`
|
||||
compose.CredentialKey
|
||||
}
|
||||
var ask []asked
|
||||
for _, k := range asConsumer {
|
||||
ask = append(ask, asked{Role: "consumer", CredentialKey: k})
|
||||
}
|
||||
for _, k := range asProvider {
|
||||
ask = append(ask, asked{Role: "provider", CredentialKey: k})
|
||||
}
|
||||
raw, err := r.c.env.Own(ctx, map[string]any{"credentials": ask})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("the vault's own: %w", err)
|
||||
}
|
||||
var a struct {
|
||||
Node string `json:"node"`
|
||||
Key string `json:"key"`
|
||||
Credentials []struct {
|
||||
Role string `json:"role"`
|
||||
compose.CredentialKey
|
||||
Name string `json:"name"`
|
||||
Generation int `json:"generation"`
|
||||
Sealed json.RawMessage `json:"sealed"`
|
||||
} `json:"credentials"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &a); err != nil {
|
||||
return nil, nil, fmt.Errorf("the vault's own answer is not one: %w", err)
|
||||
}
|
||||
if a.Node != node {
|
||||
return nil, nil, fmt.Errorf("the vault answered %q's credentials to %s", a.Node, node)
|
||||
}
|
||||
ids := make([]string, len(a.Credentials))
|
||||
fresh := make([]string, len(a.Credentials))
|
||||
for i, c := range a.Credentials {
|
||||
ids[i] = c.Role + "|" + c.Name + "|" + c.Provision + "|" + c.Consumer + "|" + c.ConsumerModule + "|" + c.Local
|
||||
if !r.c.keeps(ids[i], c.Generation, a.Key) {
|
||||
fresh[i] = r.c.runningBox(string(c.Sealed))
|
||||
}
|
||||
}
|
||||
r.c.mu.Lock()
|
||||
defer r.c.mu.Unlock()
|
||||
var creds []compose.Credential
|
||||
var issued []compose.Issued
|
||||
for i, c := range a.Credentials {
|
||||
// The value first had is kept while its generation stands, so sealing it again moves no composition; and the
|
||||
// first had is the box the node runs when it opens to the same value (runningBox).
|
||||
id := ids[i]
|
||||
kept, ok := r.c.sealedOwn[id]
|
||||
if !ok || kept.generation != c.Generation || kept.key != a.Key {
|
||||
kept = ownSealed{generation: c.Generation, key: a.Key, sealed: fresh[i]}
|
||||
if kept.sealed == "" {
|
||||
kept.sealed = string(c.Sealed)
|
||||
}
|
||||
r.c.sealedOwn[id] = kept
|
||||
}
|
||||
k := c.CredentialKey
|
||||
switch c.Role {
|
||||
case "consumer":
|
||||
creds = append(creds, compose.Credential{Provision: k.Provision, Consumer: k.Consumer, ConsumerModule: k.ConsumerModule,
|
||||
Provider: k.Provider, Local: k.Local, State: compose.CredentialHeld, Sealed: kept.sealed})
|
||||
case "provider":
|
||||
issued = append(issued, compose.Issued{Provision: k.Provision, Consumer: k.Consumer, ConsumerModule: k.ConsumerModule,
|
||||
Local: k.Local, Provider: k.Provider, ForProvider: kept.sealed})
|
||||
}
|
||||
}
|
||||
return creds, issued, nil
|
||||
}
|
||||
|
||||
// keeps says the composer holds id's sealed value at this generation and key already.
|
||||
func (c *Composer) keeps(id string, generation int, key string) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
kept, ok := c.sealedOwn[id]
|
||||
return ok && kept.generation == generation && kept.key == key
|
||||
}
|
||||
@@ -0,0 +1,721 @@
|
||||
package selfcompose
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/buscompose"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
)
|
||||
|
||||
const lemurs = `{"module":"lemurs","version":"1","capabilities":["systemd"],"resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`
|
||||
|
||||
// fakeBus is the buckets the composer watches, each watch a channel that gets the updates a test writes to its bucket
|
||||
// whose key its pattern matches, as the bus's own watches do, and the keys Get reads. It keeps every watch and get
|
||||
// made, so a test checks each against the node's grant.
|
||||
type fakeBus struct {
|
||||
mu sync.Mutex
|
||||
watches []patternWatch
|
||||
kv map[string][]byte // bucket/key
|
||||
said []string
|
||||
reads [][2]string // bucket, pattern or key: every watch and get
|
||||
}
|
||||
|
||||
func newFakeBus() *fakeBus {
|
||||
return &fakeBus{kv: map[string][]byte{}}
|
||||
}
|
||||
|
||||
func (b *fakeBus) env() Env {
|
||||
return Env{Node: "n1",
|
||||
Watch: func(_ context.Context, bucket, pattern string) (<-chan agent.Update, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
w := patternWatch{bucket: bucket, pattern: pattern, ch: make(chan agent.Update, 64)}
|
||||
b.watches = append(b.watches, w)
|
||||
b.reads = append(b.reads, [2]string{bucket, pattern})
|
||||
return w.ch, nil
|
||||
},
|
||||
Get: func(_ context.Context, bucket, key string) ([]byte, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.reads = append(b.reads, [2]string{bucket, key})
|
||||
return b.kv[bucket+"/"+key], nil
|
||||
},
|
||||
Say: func(s string) { b.mu.Lock(); b.said = append(b.said, s); b.mu.Unlock() }}
|
||||
}
|
||||
|
||||
// send gives u to every watch of bucket whose pattern matches its key; Caught to every watch of bucket.
|
||||
func (b *fakeBus) send(bucket string, u agent.Update) {
|
||||
b.mu.Lock()
|
||||
ws := append([]patternWatch(nil), b.watches...)
|
||||
b.mu.Unlock()
|
||||
for _, w := range ws {
|
||||
if w.bucket == bucket && (u.Caught || matches(w.pattern, u.Key)) {
|
||||
w.ch <- u
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *fakeBus) put(t *testing.T, bucket, key string, value []byte) {
|
||||
t.Helper()
|
||||
b.send(bucket, agent.Update{Key: key, Value: value})
|
||||
}
|
||||
|
||||
// grantAllows says whether the node's grant (buscompose.PermissionsFor, KindNode) lets it watch or read pattern of
|
||||
// bucket: the consumer a watch or a get makes (`$JS.API.CONSUMER.CREATE.KV_<bucket>.<name>.$KV.<bucket>.<pattern>`)
|
||||
// and the stream's information, matched as the bus matches a permission against a subject.
|
||||
func grantAllows(t *testing.T, node, bucket, pattern string) bool {
|
||||
t.Helper()
|
||||
perms, err := buscompose.PermissionsFor(buscompose.Principal{Kind: buscompose.KindNode, Node: node})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
allowed := func(subject string) bool {
|
||||
for _, deny := range perms.PublishDeny {
|
||||
if subjectMatches(deny, subject) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
for _, p := range perms.Publish {
|
||||
if subjectMatches(p, subject) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
stream := "KV_" + bucket
|
||||
return allowed("$JS.API.STREAM.INFO."+stream) && allowed("$JS.API.CONSUMER.CREATE."+stream+".c1.$KV."+bucket+"."+pattern)
|
||||
}
|
||||
|
||||
// subjectMatches is the bus's match of a permission against a subject: `*` any one token, `>` one or more to the end.
|
||||
func subjectMatches(permission, subject string) bool {
|
||||
p, s := strings.Split(permission, "."), strings.Split(subject, ".")
|
||||
for i, tok := range p {
|
||||
if tok == ">" {
|
||||
return len(s) > i
|
||||
}
|
||||
if i >= len(s) || (tok != "*" && tok != s[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(p) == len(s)
|
||||
}
|
||||
|
||||
func next(t *testing.T, out <-chan agent.Update) agent.Update {
|
||||
t.Helper()
|
||||
select {
|
||||
case u := <-out:
|
||||
return u
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("no update from the composer")
|
||||
return agent.Update{}
|
||||
}
|
||||
}
|
||||
|
||||
func quiet(t *testing.T, out <-chan agent.Update) {
|
||||
t.Helper()
|
||||
select {
|
||||
case u := <-out:
|
||||
t.Fatalf("the composer handed %s, and nothing changed", u.Key)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
|
||||
// Issue 1126 (ADR 0547 §2): the node composes its wanted modules and its own composition from its inputs on the bus once
|
||||
// every input was read, hands each, then Caught; an input that changes nothing composed hands nothing; a module no want
|
||||
// names any more is taken back; a node that cannot compose says so once and hands nothing.
|
||||
func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
b := newFakeBus()
|
||||
fp := delivery.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
digest := delivery.ManifestDigest([]byte(lemurs))
|
||||
man, _ := delivery.EncodeManifest(delivery.Manifest{Module: "lemurs", JSON: lemurs})
|
||||
key, _ := delivery.ManifestsKey(digest)
|
||||
b.kv[delivery.ManifestsBucket+"/"+key] = man
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
env := b.env()
|
||||
// The vault's own: keeper's password sealed to n1's key, sealed again (other bytes) at every ask.
|
||||
var asks, generation int
|
||||
sealedAt := map[string]int{} // each box handed, by the generation it opens to
|
||||
env.Own = func(context.Context, map[string]any) (json.RawMessage, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
asks++
|
||||
sealedAt[fmt.Sprintf(`{"ask":%d}`, asks)] = generation + 1
|
||||
return json.RawMessage(fmt.Sprintf(`{"node":"n1","key":"sk","own":[{"module":"keeper","name":"password","origin":"made",`+
|
||||
`"generation":%d,"sealed":{"ask":%d}}]}`, generation+1, asks)), nil
|
||||
}
|
||||
secretChanged := make(chan []byte, 1)
|
||||
env.Hear = func(subject string) (<-chan []byte, func(), error) {
|
||||
if subject != VaultSecretChanged {
|
||||
t.Errorf("heard %s", subject)
|
||||
}
|
||||
return secretChanged, func() {}, nil
|
||||
}
|
||||
c := New(env)
|
||||
out, err := c.Watch(ctx, "n1.>")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// oldie is placed and no want names it (a module at a build of the old form, nats on novox): the node runs it inside
|
||||
// its own composition, and composes it there again, even before the agent has applied anything (Running empty).
|
||||
oldie := `{"module":"oldie","version":"1","capabilities":["systemd"],"resources":[{"id":"service","type":"service","unit":"oldie.service","state":"running","boot":"enabled"}]}`
|
||||
odigest := delivery.ManifestDigest([]byte(oldie))
|
||||
oman, _ := delivery.EncodeManifest(delivery.Manifest{Module: "oldie", JSON: oldie})
|
||||
okey, _ := delivery.ManifestsKey(odigest)
|
||||
b.kv[delivery.ManifestsBucket+"/"+okey] = oman
|
||||
omod, _ := delivery.EncodeModule(delivery.Module{FP2: delivery.SourcePrefix2 + strings.Repeat("c", 64), Manifest: odigest, Repository: "novox/o", Commit: "c"})
|
||||
place, _ := delivery.EncodePlacement(delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"},
|
||||
Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}}})
|
||||
have, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}})
|
||||
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"})
|
||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fp, Self: true})
|
||||
nodeWant, _ := delivery.EncodeNodeWant(delivery.NodeWant{Self: true})
|
||||
b.put(t, delivery.PlacementBucket, "placement.n1", place)
|
||||
b.put(t, delivery.HaveBucket, "n1.node", have)
|
||||
oldHave, _ := delivery.EncodeHave(delivery.Have{Build: "build-1", Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||
b.put(t, delivery.HaveBucket, "n1.oldie", oldHave)
|
||||
b.put(t, delivery.ModulesBucket, "lemurs", mod)
|
||||
b.put(t, delivery.ModulesBucket, "oldie", omod)
|
||||
b.put(t, delivery.AssignerWantsBucket, "want.n1.lemurs", want)
|
||||
b.put(t, delivery.AssignerWantsBucket, "want.n1.node", nodeWant)
|
||||
for _, bucket := range []string{delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.HaveBucket, delivery.ModulesBucket, ResolvedBucket} {
|
||||
b.send(bucket, agent.Update{Caught: true})
|
||||
}
|
||||
got := map[string]bool{}
|
||||
var own delivery.Composed
|
||||
for {
|
||||
u := next(t, out)
|
||||
if u.Caught {
|
||||
break
|
||||
}
|
||||
c, err := delivery.DecodeComposed(u.Value)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", u.Key, err)
|
||||
}
|
||||
if c.Node != "n1" {
|
||||
t.Errorf("%s composed for %q", u.Key, c.Node)
|
||||
}
|
||||
if c.Module == "" {
|
||||
own = c
|
||||
}
|
||||
got[u.Key] = true
|
||||
}
|
||||
var ownText []string
|
||||
for _, r := range own.Resources {
|
||||
ownText = append(ownText, string(r))
|
||||
}
|
||||
if !strings.Contains(strings.Join(ownText, " "), `"id":"oldie.service"`) {
|
||||
t.Errorf("the node's own composition, composed before anything was applied, lacks oldie: %v", ownText)
|
||||
}
|
||||
// Issue 1126, 2026-10-27 09:21: every watch and get the composer makes is one the node's grant allows; a watch of
|
||||
// every have key (`>`) was refused by the bus and stopped every node agent.
|
||||
b.mu.Lock()
|
||||
for _, r := range b.reads {
|
||||
if !grantAllows(t, "n1", r[0], r[1]) {
|
||||
t.Errorf("the composer reads %s of %s, which the node's grant does not allow", r[1], r[0])
|
||||
}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
lemursKey, _ := delivery.ComposedKey("n1", "lemurs", fp)
|
||||
nodeKey, _ := delivery.ComposedNodeKey("n1")
|
||||
if !got[lemursKey] || !got[nodeKey] || len(got) != 2 {
|
||||
b.mu.Lock()
|
||||
t.Logf("said: %v", b.said)
|
||||
b.mu.Unlock()
|
||||
t.Fatalf("handed %v, want %s and %s", got, lemursKey, nodeKey)
|
||||
}
|
||||
|
||||
// A config value of another module, and a value changing nothing composed, hand nothing.
|
||||
b.put(t, ResolvedBucket, "n1.other.k", []byte(`{"value":1}`))
|
||||
quiet(t, out)
|
||||
|
||||
// A want removed takes its composition back.
|
||||
removed, _ := delivery.EncodeWant(delivery.Want{Removed: true})
|
||||
b.put(t, delivery.AssignerWantsBucket, "want.n1.lemurs", removed)
|
||||
if u := next(t, out); u.Key != lemursKey || !u.Deleted {
|
||||
t.Fatalf("a removed want handed %+v", u)
|
||||
}
|
||||
|
||||
// A want of a module whose manifest cannot be read: said once, nothing handed, what was handed stands.
|
||||
gone := delivery.SourcePrefix2 + strings.Repeat("b", 64)
|
||||
missing, _ := delivery.EncodeWant(delivery.Want{FP: gone, Self: true})
|
||||
b.put(t, delivery.AssignerWantsBucket, "want.n1.lemurs", missing)
|
||||
quiet(t, out)
|
||||
b.put(t, delivery.AssignerWantsBucket, "want.n1.lemurs", missing)
|
||||
quiet(t, out)
|
||||
b.mu.Lock()
|
||||
n := 0
|
||||
for _, s := range b.said {
|
||||
if strings.Contains(s, "lemurs is wanted at") {
|
||||
n++
|
||||
}
|
||||
}
|
||||
b.mu.Unlock()
|
||||
if n != 1 {
|
||||
t.Errorf("the fault was said %d times, want once: %v", n, b.said)
|
||||
}
|
||||
|
||||
// An own secret comes sealed from the vault into the composition; sealed again at the same generation it is the value
|
||||
// first had, so nothing is handed again; a new generation is handed.
|
||||
b.put(t, delivery.AssignerWantsBucket, "want.n1.lemurs", want)
|
||||
if u := next(t, out); u.Key != lemursKey {
|
||||
t.Fatalf("lemurs wanted back handed %s", u.Key)
|
||||
}
|
||||
keeper := `{"module":"keeper","version":"1","own-secrets":{"password":"/var/lib/mesh/keeper/password"},` +
|
||||
`"resources":[{"id":"state","type":"directory","path":"/var/lib/mesh/keeper","mode":"0700"}]}`
|
||||
kdigest := delivery.ManifestDigest([]byte(keeper))
|
||||
kman, _ := delivery.EncodeManifest(delivery.Manifest{Module: "keeper", JSON: keeper})
|
||||
kkey, _ := delivery.ManifestsKey(kdigest)
|
||||
b.mu.Lock()
|
||||
b.kv[delivery.ManifestsBucket+"/"+kkey] = kman
|
||||
b.mu.Unlock()
|
||||
kmod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: kdigest, Repository: "novox/k", Commit: "c"})
|
||||
both, _ := delivery.EncodePlacement(delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"},
|
||||
Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "oldie"}, {Module: "keeper"}}})
|
||||
b.put(t, delivery.ModulesBucket, "keeper", kmod)
|
||||
b.put(t, delivery.PlacementBucket, "placement.n1", both)
|
||||
b.put(t, delivery.AssignerWantsBucket, "want.n1.keeper", want)
|
||||
keeperKey, _ := delivery.ComposedKey("n1", "keeper", fp)
|
||||
var handed string
|
||||
for handed == "" {
|
||||
if u := next(t, out); u.Key == keeperKey {
|
||||
handed = string(u.Value)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(handed, `{\"ask\":`) {
|
||||
t.Fatalf("keeper's composition does not carry the sealed value: %s", handed)
|
||||
}
|
||||
before := handed
|
||||
b.put(t, ResolvedBucket, "n1.other.k", []byte(`{"value":2}`))
|
||||
quiet(t, out)
|
||||
b.mu.Lock()
|
||||
generation++
|
||||
b.mu.Unlock()
|
||||
// The vault says the secret changed: the node composes again on that word, with no other input moving.
|
||||
secretChanged <- []byte(`{"name":"own/n1/keeper/password","generation":2}`)
|
||||
rotated := next(t, out)
|
||||
if rotated.Key != keeperKey {
|
||||
t.Fatalf("a new generation handed %s, want keeper again", rotated.Key)
|
||||
}
|
||||
|
||||
// Review of L2 (round 2): rotated, then the node-engine restarts. The agent has applied nothing yet, so the box the
|
||||
// node runs comes from keeper's have (Held); the vault seals the value again, other bytes, and the box the have holds
|
||||
// is kept, since it opens to the same value: the composition does not move. The box from before the rotation opens
|
||||
// to the old value, so the vault's is taken.
|
||||
boxOf := func(raw string) string {
|
||||
c, err := delivery.DecodeComposed([]byte(raw))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sealed := agent.SealedIn(c)
|
||||
if len(sealed) != 1 {
|
||||
t.Fatalf("keeper's composition carries %v", sealed)
|
||||
}
|
||||
return sealed[0]
|
||||
}
|
||||
// And the node's own key (review of L2, round 3, finding 1): a core box its own composition carries is held there, and
|
||||
// kept after a restart as a module's is, so the node's own composition does not move either.
|
||||
for _, row := range []struct {
|
||||
name, held, in string
|
||||
kept bool
|
||||
}{{"the have holds the rotated value", boxOf(string(rotated.Value)), "keeper", true},
|
||||
{"the have holds the value before the rotation", boxOf(before), "keeper", false},
|
||||
{"the node's own key holds the rotated value", boxOf(string(rotated.Value)), agent.NodeModule, true}} {
|
||||
restarted := env
|
||||
restarted.Running = func() []delivery.Composed { return nil }
|
||||
restarted.Held = func(module string) []string {
|
||||
if module == row.in {
|
||||
return []string{row.held}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
restarted.Unseal = func(box string) ([]byte, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if g, ok := sealedAt[box]; ok {
|
||||
return []byte(fmt.Sprintf("password-%d", g)), nil
|
||||
}
|
||||
return nil, fmt.Errorf("not sealed to n1: %s", box)
|
||||
}
|
||||
again := New(restarted)
|
||||
again.wants = map[string]delivery.Want{"keeper": {FP: fp, Self: true}, agent.NodeModule: {}}
|
||||
rows, err := reader{c: again}.OwnSecrets(ctx, "n1")
|
||||
if err != nil || len(rows) != 1 {
|
||||
t.Fatalf("%s: own secrets %+v, %v", row.name, rows, err)
|
||||
}
|
||||
if kept := rows[0].Sealed == row.held; kept != row.kept {
|
||||
t.Errorf("%s: composed with %s (held %s), kept %v, want %v", row.name, rows[0].Sealed, row.held, kept, row.kept)
|
||||
}
|
||||
}
|
||||
|
||||
// The pair credentials asked (ADR 0550 §5): the keys go to the vault's own with their role, a consumer's comes back as
|
||||
// a credential held, a provider's as the grant's copy, each sealed to n1's key, the value first had kept.
|
||||
var sent map[string]any
|
||||
pairEnv := b.env()
|
||||
pairEnv.Own = func(_ context.Context, args map[string]any) (json.RawMessage, error) {
|
||||
sent = args
|
||||
return json.RawMessage(`{"node":"n1","key":"sk","credentials":[` +
|
||||
`{"role":"consumer","provision":"pg","consumer":"n1","consumer_module":"web","provider":"n2","name":"pair/n2/pg/n1/web","generation":1,"sealed":{"c":1}},` +
|
||||
`{"role":"provider","provision":"s3","consumer":"n3","consumer_module":"files","provider":"n1","name":"provision/n1/s3","generation":2,"sealed":{"p":1}}]}`), nil
|
||||
}
|
||||
// The node runs the consumer's credential already, sealed otherwise (the controller sealed it, or the vault at
|
||||
// another ask): that box opens to the same value, so it is kept, and the composition does not move.
|
||||
pairEnv.Running = func() []delivery.Composed {
|
||||
return []delivery.Composed{{Module: "web", Resources: []json.RawMessage{json.RawMessage(`{"id":"web.secret-pg","sealed":"{\"run\":1}"}`)}}}
|
||||
}
|
||||
opened := map[string]string{`{"c":1}`: "pg-pw", `{"run":1}`: "pg-pw", `{"p":1}`: "s3-pw"}
|
||||
pairEnv.Unseal = func(box string) ([]byte, error) {
|
||||
if v, ok := opened[box]; ok {
|
||||
return []byte(v), nil
|
||||
}
|
||||
return nil, fmt.Errorf("not sealed to n1: %s", box)
|
||||
}
|
||||
var refusals []string
|
||||
pairEnv.Refused = func(why string) { refusals = append(refusals, why) }
|
||||
pc := New(pairEnv)
|
||||
consumer := []compose.CredentialKey{{Provision: "pg", Consumer: "n1", ConsumerModule: "web", Provider: "n2"}}
|
||||
provider := []compose.CredentialKey{{Provision: "s3", Consumer: "n3", ConsumerModule: "files", Provider: "n1"}}
|
||||
creds, issued, err := reader{c: pc}.Credentials(ctx, "n1", consumer, provider)
|
||||
if err != nil || len(creds) != 1 || len(issued) != 1 {
|
||||
t.Fatalf("credentials %+v issued %+v, %v", creds, issued, err)
|
||||
}
|
||||
if creds[0].State != compose.CredentialHeld || creds[0].Provider != "n2" || creds[0].Sealed != `{"run":1}` ||
|
||||
issued[0].Consumer != "n3" || issued[0].ForProvider != `{"p":1}` {
|
||||
t.Errorf("credentials %+v issued %+v", creds, issued)
|
||||
}
|
||||
// A node that cannot compose tells the agent once, and once again when it composes (the agent's haves say it).
|
||||
pc.refused("a stand-in for web's password")
|
||||
pc.refused("a stand-in for web's password")
|
||||
pc.refused("")
|
||||
if len(refusals) != 2 || refusals[0] == "" || refusals[1] != "" {
|
||||
t.Errorf("the agent was told %q", refusals)
|
||||
}
|
||||
if asked, _ := json.Marshal(sent["credentials"]); !strings.Contains(string(asked), `"role":"consumer"`) || !strings.Contains(string(asked), `"role":"provider"`) {
|
||||
t.Errorf("the vault was asked %s", asked)
|
||||
}
|
||||
// ADR 0559 §4: the node answers placement candidates in shadow, placed or not, applying nothing and handing no
|
||||
// composition: the same; a module the group adds, its secret not yet issued; a module removed; a placement it cannot
|
||||
// compose; a group whose keys went is no longer answered. What runs is never composed from a candidate: the
|
||||
// Composer does not read the state.
|
||||
for _, in := range New(env).inputs() {
|
||||
if in.bucket == delivery.PlacementCandidatesBucket {
|
||||
t.Fatal("the node's own composer reads placement candidates")
|
||||
}
|
||||
}
|
||||
sb := newPatternBus()
|
||||
trunk := delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"}, Modules: []delivery.PlacedModule{{Module: "lemurs"}}}
|
||||
mods := map[string]delivery.Module{"lemurs": {FP2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"},
|
||||
"keeper": {FP2: fp, Manifest: kdigest, Repository: "novox/k", Commit: "c"}}
|
||||
readOnly := &fakeBus{kv: map[string][]byte{delivery.ManifestsBucket + "/" + key: man, delivery.ManifestsBucket + "/" + kkey: kman}}
|
||||
senv := readOnly.env()
|
||||
senv.Watch = sb.watch
|
||||
senv.Own = func(context.Context, map[string]any) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"node":"n1","key":"sk","own":[]}`), nil // nothing issued for keeper: it is not placed here
|
||||
}
|
||||
// What n1 runs: lemurs and its own composition, as its composer composes them from the trunk.
|
||||
running, _, err := frombus.Compose(ctx, fixedReader{placements: map[string]delivery.Placement{"n1": trunk},
|
||||
records: map[string]delivery.NodeRecord{"n1": {Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}},
|
||||
modules: mods, manifests: map[string][]byte{digest: []byte(lemurs), kdigest: []byte(keeper)}}, frombus.Mesh{}, frombus.Scope{Node: "n1"}, map[string]string{"lemurs": fp})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
senv.Running = func() []delivery.Composed {
|
||||
var out []delivery.Composed
|
||||
for _, c := range running {
|
||||
out = append(out, c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
answered := make(chan map[string]delivery.CandidateAnswer, 8)
|
||||
sh := NewShadow(senv, func(_ context.Context, a map[string]delivery.CandidateAnswer) { answered <- a })
|
||||
sctx, scancel := context.WithCancel(ctx)
|
||||
defer scancel()
|
||||
// Review of L2b, round 1, finding 2: the candidates' state is not there when the node starts (a delivery before the
|
||||
// indexer made it): the shadow tries again, and answers once it appears.
|
||||
sb.missing(delivery.PlacementCandidatesBucket, true)
|
||||
go sh.Keep(sctx, func(int) time.Duration { return 5 * time.Millisecond })
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
sb.missing(delivery.PlacementCandidatesBucket, false)
|
||||
sb.made(6)
|
||||
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}})
|
||||
tp, _ := delivery.EncodePlacement(trunk)
|
||||
sb.put(delivery.PlacementBucket, "placement.n1", tp, 1)
|
||||
sb.put(delivery.HaveBucket, "n1.node", rec, 1)
|
||||
for name, m := range mods {
|
||||
raw, _ := delivery.EncodeModule(m)
|
||||
sb.put(delivery.ModulesBucket, name, raw, 1)
|
||||
}
|
||||
candidate := func(group string, rev uint64, modules ...string) {
|
||||
p := trunk
|
||||
p.Modules = nil
|
||||
for _, m := range modules {
|
||||
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
raw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Group: group,
|
||||
Pull: delivery.Head{Repository: delivery.CatalogueRepository, Number: 1, Head: "h-" + group}, Placement: &p})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k, _ := delivery.PlacementCandidateKey(group, "n1")
|
||||
sb.put(delivery.PlacementCandidatesBucket, k, raw, rev)
|
||||
}
|
||||
candidate("same", 3, "lemurs")
|
||||
candidate("adds", 4, "lemurs", "keeper")
|
||||
candidate("drops", 5)
|
||||
candidate("ghost", 6, "lemurs", "ghost")
|
||||
sb.caught()
|
||||
var answers map[string]delivery.CandidateAnswer
|
||||
waitFor := func(why string, ok func(map[string]delivery.CandidateAnswer) bool) {
|
||||
t.Helper()
|
||||
deadline := time.After(10 * time.Second)
|
||||
for answers == nil || !ok(answers) {
|
||||
select {
|
||||
case answers = <-answered:
|
||||
case <-deadline:
|
||||
t.Fatalf("%s: last answers %+v", why, answers)
|
||||
}
|
||||
}
|
||||
}
|
||||
waitFor("every group answered", func(a map[string]delivery.CandidateAnswer) bool { return len(a) == 4 })
|
||||
// Every watch and get the shadow makes is one the node's grant allows (issue 1126, 2026-10-27 09:21).
|
||||
sb.mu.Lock()
|
||||
for _, w := range sb.watches {
|
||||
if !grantAllows(t, "n1", w.bucket, w.pattern) {
|
||||
t.Errorf("the shadow watches %s of %s, which the node's grant does not allow", w.pattern, w.bucket)
|
||||
}
|
||||
}
|
||||
sb.mu.Unlock()
|
||||
readOnly.mu.Lock()
|
||||
for _, r := range readOnly.reads {
|
||||
if !grantAllows(t, "n1", r[0], r[1]) {
|
||||
t.Errorf("the shadow reads %s of %s, which the node's grant does not allow", r[1], r[0])
|
||||
}
|
||||
}
|
||||
readOnly.mu.Unlock()
|
||||
for group, want := range map[string]func(delivery.CandidateAnswer) bool{
|
||||
"same": func(a delivery.CandidateAnswer) bool {
|
||||
return a.Verdict == delivery.CandidateSame && a.Head == "h-same" && a.Revision == 3 && len(a.Same) == 2
|
||||
},
|
||||
"adds": func(a delivery.CandidateAnswer) bool {
|
||||
return a.Verdict == delivery.CandidateDiffers && len(a.Added) == 1 && a.Added[0].Module == "keeper" &&
|
||||
len(a.Added[0].Unissued) == 1 && len(a.Refusals) == 0
|
||||
},
|
||||
"drops": func(a delivery.CandidateAnswer) bool {
|
||||
return a.Verdict == delivery.CandidateDiffers && len(a.Removed) == 1 && a.Removed[0] == "lemurs"
|
||||
},
|
||||
"ghost": func(a delivery.CandidateAnswer) bool {
|
||||
return a.Verdict == delivery.CandidateRefused && a.Refused != ""
|
||||
},
|
||||
} {
|
||||
if !want(answers[group]) {
|
||||
t.Errorf("%s: %+v", group, answers[group])
|
||||
}
|
||||
}
|
||||
// The node's own key changing (its answers written) composes nothing again; a group's key deleted drops its answer.
|
||||
sb.put(delivery.HaveBucket, "n1.node", rec, 2)
|
||||
select {
|
||||
case a := <-answered:
|
||||
t.Errorf("answers handed again for nothing changed: %+v", a)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
k, _ := delivery.PlacementCandidateKey("same", "n1")
|
||||
sb.del(delivery.PlacementCandidatesBucket, k)
|
||||
waitFor("the deleted group's answer stays", func(a map[string]delivery.CandidateAnswer) bool { _, ok := a["same"]; return !ok && len(a) == 3 })
|
||||
|
||||
// Review of L2b, round 2: a watch ends while the link stays; a group's key is deleted before the shadow watches
|
||||
// again. The new run reads afresh, so that group is no longer answered.
|
||||
sb.missing(delivery.PlacementCandidatesBucket, true)
|
||||
sb.end(delivery.ModulesBucket)
|
||||
dropsKey, _ := delivery.PlacementCandidateKey("drops", "n1")
|
||||
sb.del(delivery.PlacementCandidatesBucket, dropsKey)
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
sb.missing(delivery.PlacementCandidatesBucket, false)
|
||||
waitFor("a group deleted while the shadow was down is still answered", func(a map[string]delivery.CandidateAnswer) bool {
|
||||
_, ok := a["drops"]
|
||||
return !ok && len(a) == 2
|
||||
})
|
||||
|
||||
// Issue 1126, the seed's review: a module n1 runs inside its own composition (nats, at a build of the old form) is
|
||||
// composed there again, not at the trunk's build as one added; lemurs at the build it runs, keeper (added) at the
|
||||
// trunk's.
|
||||
inOwnRunning := []delivery.Composed{{Node: "n1", Module: "lemurs", FP: fp}}
|
||||
other := "src2:" + strings.Repeat("9", 64)
|
||||
gotFps := shadowBuilds(inOwnRunning, delivery.Placement{Node: "n1", Modules: []delivery.PlacedModule{{Module: "lemurs"}, {Module: "nats"}, {Module: "keeper"}}},
|
||||
map[string]delivery.Module{"lemurs": {FP2: other}, "nats": {FP2: other}, "keeper": {FP2: other}}, []string{"nats"})
|
||||
if fmt.Sprint(gotFps) != fmt.Sprint(map[string]string{"lemurs": fp, "keeper": other}) {
|
||||
t.Errorf("shadow builds %v; want lemurs at what runs, keeper at the trunk's, nats at none (folded)", gotFps)
|
||||
}
|
||||
}
|
||||
|
||||
// patternBus is a bus whose every watch gets the updates of its bucket matching its pattern, as the bus's own watches
|
||||
// do: two watches of one bucket each see what matches theirs.
|
||||
type patternBus struct {
|
||||
mu sync.Mutex
|
||||
watches []patternWatch
|
||||
absent map[string]bool
|
||||
// kv is each bucket's keys now; once replay is set, a new watch gets those matching it and then Caught, as a watch
|
||||
// of the bus's own does.
|
||||
kv map[string]map[string]agent.Update
|
||||
replay bool
|
||||
}
|
||||
|
||||
// end ends every watch of a bucket, as a watch the bus drops, and from then on each new watch replays what is there.
|
||||
func (b *patternBus) end(bucket string) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
b.replay = true
|
||||
kept := b.watches[:0]
|
||||
for _, w := range b.watches {
|
||||
if w.bucket == bucket {
|
||||
close(w.ch)
|
||||
continue
|
||||
}
|
||||
kept = append(kept, w)
|
||||
}
|
||||
b.watches = kept
|
||||
}
|
||||
|
||||
// missing makes a bucket's watch fail, as one the bus holds no state for, or work again.
|
||||
func (b *patternBus) missing(bucket string, gone bool) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.absent == nil {
|
||||
b.absent = map[string]bool{}
|
||||
}
|
||||
b.absent[bucket] = gone
|
||||
}
|
||||
|
||||
type patternWatch struct {
|
||||
bucket, pattern string
|
||||
ch chan agent.Update
|
||||
}
|
||||
|
||||
func newPatternBus() *patternBus { return &patternBus{} }
|
||||
|
||||
func (b *patternBus) watch(_ context.Context, bucket, pattern string) (<-chan agent.Update, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if b.absent[bucket] {
|
||||
return nil, fmt.Errorf("nats: bucket not found")
|
||||
}
|
||||
w := patternWatch{bucket: bucket, pattern: pattern, ch: make(chan agent.Update, 64)}
|
||||
b.watches = append(b.watches, w)
|
||||
if b.replay {
|
||||
for key, u := range b.kv[bucket] {
|
||||
if matches(pattern, key) {
|
||||
w.ch <- u
|
||||
}
|
||||
}
|
||||
w.ch <- agent.Update{Caught: true}
|
||||
}
|
||||
return w.ch, nil
|
||||
}
|
||||
|
||||
func matches(pattern, key string) bool {
|
||||
p, k := strings.Split(pattern, "."), strings.Split(key, ".")
|
||||
for i, tok := range p {
|
||||
if tok == ">" {
|
||||
return len(k) > i
|
||||
}
|
||||
if i >= len(k) || (tok != "*" && tok != k[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(p) == len(k)
|
||||
}
|
||||
|
||||
func (b *patternBus) send(bucket string, u agent.Update) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
if !u.Caught {
|
||||
if b.kv == nil {
|
||||
b.kv = map[string]map[string]agent.Update{}
|
||||
}
|
||||
if b.kv[bucket] == nil {
|
||||
b.kv[bucket] = map[string]agent.Update{}
|
||||
}
|
||||
if u.Deleted {
|
||||
delete(b.kv[bucket], u.Key)
|
||||
} else {
|
||||
b.kv[bucket][u.Key] = u
|
||||
}
|
||||
}
|
||||
for _, w := range b.watches {
|
||||
if w.bucket == bucket && (u.Caught || matches(w.pattern, u.Key)) {
|
||||
select {
|
||||
case w.ch <- u:
|
||||
default: // a watch of a run that ended reads no more
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *patternBus) put(bucket, key string, value []byte, rev uint64) {
|
||||
b.send(bucket, agent.Update{Key: key, Value: value, Revision: rev})
|
||||
}
|
||||
|
||||
func (b *patternBus) del(bucket, key string) { b.send(bucket, agent.Update{Key: key, Deleted: true}) }
|
||||
|
||||
// made waits until n watches are made, so nothing put is lost.
|
||||
func (b *patternBus) made(n int) {
|
||||
for {
|
||||
b.mu.Lock()
|
||||
got := len(b.watches)
|
||||
b.mu.Unlock()
|
||||
if got >= n {
|
||||
return
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// caught says every watch caught up.
|
||||
func (b *patternBus) caught() {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
for _, w := range b.watches {
|
||||
w.ch <- agent.Update{Caught: true}
|
||||
}
|
||||
}
|
||||
|
||||
// fixedReader is frombus's Reader over values held, for composing what a node runs.
|
||||
type fixedReader struct {
|
||||
placements map[string]delivery.Placement
|
||||
records map[string]delivery.NodeRecord
|
||||
modules map[string]delivery.Module
|
||||
manifests map[string][]byte
|
||||
}
|
||||
|
||||
func (r fixedReader) Placements(context.Context) (map[string]delivery.Placement, error) {
|
||||
return r.placements, nil
|
||||
}
|
||||
func (r fixedReader) Records(context.Context) (map[string]delivery.NodeRecord, error) {
|
||||
return r.records, nil
|
||||
}
|
||||
func (r fixedReader) Modules(context.Context) (map[string]delivery.Module, error) {
|
||||
return r.modules, nil
|
||||
}
|
||||
func (r fixedReader) Manifest(_ context.Context, d string) ([]byte, error) {
|
||||
if raw, ok := r.manifests[d]; ok {
|
||||
return raw, nil
|
||||
}
|
||||
return nil, fmt.Errorf("no manifest %s", d)
|
||||
}
|
||||
func (r fixedReader) Values(context.Context, string, string) (map[string]any, error) { return nil, nil }
|
||||
func (r fixedReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (r fixedReader) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
@@ -0,0 +1,376 @@
|
||||
package selfcompose
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
)
|
||||
|
||||
// Shadow is the node's answer to placement candidates (novox/hq ADR 0559 §4, issue 1126): for every group with a key in
|
||||
// the indexer's state `placement-candidates`, it composes this node in shadow from the trunk's placement with the group's
|
||||
// keys laid over it and its own inputs (the sealed values and boxes it holds among them), compares the result with what
|
||||
// the node runs, and hands the answers to be written into the node's own have key, `<node>.node`, under `candidates`.
|
||||
// It applies nothing and hands no composition: what runs is still composed from `placement` alone, by the Composer.
|
||||
//
|
||||
// **Event-driven, like the Composer.** It watches every candidate key, the trunk's placement, every node's record, the
|
||||
// indexer's modules, this node's config values, and this node's module haves (what it runs moved); a burst of changes
|
||||
// answers once. It runs whether or not the node is placed yet: a node that applies the controller's compositions answers
|
||||
// against what it runs from them. An answer is handed only when it changes, and a group's answer is dropped when its
|
||||
// keys go.
|
||||
type Shadow struct {
|
||||
c *Composer
|
||||
answer func(ctx context.Context, answers map[string]delivery.CandidateAnswer)
|
||||
|
||||
mu sync.Mutex
|
||||
candidates map[string]map[string]candidateKey // by group, by node
|
||||
answers map[string]delivery.CandidateAnswer
|
||||
}
|
||||
|
||||
// candidateKey is one candidate key as last read: its value and revision.
|
||||
type candidateKey struct {
|
||||
c delivery.PlacementCandidate
|
||||
rev uint64
|
||||
}
|
||||
|
||||
// NewShadow is the shadow composer of one node; answer writes the node's answers into its own have key.
|
||||
func NewShadow(env Env, answer func(ctx context.Context, answers map[string]delivery.CandidateAnswer)) *Shadow {
|
||||
return &Shadow{c: New(env), answer: answer, candidates: map[string]map[string]candidateKey{},
|
||||
answers: map[string]delivery.CandidateAnswer{}}
|
||||
}
|
||||
|
||||
// Keep runs the shadow until ctx ends, beginning again after backoff(try) whenever it stops: an input it cannot watch
|
||||
// yet (the candidates' state not made, or the node's grant of it not loaded at a delivery) or a watch that ended is
|
||||
// tried again rather than leaving the node answering nothing for the rest of the link (review of L2b, round 1, finding
|
||||
// 2). Each distinct reason is said once.
|
||||
func (s *Shadow) Keep(ctx context.Context, backoff func(try int) time.Duration) {
|
||||
said := ""
|
||||
for try := 1; ctx.Err() == nil; try++ {
|
||||
began := time.Now()
|
||||
err := s.Run(ctx)
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if time.Since(began) > time.Minute {
|
||||
try = 1 // it ran: a later stop begins the waits again
|
||||
}
|
||||
why := "it stopped"
|
||||
if err != nil {
|
||||
why = err.Error()
|
||||
}
|
||||
if why != said {
|
||||
s.c.env.Say("this node answers no placement candidate for now, and tries again: " + why)
|
||||
said = why
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-time.After(backoff(try)):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Run answers until ctx ends or an input's watch ends.
|
||||
func (s *Shadow) Run(ctx context.Context) error {
|
||||
// Each run watches afresh and starts from nothing read: what changed while it was down (a candidate key deleted)
|
||||
// is then not missed, and the watches of a run before are not left open (review of L2b, round 2).
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
s.mu.Lock()
|
||||
s.candidates = map[string]map[string]candidateKey{}
|
||||
s.c = New(s.c.env)
|
||||
s.mu.Unlock()
|
||||
c := s.c
|
||||
node := c.env.Node
|
||||
ins := []input{
|
||||
{delivery.PlacementCandidatesBucket, ">", nil, nil},
|
||||
{delivery.PlacementBucket, ">", (*Composer).takePlacement, nil},
|
||||
{delivery.HaveBucket, "*." + delivery.HaveNodeWord, nil, nil},
|
||||
{delivery.ModulesBucket, ">", (*Composer).takeModule, nil},
|
||||
{ResolvedBucket, node + ".>", (*Composer).takeValue, nil},
|
||||
// What this node runs moved: an apply ends in its module's have.
|
||||
{delivery.HaveBucket, node + ".>", nil, nil},
|
||||
}
|
||||
type tagged struct {
|
||||
in int
|
||||
u agent.Update
|
||||
ok bool
|
||||
}
|
||||
merged := make(chan tagged, 64)
|
||||
for i, in := range ins {
|
||||
ch, err := c.env.Watch(ctx, in.bucket, in.pattern)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the node answers placement candidates from %s, which cannot be watched: %w", in.bucket, err)
|
||||
}
|
||||
go func(i int, ch <-chan agent.Update) {
|
||||
for u := range ch {
|
||||
select {
|
||||
case merged <- tagged{in: i, u: u, ok: true}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
select {
|
||||
case merged <- tagged{in: i}:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}(i, ch)
|
||||
}
|
||||
caught := make([]bool, len(ins))
|
||||
ready, dirty := false, false
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case t := <-merged:
|
||||
if !t.ok {
|
||||
return fmt.Errorf("the watch of %s ended; the node answers no placement candidate until its agent starts again", ins[t.in].bucket)
|
||||
}
|
||||
switch {
|
||||
case t.u.Caught:
|
||||
caught[t.in] = true
|
||||
case t.in == 0:
|
||||
dirty = s.takeCandidate(t.u) || dirty
|
||||
case t.in == 2:
|
||||
dirty = s.takeRecord(t.u) || dirty
|
||||
case t.in == 5:
|
||||
c.takeModuleHave(t.u)
|
||||
dirty = !isNodeKey(t.u.Key) || dirty
|
||||
default:
|
||||
ins[t.in].take(c, t.u)
|
||||
dirty = true
|
||||
}
|
||||
if len(merged) > 0 {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if !ready {
|
||||
if ready = all(caught); !ready {
|
||||
continue
|
||||
}
|
||||
dirty = true
|
||||
}
|
||||
if dirty {
|
||||
dirty = false
|
||||
s.answerAll(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isNodeKey(key string) bool {
|
||||
_, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
return err == nil && isNode
|
||||
}
|
||||
|
||||
// takeCandidate takes one candidate key, and says whether it changed anything.
|
||||
func (s *Shadow) takeCandidate(u agent.Update) bool {
|
||||
group, node, err := delivery.ParsePlacementCandidateKey(u.Key)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if u.Deleted {
|
||||
if _, ok := s.candidates[group][node]; !ok {
|
||||
return false
|
||||
}
|
||||
delete(s.candidates[group], node)
|
||||
if len(s.candidates[group]) == 0 {
|
||||
delete(s.candidates, group)
|
||||
}
|
||||
return true
|
||||
}
|
||||
c, err := delivery.DecodePlacementCandidate(u.Value)
|
||||
if err != nil || c.Group != group || (c.Placement != nil && c.Placement.Node != node) {
|
||||
// Not one: read as gone.
|
||||
if _, ok := s.candidates[group][node]; ok {
|
||||
delete(s.candidates[group], node)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
if s.candidates[group] == nil {
|
||||
s.candidates[group] = map[string]candidateKey{}
|
||||
}
|
||||
s.candidates[group][node] = candidateKey{c: c, rev: u.Revision}
|
||||
return true
|
||||
}
|
||||
|
||||
// takeRecord takes a node's record, and says whether it changed: the node's own key changes with each answer written,
|
||||
// which moves nothing composed.
|
||||
func (s *Shadow) takeRecord(u agent.Update) bool {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(u.Key)
|
||||
if err != nil || !isNode {
|
||||
return false
|
||||
}
|
||||
c := s.c
|
||||
c.mu.Lock()
|
||||
before, had := c.records[node]
|
||||
c.mu.Unlock()
|
||||
c.takeRecord(u)
|
||||
c.mu.Lock()
|
||||
after, has := c.records[node]
|
||||
c.mu.Unlock()
|
||||
return had != has || (has && !delivery.SameNodeRecord(before, after))
|
||||
}
|
||||
|
||||
// answerAll answers every group and hands the answers when any changed.
|
||||
func (s *Shadow) answerAll(ctx context.Context) {
|
||||
c := s.c
|
||||
node := c.env.Node
|
||||
s.mu.Lock()
|
||||
groups := make([]string, 0, len(s.candidates))
|
||||
for g := range s.candidates {
|
||||
groups = append(groups, g)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
sort.Strings(groups)
|
||||
next := map[string]delivery.CandidateAnswer{}
|
||||
for _, group := range groups {
|
||||
if a, ok := s.answerGroup(ctx, group); ok {
|
||||
next[group] = a
|
||||
}
|
||||
}
|
||||
s.mu.Lock()
|
||||
same := len(next) == len(s.answers)
|
||||
for g, a := range next {
|
||||
if old, ok := s.answers[g]; !ok || !sameAnswer(old, a) {
|
||||
same = false
|
||||
}
|
||||
}
|
||||
if !same {
|
||||
s.answers = next
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if !same && s.answer != nil {
|
||||
out := make(map[string]delivery.CandidateAnswer, len(next))
|
||||
for g, a := range next {
|
||||
out[g] = a
|
||||
}
|
||||
s.answer(ctx, out)
|
||||
c.env.Say(fmt.Sprintf("%s answered %d placement candidate group(s)", node, len(out)))
|
||||
}
|
||||
}
|
||||
|
||||
// answerGroup is the node's answer to one group, and false when the node has nothing to answer: neither the trunk nor
|
||||
// the group places it.
|
||||
func (s *Shadow) answerGroup(ctx context.Context, group string) (delivery.CandidateAnswer, bool) {
|
||||
c := s.c
|
||||
node := c.env.Node
|
||||
s.mu.Lock()
|
||||
keys := s.candidates[group]
|
||||
given := make(map[string]*delivery.Placement, len(keys))
|
||||
var head string
|
||||
for n, k := range keys {
|
||||
given[n] = k.c.Placement
|
||||
head = k.c.Pull.Head
|
||||
}
|
||||
// The revision answered is the node's own key's, 0 when the group has none for it (judged by the head alone).
|
||||
own, mine := keys[node]
|
||||
rev := own.rev
|
||||
s.mu.Unlock()
|
||||
|
||||
c.mu.Lock()
|
||||
_, placed := c.placements[node]
|
||||
modules := make(map[string]delivery.Module, len(c.modules))
|
||||
for k, v := range c.modules {
|
||||
modules[k] = v
|
||||
}
|
||||
c.mu.Unlock()
|
||||
if !placed && !mine {
|
||||
return delivery.CandidateAnswer{}, false
|
||||
}
|
||||
|
||||
var running []delivery.Composed
|
||||
if c.env.Running != nil {
|
||||
running = c.env.Running()
|
||||
}
|
||||
if p, ok := given[node]; ok && p == nil {
|
||||
// The head removes this node's placement file: everything it runs is removed.
|
||||
a := delivery.CandidateAnswer{Head: head, Revision: rev}
|
||||
for _, r := range running {
|
||||
if r.Module == "" {
|
||||
a.Removed = append(a.Removed, delivery.HaveNodeWord)
|
||||
} else {
|
||||
a.Removed = append(a.Removed, r.Module)
|
||||
}
|
||||
}
|
||||
sort.Strings(a.Removed)
|
||||
a.Verdict = delivery.VerdictOf(a)
|
||||
return a, true
|
||||
}
|
||||
|
||||
placement := given[node]
|
||||
if placement == nil {
|
||||
c.mu.Lock()
|
||||
p := c.placements[node]
|
||||
c.mu.Unlock()
|
||||
placement = &p
|
||||
}
|
||||
// What it runs inside its own composition, of what the candidate places (the seed's trunk places nothing).
|
||||
c.mu.Lock()
|
||||
inOwn := c.inOwnOf(*placement)
|
||||
c.mu.Unlock()
|
||||
fps := shadowBuilds(running, *placement, modules, inOwn)
|
||||
kept := map[string]string{}
|
||||
for module, fp := range fps {
|
||||
if mod, ok := modules[module]; ok && mod.FP2 == fp {
|
||||
continue
|
||||
}
|
||||
digest, err := c.builtManifest(ctx, module, fp)
|
||||
if err != nil {
|
||||
return delivery.CandidateAnswer{Head: head, Revision: rev, Verdict: delivery.CandidateRefused,
|
||||
Refused: fmt.Sprintf("%s runs at %s, whose manifest cannot be read: %v", module, fp, err)}, true
|
||||
}
|
||||
kept[module] = digest
|
||||
}
|
||||
// The boxes it runs, by what each module's have and its own key say after a restart (Held), are reused for the
|
||||
// modules it runs: a value sealed again is no difference.
|
||||
held := map[string]delivery.Want{agent.NodeModule: {}}
|
||||
for module := range fps {
|
||||
held[module] = delivery.Want{FP: fps[module]}
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.wants = held
|
||||
c.mu.Unlock()
|
||||
a := frombus.Answer(ctx, reader{c: c}, c.env.Mesh, frombus.Scope{Node: node, Kept: kept, Placements: given, InOwn: inOwn}, fps, running, head, rev)
|
||||
return a, true
|
||||
}
|
||||
|
||||
// shadowBuilds is the build each module placed is composed at in shadow: the one it runs, and for one the candidate
|
||||
// adds, the trunk's. A module it runs inside its own composition (inOwn, from its haves: mesh-wireguard, one at a build
|
||||
// of the old form) is given none, so it is folded there again, as the node's own composer folds it, rather than
|
||||
// composed at the trunk's build as if the candidate added it (issue 1126: nats and photos have a trunk build that no
|
||||
// want names).
|
||||
func shadowBuilds(running []delivery.Composed, placement delivery.Placement, modules map[string]delivery.Module, inOwn []string) map[string]string {
|
||||
fps := map[string]string{}
|
||||
for _, r := range running {
|
||||
if r.Module != "" && r.FP != "" {
|
||||
fps[r.Module] = r.FP
|
||||
}
|
||||
}
|
||||
folded := map[string]bool{}
|
||||
for _, m := range inOwn {
|
||||
folded[m] = true
|
||||
}
|
||||
for _, m := range placement.Modules {
|
||||
if _, runs := fps[m.Module]; !runs && !folded[m.Module] {
|
||||
if mod, ok := modules[m.Module]; ok && mod.FP2 != "" {
|
||||
fps[m.Module] = mod.FP2
|
||||
}
|
||||
}
|
||||
}
|
||||
return fps
|
||||
}
|
||||
|
||||
func sameAnswer(a, b delivery.CandidateAnswer) bool {
|
||||
x, errA := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"g": a}})
|
||||
y, errB := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"g": b}})
|
||||
return errA == nil && errB == nil && string(x) == string(y)
|
||||
}
|
||||
@@ -20,6 +20,13 @@
|
||||
// (VaultKey), the have of the bus module carries the bcrypt hash of every user of the list on the machine, sealed to
|
||||
// that key (delivery.AccountsHanded), so the vault adopts the accounts it holds no value for. It is sealed again only
|
||||
// when the list on the machine or the vault's key changes.
|
||||
//
|
||||
// **From take-over it follows the grants part's list instead** (ADR 0547 clauses 7 and 8, issue 1945). On the node whose
|
||||
// compositions carry the list, it also watches the grants part's one key `user-list.sealed`. While that key says
|
||||
// FollowGrants, the list is the object the key names, read by its name, opened with this node's sealing key and checked
|
||||
// against the key's digest, and written to the same file (mode 0600) only when the file's digest differs; otherwise the
|
||||
// controller's compositions are the source, as before. Never both. The have says which it follows and this node's key,
|
||||
// which the grants part seals to.
|
||||
package userlist
|
||||
|
||||
import (
|
||||
@@ -36,6 +43,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
"git.novox.be/novox/mesh-sdk/go/seal"
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
)
|
||||
|
||||
@@ -64,6 +72,19 @@ type Env struct {
|
||||
Write func(ctx context.Context, module string, resource json.RawMessage, body []byte) (digest string, changed bool, err error)
|
||||
// Report says what was written, in the module's have.
|
||||
Report func(ctx context.Context, module string, w Written)
|
||||
// Grants watches the grants part's state (its key delivery.GrantsUserListKey); nil follows the controller's alone.
|
||||
Grants agent.Watcher
|
||||
// Fetch reads an object of the grants part's bucket by its name; Open opens a box sealed to this node's key; Key is
|
||||
// this node's sealing key's public half (SPKI PEM), said in the have so the grants part seals to it ("" unknown).
|
||||
Fetch func(ctx context.Context, name string) ([]byte, error)
|
||||
Open func(seal.Box) ([]byte, error)
|
||||
Key func() string
|
||||
// Kept is the grants part's key as last heard on this machine and its revision (nil when none), and Keep keeps it:
|
||||
// what the follower follows while the key cannot be watched (ADR 0547 clause 8). Backoff is the wait before the
|
||||
// try-th watch again.
|
||||
Kept func() ([]byte, uint64)
|
||||
Keep func(value []byte, rev uint64)
|
||||
Backoff func(try int) time.Duration
|
||||
// VaultKey is the vault's own public key (SPKI PEM) as its answer to node-key gave it, "" while none is known: the
|
||||
// key the loaded list's hashes are handed sealed to. Nil hands nothing.
|
||||
VaultKey func() string
|
||||
@@ -91,6 +112,13 @@ type Follower struct {
|
||||
state Written
|
||||
mod string
|
||||
said map[string]bool
|
||||
// grants is the grants part's key as last heard (nil when none), and following the source last taken: FollowGrants
|
||||
// or FollowController.
|
||||
grants *delivery.GrantsUserList
|
||||
grantsRev uint64
|
||||
following string
|
||||
// unwatched is why the grants part's key is not watched now; empty while it is, or where it need not be.
|
||||
unwatched string
|
||||
}
|
||||
|
||||
// New is a follower for a node.
|
||||
@@ -101,7 +129,18 @@ func New(env Env) *Follower {
|
||||
if env.Say == nil {
|
||||
env.Say = func(string) {}
|
||||
}
|
||||
return &Follower{env: env, kick: make(chan struct{}, 1), srcs: map[string]source{}, said: map[string]bool{}}
|
||||
if env.Backoff == nil {
|
||||
env.Backoff = func(int) time.Duration { return time.Minute }
|
||||
}
|
||||
f := &Follower{env: env, kick: make(chan struct{}, 1), srcs: map[string]source{}, said: map[string]bool{}}
|
||||
if env.Grants != nil && env.Kept != nil {
|
||||
if raw, rev := env.Kept(); raw != nil {
|
||||
if g, err := delivery.DecodeGrantsUserList(raw); err == nil {
|
||||
f.grants, f.grantsRev = &g, rev
|
||||
}
|
||||
}
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// Again asks the follower to look at the file again, though no composition changed: after an apply of the bus module,
|
||||
@@ -128,6 +167,33 @@ func (f *Follower) Run(ctx context.Context) error {
|
||||
return fmt.Errorf("the user list cannot be followed: %w", err)
|
||||
}
|
||||
caught := false
|
||||
// The grants part's key, on the node whose compositions carry the list: the bus node alone is granted it (mesh-sdk
|
||||
// buscompose.GrantsUserListFollows). Until its watch caught up nothing is written, so a new follower (one per link)
|
||||
// never writes the controller's list after take-over before it heard the switch; a watch refused or ended is tried
|
||||
// again on the backoff, and meanwhile the follow last heard (kept on the machine) holds.
|
||||
var grants <-chan agent.Update
|
||||
var retry <-chan time.Time
|
||||
opened, grantsCaught, tries := false, false, 0
|
||||
failed := func(why string) {
|
||||
grants, tries = nil, tries+1
|
||||
retry = time.After(f.env.Backoff(tries))
|
||||
f.mu.Lock()
|
||||
f.unwatched = why
|
||||
f.mu.Unlock()
|
||||
f.sayOnce("the grants part's user-list key is not watched, tried again: " + why)
|
||||
}
|
||||
open := func() {
|
||||
if grants != nil || retry != nil || f.env.Grants == nil || !f.busNode() {
|
||||
return
|
||||
}
|
||||
opened = true
|
||||
ch, err := f.env.Grants.Watch(ctx, delivery.GrantsUserListKey)
|
||||
if err != nil {
|
||||
failed(err.Error())
|
||||
return
|
||||
}
|
||||
grants = ch
|
||||
}
|
||||
for {
|
||||
again := false
|
||||
select {
|
||||
@@ -135,6 +201,8 @@ func (f *Follower) Run(ctx context.Context) error {
|
||||
return nil
|
||||
case <-f.kick:
|
||||
again = true
|
||||
case <-retry:
|
||||
retry = nil
|
||||
case u, ok := <-updates:
|
||||
if !ok {
|
||||
return fmt.Errorf("the watch of the compositions ended")
|
||||
@@ -144,13 +212,99 @@ func (f *Follower) Run(ctx context.Context) error {
|
||||
} else {
|
||||
f.take(u)
|
||||
}
|
||||
case u, ok := <-grants:
|
||||
if !ok {
|
||||
failed("its watch ended")
|
||||
break
|
||||
}
|
||||
if u.Caught {
|
||||
grantsCaught, tries = true, 0
|
||||
f.mu.Lock()
|
||||
f.unwatched = ""
|
||||
f.mu.Unlock()
|
||||
break
|
||||
}
|
||||
f.takeGrants(u)
|
||||
}
|
||||
if caught {
|
||||
f.settle(ctx, again)
|
||||
if !caught {
|
||||
continue
|
||||
}
|
||||
open()
|
||||
if opened && !grantsCaught && grants != nil {
|
||||
continue // the switch is not heard yet: nothing is written before it is
|
||||
}
|
||||
f.mu.Lock()
|
||||
if f.unwatched != f.state.Unwatched {
|
||||
again = true // the have says whether the key is watched
|
||||
}
|
||||
f.mu.Unlock()
|
||||
f.settle(ctx, again)
|
||||
}
|
||||
}
|
||||
|
||||
// busNode says some composition of this node carries the bus's user list.
|
||||
func (f *Follower) busNode() bool {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.cur.key != "" || len(f.srcs) > 0
|
||||
}
|
||||
|
||||
// takeGrants keeps the grants part's key as heard; a value it cannot read is said and leaves the controller's list
|
||||
// followed (no list is followed that the node cannot check).
|
||||
func (f *Follower) takeGrants(u agent.Update) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.grantsRev = u.Revision
|
||||
if u.Deleted {
|
||||
f.grants = nil
|
||||
f.keep(nil, 0)
|
||||
return
|
||||
}
|
||||
g, err := delivery.DecodeGrantsUserList(u.Value)
|
||||
if err != nil {
|
||||
f.grants = nil
|
||||
f.keep(nil, 0)
|
||||
f.sayOnceLocked("the grants part's user-list key cannot be read, so the controller's list is followed: " + err.Error())
|
||||
return
|
||||
}
|
||||
f.grants = &g
|
||||
f.keep(u.Value, u.Revision)
|
||||
}
|
||||
|
||||
// keep keeps the key last heard on the machine (Env.Keep), so a follower that cannot watch it follows what it said.
|
||||
func (f *Follower) keep(value []byte, rev uint64) {
|
||||
if f.env.Keep != nil {
|
||||
f.env.Keep(value, rev)
|
||||
}
|
||||
}
|
||||
|
||||
// Following is which list the follower takes now: the grants part's while its key says so, the controller's otherwise.
|
||||
func (f *Follower) Following() string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.followingLocked()
|
||||
}
|
||||
|
||||
func (f *Follower) followingLocked() string {
|
||||
if f.grants != nil && f.grants.Follow == delivery.FollowGrants {
|
||||
return delivery.FollowGrants
|
||||
}
|
||||
return delivery.FollowController
|
||||
}
|
||||
|
||||
func (f *Follower) unwatchedNow() string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.unwatched
|
||||
}
|
||||
|
||||
func (f *Follower) key() string {
|
||||
if f.env.Key == nil {
|
||||
return ""
|
||||
}
|
||||
return f.env.Key()
|
||||
}
|
||||
|
||||
// take keeps a composition key's change when it carries the bus's user list, and forgets a key deleted.
|
||||
func (f *Follower) take(u agent.Update) {
|
||||
node, module, _, isNode, err := delivery.ParseComposedKey(u.Key)
|
||||
@@ -200,6 +354,17 @@ func (f *Follower) take(u agent.Update) {
|
||||
// A source never goes back: a key deleted, or a newer one refused, leaves the list written as it is, since an older
|
||||
// list would take away accounts a newer one gave.
|
||||
func (f *Follower) settle(ctx context.Context, again bool) {
|
||||
f.mu.Lock()
|
||||
follow := f.followingLocked()
|
||||
if follow != f.following {
|
||||
// The switch turned: the other source is written, though neither changed.
|
||||
again = true
|
||||
}
|
||||
f.mu.Unlock()
|
||||
if follow == delivery.FollowGrants {
|
||||
f.settleGrants(ctx, again)
|
||||
return
|
||||
}
|
||||
f.mu.Lock()
|
||||
newest := f.cur
|
||||
for _, s := range f.srcs {
|
||||
@@ -215,7 +380,8 @@ func (f *Follower) settle(ctx context.Context, again bool) {
|
||||
at := fmt.Sprintf("%s@%d", newest.key, newest.rev)
|
||||
f.mu.Unlock()
|
||||
|
||||
w := Written{Source: newest.key, Revision: newest.rev, Wanted: newest.wanted, At: f.env.Now().UTC()}
|
||||
w := Written{Source: newest.key, Revision: newest.rev, Wanted: newest.wanted, At: f.env.Now().UTC(),
|
||||
Follows: delivery.FollowController, Key: f.key(), Unwatched: f.unwatchedNow()}
|
||||
var changed bool
|
||||
body, err := f.env.Resolve(ctx, newest.resource)
|
||||
if err == nil {
|
||||
@@ -248,6 +414,7 @@ func (f *Follower) settle(ctx context.Context, again bool) {
|
||||
f.mu.Lock()
|
||||
f.mod, f.state = newest.module, w
|
||||
f.written = err == nil
|
||||
f.following = delivery.FollowController
|
||||
f.mu.Unlock()
|
||||
if err != nil {
|
||||
f.sayOnce(fmt.Sprintf("the bus's user list of %s could not be written: %v", at, err))
|
||||
@@ -256,12 +423,149 @@ func (f *Follower) settle(ctx context.Context, again bool) {
|
||||
at, w.Users, short(w.Digest)))
|
||||
}
|
||||
// Reported when what it says changed: a look again that found the file as it was says nothing new.
|
||||
if f.env.Report != nil && (changed || prev.Source != w.Source || prev.Revision != w.Revision ||
|
||||
prev.Digest != w.Digest || prev.Error != w.Error || handedKey(prev.Handed) != handedKey(w.Handed)) {
|
||||
if f.env.Report != nil && (changed || reportable(prev, w)) {
|
||||
f.env.Report(ctx, newest.module, w)
|
||||
}
|
||||
}
|
||||
|
||||
// reportable says a have's user list says something new: a look again that found the file as it was says nothing.
|
||||
func reportable(prev, w Written) bool {
|
||||
return prev.Source != w.Source || prev.Revision != w.Revision || prev.Digest != w.Digest || prev.Error != w.Error ||
|
||||
prev.Follows != w.Follows || prev.Key != w.Key || handedKey(prev.Handed) != handedKey(w.Handed) ||
|
||||
prev.Unwatched != w.Unwatched
|
||||
}
|
||||
|
||||
// settleGrants writes the grants part's list, the object its key names, into the file the controller's composition of
|
||||
// the bus module declares, made 0600 (ADR 0547 clause 7): when the key moved, when the switch turned, or when asked.
|
||||
func (f *Follower) settleGrants(ctx context.Context, again bool) {
|
||||
f.mu.Lock()
|
||||
g, rev := *f.grants, f.grantsRev
|
||||
file := f.cur
|
||||
for _, s := range f.srcs {
|
||||
if s.rev > file.rev {
|
||||
file = s
|
||||
}
|
||||
}
|
||||
if !again && f.following == delivery.FollowGrants && f.state.Revision == rev && f.written {
|
||||
f.mu.Unlock()
|
||||
return
|
||||
}
|
||||
f.mu.Unlock()
|
||||
|
||||
w := Written{Source: delivery.GrantsUserListSource, Revision: rev, Wanted: g.Digest, At: f.env.Now().UTC(),
|
||||
Follows: delivery.FollowGrants, Key: f.key(), Unwatched: f.unwatchedNow()}
|
||||
var err error
|
||||
var changed bool
|
||||
resource, rerr := mode0600(file.resource)
|
||||
switch {
|
||||
case file.key == "" || rerr != nil:
|
||||
err = fmt.Errorf("no composition of this node declares the bus's user-list file, so the grants part's list has nowhere to be written")
|
||||
case g.Digest == "":
|
||||
err = fmt.Errorf("the grants part's key names no list yet; the list on the machine is kept")
|
||||
case f.env.Fetch == nil || f.env.Open == nil:
|
||||
err = fmt.Errorf("this engine cannot read or open the grants part's list")
|
||||
case g.SealedTo != "" && w.Key != "" && g.SealedTo != delivery.SealingKeyID(w.Key):
|
||||
err = fmt.Errorf("the grants part's list is sealed to another key than this node's (%s); the list on the machine is kept until it seals to this one", g.SealedTo)
|
||||
}
|
||||
var body []byte
|
||||
if err == nil {
|
||||
var object []byte
|
||||
if object, err = f.env.Fetch(ctx, g.Object); err == nil {
|
||||
body, err = delivery.OpenUserList(object, g.Digest, f.env.Open)
|
||||
}
|
||||
if err != nil {
|
||||
err = fmt.Errorf("the grants part's list %s could not be had: %w", short(g.Digest), err)
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
w.Users = Users(body)
|
||||
if err = f.guard(resource, body); err != nil {
|
||||
if disk, derr := f.env.OnDisk(resource); derr == nil && disk != nil {
|
||||
w.Digest = Digest(disk)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
w.Digest, changed, err = f.env.Write(ctx, file.module, resource, body)
|
||||
} else if w.Digest == "" && file.key != "" && f.env.OnDisk != nil {
|
||||
if disk, derr := f.env.OnDisk(resource); derr == nil && disk != nil {
|
||||
w.Digest = Digest(disk)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
w.Error = err.Error()
|
||||
}
|
||||
f.mu.Lock()
|
||||
prev := f.state
|
||||
// The grants part's list is not handed to the vault: the handing is of what the controller's list loaded (clause 9).
|
||||
w.Handed = prev.Handed
|
||||
f.mod, f.state = file.module, w
|
||||
f.written = err == nil
|
||||
f.following = delivery.FollowGrants
|
||||
f.mu.Unlock()
|
||||
at := fmt.Sprintf("%s@%d", delivery.GrantsUserListSource, rev)
|
||||
if err != nil {
|
||||
f.sayOnce(fmt.Sprintf("the bus's user list of %s could not be written: %v", at, err))
|
||||
} else if changed {
|
||||
f.env.Say(fmt.Sprintf("the bus's user list from the grants part (%s, revision %d) is written alone (%d users, %s); the bus reloads it itself",
|
||||
at, g.Revision, w.Users, short(w.Digest)))
|
||||
}
|
||||
if f.env.Report != nil && file.module != "" && (changed || reportable(prev, w)) {
|
||||
f.env.Report(ctx, file.module, w)
|
||||
}
|
||||
}
|
||||
|
||||
// Hold is c as the apply may write it (C3 of the switch's review): while the grants part's list is followed, the bus
|
||||
// module's composition from the controller carries the controller's list, so its user-list resource holds the list on the
|
||||
// machine instead (made 0600), or is left out when the machine holds none. A list on the machine that cannot be read
|
||||
// is an error: left out, the apply would remove the file as an orphan and the bus would refuse every connection at its
|
||||
// next reload, so the bus module's apply waits instead. Any other composition, or the controller's list followed, is c
|
||||
// unchanged.
|
||||
func (f *Follower) Hold(c delivery.Composed) (delivery.Composed, error) {
|
||||
if f.Following() != delivery.FollowGrants {
|
||||
return c, nil
|
||||
}
|
||||
out := c
|
||||
out.Resources = nil
|
||||
for _, raw := range c.Resources {
|
||||
if r, _, _, ok := userList([]json.RawMessage{raw}); ok {
|
||||
var held json.RawMessage
|
||||
body, err := f.env.OnDisk(r)
|
||||
if err != nil {
|
||||
return c, fmt.Errorf("the user list on the machine could not be read, so the apply waits and the file stays: %w", err)
|
||||
}
|
||||
if body != nil {
|
||||
var m map[string]any
|
||||
if json.Unmarshal(r, &m) == nil {
|
||||
delete(m, "digest")
|
||||
delete(m, "size")
|
||||
m["content"], m["mode"] = string(body), "0600"
|
||||
held, _ = json.Marshal(m)
|
||||
}
|
||||
}
|
||||
if held != nil {
|
||||
out.Resources = append(out.Resources, held)
|
||||
}
|
||||
continue
|
||||
}
|
||||
out.Resources = append(out.Resources, raw)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// mode0600 is a user-list resource made mode 0600, whatever its composition said (ADR 0547 clause 7).
|
||||
func mode0600(raw json.RawMessage) (json.RawMessage, error) {
|
||||
if len(raw) == 0 {
|
||||
return nil, fmt.Errorf("no resource")
|
||||
}
|
||||
var r map[string]any
|
||||
if err := json.Unmarshal(raw, &r); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r["mode"] = "0600"
|
||||
return json.Marshal(r)
|
||||
}
|
||||
|
||||
// hand is the handing of the list on the machine to the vault: the one before when it is of the same list and key, a
|
||||
// new one sealed now otherwise, and none while the vault's key is not known or the list holds no hash.
|
||||
func (f *Follower) hand(loaded []byte, before *delivery.AccountsHanded) *delivery.AccountsHanded {
|
||||
|
||||
@@ -33,18 +33,29 @@ func fpOf(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 6
|
||||
// feed is a watch the test writes to: the keys there before the watch, Caught, then each put. watching is closed once
|
||||
// the watch began, so a put after it is never handed before the keys there before it.
|
||||
type feed struct {
|
||||
ch chan agent.Update
|
||||
rev uint64
|
||||
first []agent.Update
|
||||
ch chan agent.Update
|
||||
rev uint64
|
||||
first []agent.Update
|
||||
// watching is closed once the watch caught up, so a later put is not read before the keys there before it.
|
||||
watching chan struct{}
|
||||
once sync.Once
|
||||
// err refuses the watch, as the bus refuses a watch it grants nothing of.
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *feed) Watch(context.Context, string) (<-chan agent.Update, error) {
|
||||
if f.err != nil {
|
||||
return nil, f.err
|
||||
}
|
||||
for _, u := range f.first {
|
||||
f.ch <- u
|
||||
}
|
||||
f.ch <- agent.Update{Caught: true}
|
||||
close(f.watching)
|
||||
f.once.Do(func() {
|
||||
if f.watching != nil {
|
||||
close(f.watching)
|
||||
}
|
||||
})
|
||||
return f.ch, nil
|
||||
}
|
||||
|
||||
@@ -114,6 +125,15 @@ func comp(t *testing.T, module, fp, content string, signer ed25519.PrivateKey, p
|
||||
return k, raw
|
||||
}
|
||||
|
||||
// gput is the grants part's key as the bus node hears it at the watch's start (ADR 0547 clauses 7, 8): which list to
|
||||
// follow, and the list sealed to the node's key; one sealed to another key, or one that does not open to the key's digest.
|
||||
type gput struct {
|
||||
follow, content string
|
||||
otherKey, badDigest bool
|
||||
// after puts it once the grants part's list was written, as a hand-back comes after a take-over.
|
||||
after bool
|
||||
}
|
||||
|
||||
type put struct {
|
||||
module, fp, content string
|
||||
stranger, deleted bool
|
||||
@@ -135,6 +155,14 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
|
||||
wantUsers int
|
||||
wantErr string
|
||||
wantRefused bool
|
||||
grants []gput
|
||||
// grantsRefused refuses the watch of the grants part's key; kept is the key last heard, kept on the machine;
|
||||
// wantUnwatched says the have then says why it is not watched.
|
||||
grantsRefused bool
|
||||
kept *gput
|
||||
wantUnwatched bool
|
||||
// wantSource and wantFollows are what the last have says it took the list from and follows.
|
||||
wantSource, wantFollows string
|
||||
// vault gives the follower the vault's key; wantHanded is how many users it then hands from the list on the
|
||||
// machine (ADR 0547 clause 9).
|
||||
vault bool
|
||||
@@ -180,6 +208,30 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
|
||||
onDisk: hashed("controller", "a", "b"), wantUsers: 2, wantErr: "holds no user", wantRefused: true, wantHanded: 3},
|
||||
{name: "a failed write is said and kept", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, failWrite: true,
|
||||
wantWrites: []string{two}, wantUsers: 3, wantErr: "disk full"},
|
||||
// ADR 0547 clauses 7, 8: never both lists; the grants part's from take-over, the controller's otherwise.
|
||||
{name: "from take-over the grants part's list is written into the same file",
|
||||
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three}},
|
||||
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
|
||||
{name: "a hand-back writes the newest controller composition once",
|
||||
before: []put{{module: "nats", fp: fpOf("a"), content: two}},
|
||||
grants: []gput{{follow: delivery.FollowGrants, content: three}, {follow: delivery.FollowController, content: three, after: true}},
|
||||
wantWrites: []string{three, two}, wantUsers: 3, wantSource: "novox.nats." + fpOf("a")[len(delivery.SourcePrefix2):], wantFollows: delivery.FollowController},
|
||||
{name: "a refused watch with no key kept follows the controller's list and says it is not watched",
|
||||
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grantsRefused: true, wantUnwatched: true,
|
||||
wantWrites: []string{two}, wantUsers: 3, wantSource: "novox.nats." + fpOf("a")[len(delivery.SourcePrefix2):], wantFollows: delivery.FollowController},
|
||||
{name: "a refused watch keeps the follow last heard, never the controller's list",
|
||||
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grantsRefused: true, wantUnwatched: true,
|
||||
kept: &gput{follow: delivery.FollowGrants, content: three},
|
||||
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
|
||||
{name: "in shadow the grants part's list is not taken",
|
||||
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowController, content: three}},
|
||||
wantWrites: []string{two}, wantUsers: 3, wantSource: "novox.nats." + fpOf("a")[len(delivery.SourcePrefix2):], wantFollows: delivery.FollowController},
|
||||
{name: "a grants list that does not open to its digest is refused and the file kept",
|
||||
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three, badDigest: true}},
|
||||
onDisk: two, wantErr: "could not be had", wantRefused: true, wantSource: delivery.GrantsUserListSource},
|
||||
{name: "a grants list sealed to another key is refused and the file kept",
|
||||
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three, otherKey: true}},
|
||||
onDisk: two, wantErr: "sealed to another key", wantRefused: true, wantSource: delivery.GrantsUserListSource},
|
||||
} {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
f := &feed{ch: make(chan agent.Update, 64), watching: make(chan struct{})}
|
||||
@@ -188,7 +240,66 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
|
||||
var writes []string
|
||||
var reports []Written
|
||||
reported := make(chan struct{}, 64)
|
||||
fl := New(Env{Node: node, Composed: f,
|
||||
gf := &feed{ch: make(chan agent.Update, 64)}
|
||||
if c.grantsRefused {
|
||||
gf.err = errors.New("Permissions Violation for Publish to $JS.API.STREAM.INFO.KV_mesh-grants_user-list")
|
||||
}
|
||||
var grantsAfter []agent.Update
|
||||
var keptRaw []byte
|
||||
sealed := map[string][]byte{}
|
||||
nodeKey, err := seal.GenerateKeyPair()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
gs := c.grants
|
||||
if c.kept != nil {
|
||||
gs = append(append([]gput(nil), gs...), *c.kept)
|
||||
}
|
||||
for i, g := range gs {
|
||||
to := nodeKey.PublicKey
|
||||
if g.otherKey {
|
||||
other, _ := seal.GenerateKeyPair()
|
||||
to = other.PublicKey
|
||||
}
|
||||
digest := delivery.UserListDigest([]byte(g.content))
|
||||
if g.badDigest {
|
||||
digest = delivery.UserListDigest([]byte("another list"))
|
||||
}
|
||||
object, err := delivery.SealUserList([]byte(g.content), to)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sealed[delivery.UserListObjectName(digest)] = object
|
||||
raw, err := delivery.EncodeGrantsUserList(delivery.GrantsUserList{Follow: g.follow, Digest: digest,
|
||||
Object: delivery.UserListObjectName(digest), SealedTo: delivery.SealingKeyID(to), Node: node, Revision: uint64(i + 1)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
u := agent.Update{Key: delivery.GrantsUserListKey, Value: raw, Revision: uint64(i + 1)}
|
||||
switch {
|
||||
case c.kept != nil && i == len(gs)-1:
|
||||
keptRaw = raw
|
||||
case g.after:
|
||||
grantsAfter = append(grantsAfter, u)
|
||||
default:
|
||||
gf.first = append(gf.first, u)
|
||||
}
|
||||
}
|
||||
fl := New(Env{Node: node, Composed: f, Grants: gf,
|
||||
Kept: func() ([]byte, uint64) { return keptRaw, 9 },
|
||||
Keep: func([]byte, uint64) {},
|
||||
Backoff: func(int) time.Duration { return time.Hour },
|
||||
Fetch: func(_ context.Context, name string) ([]byte, error) {
|
||||
if o, ok := sealed[name]; ok {
|
||||
return o, nil
|
||||
}
|
||||
return nil, errors.New("no object " + name)
|
||||
},
|
||||
Open: func(b seal.Box) ([]byte, error) {
|
||||
out, err := seal.Open(b, nodeKey.PrivateKey)
|
||||
return []byte(out), err
|
||||
},
|
||||
Key: func() string { return nodeKey.PublicKey },
|
||||
Trusted: func() []ed25519.PublicKey { return []ed25519.PublicKey{controllerPub} },
|
||||
Resolve: func(_ context.Context, r json.RawMessage) ([]byte, error) {
|
||||
var res struct{ Content, Digest string }
|
||||
@@ -254,9 +365,30 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- fl.Run(ctx) }()
|
||||
<-f.watching
|
||||
// A later put after the first list is written: the first write waits for the grants part's key to catch up.
|
||||
if len(c.after) > 0 {
|
||||
for until := time.Now().Add(5 * time.Second); time.Now().Before(until); time.Sleep(5 * time.Millisecond) {
|
||||
mu.Lock()
|
||||
n := len(writes)
|
||||
mu.Unlock()
|
||||
if n > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, p := range c.after {
|
||||
f.ch <- update(p)
|
||||
}
|
||||
if len(grantsAfter) > 0 {
|
||||
for until := time.Now().Add(5 * time.Second); ; time.Sleep(5 * time.Millisecond) {
|
||||
if _, st := fl.State(); st.Source == delivery.GrantsUserListSource || time.Now().After(until) {
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, u := range grantsAfter {
|
||||
gf.ch <- u
|
||||
}
|
||||
}
|
||||
if c.again {
|
||||
// After the first write is reported, so the look again is not folded into it.
|
||||
select {
|
||||
@@ -286,6 +418,14 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
|
||||
t.Fatal("a refused list was not reported")
|
||||
}
|
||||
}
|
||||
if c.wantSource != "" {
|
||||
for until := time.Now().Add(5 * time.Second); ; {
|
||||
if _, st := fl.State(); st.Source == c.wantSource && (c.wantErr == "" || st.Error != "") || time.Now().After(until) {
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond) // nothing more is written
|
||||
cancel()
|
||||
<-done
|
||||
@@ -295,6 +435,38 @@ func TestTheBusNodeWritesTheNewestSignedUserListAlone(t *testing.T) {
|
||||
t.Fatalf("written %q, want %q", writes, c.wantWrites)
|
||||
}
|
||||
module, st := fl.State()
|
||||
if c.wantUnwatched != (st.Unwatched != "") {
|
||||
t.Fatalf("the have says unwatched %q", st.Unwatched)
|
||||
}
|
||||
// While the grants part's list is followed, an apply of the controller's composition writes not its list.
|
||||
if _, raw := comp(t, "nats", fpOf("a"), two, controllerKey, put{}); c.wantFollows != "" {
|
||||
cmp, err := delivery.DecodeComposed(raw)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
held, err := fl.Hold(cmp)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, _, _, hasList := userList(held.Resources)
|
||||
if want := c.wantFollows == delivery.FollowController; hasList != want || len(held.Resources) < 1 {
|
||||
t.Fatalf("an apply while following %s holds the controller's list: %v", c.wantFollows, hasList)
|
||||
}
|
||||
// A list on the machine that cannot be read stops the apply; it is never left out to be removed.
|
||||
if c.wantFollows == delivery.FollowGrants {
|
||||
readErr := New(Env{Node: node, Composed: f, OnDisk: func(json.RawMessage) ([]byte, error) {
|
||||
return nil, errors.New("permission denied")
|
||||
}})
|
||||
readErr.grants = &delivery.GrantsUserList{Follow: delivery.FollowGrants}
|
||||
if _, err := readErr.Hold(cmp); err == nil || !strings.Contains(err.Error(), "could not be read") {
|
||||
t.Fatalf("an unreadable list on the machine was held as %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if c.wantSource != "" && (st.Source != c.wantSource || st.Key != nodeKey.PublicKey ||
|
||||
(c.wantFollows != "" && st.Follows != c.wantFollows)) {
|
||||
t.Fatalf("the have says source %q follows %q key %t; want %q %q", st.Source, st.Follows, st.Key == nodeKey.PublicKey, c.wantSource, c.wantFollows)
|
||||
}
|
||||
switch {
|
||||
case c.wantHanded == 0 && st.Handed != nil:
|
||||
t.Fatalf("handed with no vault key or no hash: %+v", st.Handed)
|
||||
|
||||
@@ -49,6 +49,13 @@ func keep(root, name string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Retire takes a process's build out of the way before another is unpacked where it was: renamed under
|
||||
// retired/, and deleted at once if nothing runs from it, else at a later sweep. A process nobody judges
|
||||
// is replaced through it as a judged one is (novox/hq issue 734): deleting the running build before the
|
||||
// restart deleted the working directory of the process still running from it, and every command it
|
||||
// started until the restart failed with "getwd: no such file or directory".
|
||||
func Retire(root, name, dir string, now time.Time) error { return retire(root, name, dir, now) }
|
||||
|
||||
// retire takes a build that has no reader any more out of the way: renamed under retired/, and deleted
|
||||
// at once if nothing runs from it, else at a later sweep. A path that is not there is nothing to do.
|
||||
func retire(root, name, dir string, now time.Time) error {
|
||||
@@ -98,8 +105,11 @@ func Sweep(root, name string) ([]string, error) {
|
||||
return draining, nil
|
||||
}
|
||||
|
||||
// RunningFrom is every process whose executable is a file under dir — renamed there, or deleted from
|
||||
// there since it started. A variable so a test can name the machine's processes.
|
||||
// RunningFrom is every process whose executable is a file under dir, or whose working directory is in
|
||||
// it — renamed there, or deleted from there since it started. A working directory counts as much as an
|
||||
// image: a process started by the one that runs from a build (a `go version` a builder asks) works in
|
||||
// the build's directory and runs nothing from it (novox/hq issue 734). A variable so a test can name the
|
||||
// machine's processes.
|
||||
var RunningFrom = func(dir string) []int {
|
||||
return runningFromProc("/proc", dir)
|
||||
}
|
||||
@@ -116,14 +126,17 @@ func runningFromProc(proc, dir string) []int {
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
exe, err := os.Readlink(filepath.Join(proc, e.Name(), "exe"))
|
||||
if err != nil {
|
||||
// Gone between the listing and the read, a kernel thread, or another user's process this
|
||||
// engine may not read — it runs as root, so in practice the first two.
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(strings.TrimSuffix(exe, " (deleted)"), dir) {
|
||||
pids = append(pids, pid)
|
||||
for _, link := range []string{"exe", "cwd"} {
|
||||
at, err := os.Readlink(filepath.Join(proc, e.Name(), link))
|
||||
if err != nil {
|
||||
// Gone between the listing and the read, a kernel thread, or another user's process this
|
||||
// engine may not read — it runs as root, so in practice the first two.
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(strings.TrimSuffix(at, " (deleted)")+string(filepath.Separator), dir) {
|
||||
pids = append(pids, pid)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Ints(pids)
|
||||
@@ -140,14 +153,30 @@ func joinPIDs(pids []int) string {
|
||||
|
||||
// SweepAll is Sweep for every process the engine keeps anything about: what the witness does on every
|
||||
// look, so a build retired while its process drained goes once the process has.
|
||||
//
|
||||
// Every process the engine keeps a directory for, not only those it judges: a process nobody judges keeps
|
||||
// nothing but the builds it retired, and those drain the same way (novox/hq issue 734).
|
||||
func SweepAll(root string) []string {
|
||||
entries, err := os.ReadDir(filepath.Join(root, ".witness"))
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var draining []string
|
||||
for _, s := range all(root) {
|
||||
kept, err := Sweep(root, s.Process)
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
continue
|
||||
}
|
||||
name := e.Name()
|
||||
kept, err := Sweep(root, name)
|
||||
if err != nil {
|
||||
draining = append(draining, fmt.Sprintf("%s: %v", s.Process, err))
|
||||
draining = append(draining, fmt.Sprintf("%s: %v", name, err))
|
||||
}
|
||||
draining = append(draining, kept...)
|
||||
// A process nobody judges whose last retired build has drained leaves nothing to keep.
|
||||
if s, err := Load(root, name); err == nil && s.Process == "" && len(kept) == 0 {
|
||||
_ = os.Remove(retiredDir(root, name))
|
||||
_ = os.Remove(Dir(root, name))
|
||||
}
|
||||
}
|
||||
return draining
|
||||
}
|
||||
|
||||
@@ -126,8 +126,26 @@ func Save(root string, s State) error {
|
||||
return os.Rename(tmp.Name(), statePath(root, s.Process))
|
||||
}
|
||||
|
||||
// Forget is a witnessed process no longer declared: everything kept about it goes with it.
|
||||
func Forget(root, name string) error { return os.RemoveAll(Dir(root, name)) }
|
||||
// Forget is a process no longer witnessed, or no longer declared: everything kept about it goes with it,
|
||||
// but for a build a process still runs from. That one is retired and deleted once nothing runs from it
|
||||
// (draining.go): a process whose witness was taken away, or which never had one, is still running from
|
||||
// whatever was retired for it until its unit restarts (novox/hq issue 734).
|
||||
func Forget(root, name string) error {
|
||||
dir := Dir(root, name)
|
||||
if _, err := os.Lstat(dir); os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
if err := retire(root, name, previousDir(root, name), time.Now()); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Remove(statePath(root, name)); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
if draining, err := Sweep(root, name); err != nil || len(draining) > 0 {
|
||||
return err
|
||||
}
|
||||
return os.RemoveAll(dir)
|
||||
}
|
||||
|
||||
// Placing is what the applier is to do with a declared build of a witnessed process.
|
||||
type Placing struct {
|
||||
|
||||
@@ -22,6 +22,14 @@
|
||||
"type": "string",
|
||||
"description": "The operator's public keys, ed25519 in base64, one a line: the node applies a composition made by hand only when one of these signed it (ADR 0539 clause 3). Public keys, no secret. Empty trusts no operator key.",
|
||||
"default": ""
|
||||
},
|
||||
"build-cpu-share": {
|
||||
"type": "number",
|
||||
"description": "The most of this node's processors its builds may take together, in percent (1 to 100): the node-engine holds every build module's runtime of the node in one slice at idle priority under this quota, and each build step sizes its threads from it (novox/hq ADR 0553). Set per node through the node's settings layer; the composer reads it for the build seats' runtimes, the engine's own code does not.",
|
||||
"minimum": 1,
|
||||
"maximum": 100,
|
||||
"default": 50,
|
||||
"runtime": true
|
||||
}
|
||||
},
|
||||
"resources": [
|
||||
|
||||
Reference in New Issue
Block a user