diff --git a/cmd/mesh-host/filling.go b/cmd/mesh-host/filling.go index ff84109..28bcc9d 100644 --- a/cmd/mesh-host/filling.go +++ b/cmd/mesh-host/filling.go @@ -27,11 +27,7 @@ func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, s func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *filling.Engine { return &filling.Engine{ Declared: func() (*declaration.Declaration, error) { - raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer) - if err != nil { - return nil, err - } - return declaration.Parse(raw) + return declaredNow(opts.state, signer) }, UIDOf: filling.UIDOf, Lock: func() (func(), error) { diff --git a/cmd/mesh-host/judge_directories.go b/cmd/mesh-host/judge_directories.go index c65d19f..2eb2c07 100644 --- a/cmd/mesh-host/judge_directories.go +++ b/cmd/mesh-host/judge_directories.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" - "git.novox.be/novox/mesh-sdk/go/declaration" "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/store" @@ -13,16 +12,12 @@ import ( // judgeDirectories judges every directory of the declaration this node keeps, read-only (novox/hq issue 358): which // the walk that follows no link another account could make would refuse. Nothing is made, chmodded or chowned. func judgeDirectories(statePath string) (string, error) { - kept, err := store.ReadDeclared(store.DeclaredPath(statePath)) + d, err := declaredNow(statePath, nil) if errors.Is(err, store.ErrNothingDeclared) { return "", errors.New("this node keeps no declaration, so it is declared no directory") } if err != nil { - return "", err - } - d, err := declaration.Parse(kept.Declaration) - if err != nil { - return "", fmt.Errorf("the declaration this node keeps does not read: %w", err) + return "", fmt.Errorf("what runs on this node does not read: %w", err) } return apply.JudgeDirectories(d), nil } diff --git a/cmd/mesh-host/libraries.go b/cmd/mesh-host/libraries.go index 040eb2c..f2baefb 100644 --- a/cmd/mesh-host/libraries.go +++ b/cmd/mesh-host/libraries.go @@ -251,16 +251,12 @@ func startKeeping(ctx context.Context, say func(string)) { // declaration this node keeps, and run beside the applies. func accessActFrom(statePath string, timeout time.Duration) link.AccessActFunc { return func(act link.AccessAct) (string, error) { - kept, err := store.ReadDeclared(store.DeclaredPath(statePath)) + d, err := declaredNow(statePath, nil) if errors.Is(err, store.ErrNothingDeclared) { return "", errors.New("this node keeps no declaration, so it is declared no library") } if err != nil { - return "", err - } - d, err := declaration.Parse(kept.Declaration) - if err != nil { - return "", fmt.Errorf("the declaration this node keeps does not read: %w", err) + return "", fmt.Errorf("what runs on this node does not read: %w", err) } ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index c5bb6c8..eb54e8b 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -30,7 +30,6 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/declaration" - "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-host/internal/accounts" "github.com/novox/mesh-host/internal/agent" "github.com/novox/mesh-host/internal/apply" @@ -1137,6 +1136,7 @@ func runLink(ctx context.Context, opts options) error { Timeout: opts.timeout, } current := &agentNow{} + setRunsHere(runsFromAgent(current)) factsOfNow := func(ctx context.Context, ag *agent.Agent) { sayFacts(ctx, opts, mine.Membership.Signer, queue, say, ag) } @@ -1879,14 +1879,8 @@ func sayFacts(ctx context.Context, opts options, signer ed25519.PublicKey, sessi return } declared := &declaration.Declaration{} - if running := runningOf(ag); len(running) > 0 { - if d, err := declarationOf(running...); err == nil { - declared = d - } - } else if raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer); err == nil { - if d, err := declaration.Parse(raw); err == nil { - declared = d - } + if d, err := declaredNow(opts.state, signer); err == nil { + declared = d } held := map[string]bool{} for _, h := range known.Held { @@ -1896,13 +1890,6 @@ func sayFacts(ctx context.Context, opts options, signer ed25519.PublicKey, sessi session.SayFacts(ctx, report) } -func runningOf(ag *agent.Agent) []delivery.Composed { - if ag == nil { - return nil - } - return ag.Running() -} - // waitForEnrolment returns this node's identity, waiting for one if it has none. // // It applies the carried bundle first, if there is one, because that is what a first node does diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index a0dc79c..71cc76f 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -228,10 +228,10 @@ func trustedKeys(opts options, controller ed25519.PublicKey) func() []ed25519.Pu // keptByModule is the resources of the declaration the node kept before the switch, by the module they say; the node's // own are those that say none (ADR 0539 §9). -func keptByModule(opts options, signer ed25519.PublicKey) func(module string) ([]json.RawMessage, bool) { +func keptByModule(opts options, signer ed25519.PublicKey) func() map[string][]json.RawMessage { var once sync.Once var by map[string][]json.RawMessage - return func(module string) ([]json.RawMessage, bool) { + return func() map[string][]json.RawMessage { once.Do(func() { raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer) if err != nil { @@ -256,8 +256,7 @@ func keptByModule(opts options, signer ed25519.PublicKey) func(module string) ([ by[name] = append(by[name], r) } }) - r, ok := by[module] - return r, ok + return by } } @@ -399,3 +398,83 @@ func busyModules() []string { sort.Strings(out) return out } + +// keptDeclaration is the declaration the node kept before the switch, verified with the mesh's key when it is given. +func keptDeclaration(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) { + var raw []byte + if len(signer) > 0 { + var err error + if raw, err = store.LoadDeclared(store.DeclaredPath(statePath), signer); err != nil { + return nil, err + } + } else { + kept, err := store.ReadDeclared(store.DeclaredPath(statePath)) + if err != nil { + return nil, err + } + raw = kept.Declaration + } + return declaration.Parse(raw) +} + +// whatRuns is what runs on this node as the declaration of it (novox/hq ADR 0539 §9): the compositions the agent runs +// and, for every module the agent does not hold yet, what the kept declaration has for it, with the node's own parts +// (adoption, identities, groups, libraries) from the node's composition once the agent holds it and from the kept +// declaration until then. What the node judges its health by, and what the filling, the take act and the judgement of +// directories read. +func whatRuns(everything []delivery.Composed, holdsNode bool, kept func() (*declaration.Declaration, error)) (*declaration.Declaration, error) { + d, err := declarationOf(everything...) + if err != nil { + return nil, err + } + if holdsNode { + return d, nil + } + old, err := kept() + if err != nil { + if len(everything) > 0 { + return d, nil + } + return nil, err + } + d.Adoption, d.Identities, d.Groups, d.Libraries = old.Adoption, old.Identities, old.Groups, old.Libraries + d.Provisioner, d.Builds = old.Provisioner, old.Builds + d.Filled = append(d.Filled, old.Filled...) + return d, nil +} + +// runsHere says what runs on this node; the running serve sets it to the agent's view, and a command run by hand reads +// the kept declaration. +var runsHere struct { + mu sync.Mutex + fn func(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) +} + +func setRunsHere(fn func(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error)) { + runsHere.mu.Lock() + runsHere.fn = fn + runsHere.mu.Unlock() +} + +// declaredNow is the declaration of what runs on this node now. +func declaredNow(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) { + runsHere.mu.Lock() + fn := runsHere.fn + runsHere.mu.Unlock() + if fn != nil { + return fn(statePath, signer) + } + return keptDeclaration(statePath, signer) +} + +// runsFromAgent is what runs now as the agent in hand sees it. +func runsFromAgent(current *agentNow) func(string, ed25519.PublicKey) (*declaration.Declaration, error) { + return func(statePath string, signer ed25519.PublicKey) (*declaration.Declaration, error) { + kept := func() (*declaration.Declaration, error) { return keptDeclaration(statePath, signer) } + ag := current.get() + if ag == nil { + return kept() + } + return whatRuns(ag.Everything(), ag.HoldsNode(), kept) + } +} diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index f35a722..f53efb3 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -18,9 +18,7 @@ import ( "github.com/nats-io/nats-server/v2/server" "github.com/nats-io/nats.go" "github.com/novox/mesh-host/internal/agent" - "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/link" - "github.com/novox/mesh-host/internal/system" ) // The node agent whole, against a real bus with JetStream started in the test: the wants bucket, the node's composition @@ -66,15 +64,18 @@ func aServer(t *testing.T) *nats.Conn { func newItBed(t *testing.T) *itBed { t.Helper() - built, err := system.For("arch") - if err != nil || built.Confirm(context.Background(), apply.ExecRunner) != nil { - t.Skip("this machine is not one these tests can apply on") + // Not skipped off Arch: a package manager that answers is put on PATH, so the apply of files runs on any build node. + bin := t.TempDir() + if err := os.WriteFile(filepath.Join(bin, "pacman"), []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) } + t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) was := builtFor builtFor = "arch" t.Cleanup(func() { builtFor = was }) b := &itBed{t: t, conn: aServer(t), dir: t.TempDir()} + var err error if b.js, err = b.conn.JetStream(); err != nil { t.Fatal(err) } diff --git a/cmd/mesh-host/whatruns_test.go b/cmd/mesh-host/whatruns_test.go new file mode 100644 index 0000000..2b49c0e --- /dev/null +++ b/cmd/mesh-host/whatruns_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "crypto/ed25519" + "encoding/json" + "errors" + "testing" + + "git.novox.be/novox/mesh-sdk/go/declaration" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func aFileOf(module, id string) json.RawMessage { + raw, _ := json.Marshal(map[string]any{"id": module + "." + id, "type": "file", "path": "/tmp/r6/" + module + "/" + id, + "content": "x", "module": module}) + return raw +} + +// What runs is the agent's compositions and the kept declaration's resources for each module the agent does not hold; +// the node's own parts are the kept declaration's until the agent holds the node's composition (ADR 0539 §9). One module +// under the agent must not make the rest invisible to the liveness, units, accounts, libraries, filling and the take act. +func TestWhatRunsIsTheAgentsCompositionsAndTheKeptRest(t *testing.T) { + kept := &declaration.Declaration{Identities: []declaration.Identity{{Module: "db", Name: "mesh-db"}}, + Libraries: []declaration.Library{{Library: "media"}}} + everything := []delivery.Composed{ + {Node: "n1", Module: "web", Resources: []json.RawMessage{aFileOf("web", "a")}}, + {Node: "n1", Module: "db", Resources: []json.RawMessage{aFileOf("db", "a")}}, // still the kept declaration's + } + d, err := whatRuns(everything, false, func() (*declaration.Declaration, error) { return kept, nil }) + if err != nil { + t.Fatal(err) + } + if len(d.Resources) != 2 || len(d.Identities) != 1 || len(d.Libraries) != 1 { + t.Fatalf("%d resources, %d identities, %d libraries: the kept modules and the node's kept parts are missing", + len(d.Resources), len(d.Identities), len(d.Libraries)) + } + + // Once the agent holds the node's composition, its parts are the node's. + node := delivery.Composed{Node: "n1", Own: delivery.NodeOwn{Identities: []declaration.Identity{{Module: "web", Name: "mesh-web"}}}} + d, err = whatRuns(append(everything, node), true, func() (*declaration.Declaration, error) { return nil, errors.New("not read") }) + if err != nil { + t.Fatal(err) + } + if len(d.Identities) != 1 || d.Identities[0].Name != "mesh-web" || len(d.Libraries) != 0 { + t.Fatalf("the node's parts: %+v %+v", d.Identities, d.Libraries) + } +} + +// What the take act, the judgement of directories and the filling read is what runs, not the frozen kept declaration. +func TestDeclaredNowReadsWhatRunsWhenTheAgentIsRunning(t *testing.T) { + setRunsHere(func(string, ed25519.PublicKey) (*declaration.Declaration, error) { + return &declaration.Declaration{Version: 1, Resources: nil, Provisioner: "from-agent"}, nil + }) + defer setRunsHere(nil) + d, err := declaredNow("/nowhere", nil) + if err != nil || d.Provisioner != "from-agent" { + t.Fatalf("declaredNow read %+v (%v), not what runs", d, err) + } +} diff --git a/internal/agent/agent.go b/internal/agent/agent.go index d0f1b13..8c472d9 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -85,9 +85,10 @@ type Env struct { Remove func(ctx context.Context, module string, others []delivery.Composed) Result // Backup backs a module up before a new composition of it is applied; nil backs up nothing. Backup func(ctx context.Context, module string) Backup - // Kept is the resources the node last applied for a module from the declaration it kept before the switch (ADR 0539 - // §9), as JSON, and whether it held the module. Nil when the node kept none. - Kept func(module string) ([]json.RawMessage, bool) + // Kept is the resources the node last applied, by module, from the declaration it kept before the switch (ADR 0539 + // §9), as JSON; the node's own are under NodeModule. Nil when the node kept none. A module the agent does not hold yet + // still runs from it. + Kept func() map[string][]json.RawMessage // Trusted is the public keys a composition may be signed by: the controller's and the operator's (ADR 0539 §3). A // composition unsigned, signed by another key, or changed after it was signed is refused and nothing of it is // applied. Nil trusts no key, so nothing is applied. @@ -348,7 +349,8 @@ func (a *Agent) compositionFor(module, fp string) (compNow, bool) { return c, ok } -// others are the compositions the node's other modules run now, so what they hold is not taken. +// others are what the node's other modules run now, so what they hold is not taken: the compositions the agent runs and, +// for every module the agent does not hold yet, the resources the kept declaration has for it (ADR 0429 §4). func (a *Agent) others(except string) []delivery.Composed { var out []delivery.Composed mods := make([]string, 0, len(a.running)) @@ -367,9 +369,44 @@ func (a *Agent) others(except string) []delivery.Composed { out = append(out, c.C) } } + if a.env.Kept != nil { + kept := a.env.Kept() + names := make([]string, 0, len(kept)) + for m := range kept { + if _, held := a.running[m]; !held && m != except { + names = append(names, m) + } + } + sort.Strings(names) + for _, m := range names { + c := delivery.Composed{Node: a.env.Node, Module: m, Resources: kept[m]} + if m == NodeModule { + c.Module = "" + } + out = append(out, c) + } + } return out } +// Everything is what the node runs: the compositions the agent runs, and the kept declaration's resources for each module +// the agent does not hold yet. What the node judges its health by, and what the filling, the take act and the judgement +// of directories read, is built from it. +func (a *Agent) Everything() []delivery.Composed { + a.mu.Lock() + defer a.mu.Unlock() + return a.others("\x00") +} + +// HoldsNode says whether the agent runs the node's own composition, so the node's parts are its and not the kept +// declaration's. +func (a *Agent) HoldsNode() bool { + a.mu.Lock() + defer a.mu.Unlock() + h, ok := a.running[NodeModule] + return ok && h.Phase != delivery.PhaseRemoved +} + func (a *Agent) settleOne(ctx context.Context, module string) { a.mu.Lock() w := a.wants[module] @@ -434,19 +471,24 @@ func (a *Agent) baseline(ctx context.Context, module string, w wantNow, comp com a.mu.Lock() a.prior[module] = true a.mu.Unlock() - if h.FP == w.Want.FP && h.Composed == w.Want.Composed && - (h.Phase == "" || h.Phase == string(delivery.PhaseRunning)) { - a.mu.Lock() - a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning} - a.mu.Unlock() - return true + if h.Composed != "" { + // This node-engine wrote it: what it says is what runs, and the kept declaration says nothing more. A + // failed apply or a different composition is applied again. + if h.FP == w.Want.FP && h.Composed == w.Want.Composed && + (h.Phase == "" || h.Phase == string(delivery.PhaseRunning)) { + a.mu.Lock() + a.running[module] = delivery.Have{FP: h.FP, Composed: h.Composed, Phase: delivery.PhaseRunning} + a.mu.Unlock() + return true + } + return false } } } if a.env.Kept == nil { return false } - kept, ok := a.env.Kept(module) + kept, ok := a.env.Kept()[module] if ok { a.mu.Lock() a.prior[module] = true diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index 048f457..c8ac2f8 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -110,15 +110,17 @@ type bed struct { failWith string backup func(module string) Backup kept map[string][]json.RawMessage + others [][]delivery.Composed env Env } func newBed(t *testing.T) *bed { b := &bed{t: t, wants: newFeed(), comps: newFeed(), haves: &haves{}} b.env = Env{Node: node, Wants: b.wants, Composed: b.comps, Haves: b.haves, - Apply: func(_ context.Context, c delivery.Composed, _ []delivery.Composed) Result { + Apply: func(_ context.Context, c delivery.Composed, others []delivery.Composed) Result { b.mu.Lock() defer b.mu.Unlock() + b.others = append(b.others, others) name := c.Module if name == "" { name = NodeModule @@ -144,10 +146,7 @@ func newBed(t *testing.T) *bed { } return Backup{Done: true} }, - Kept: func(m string) ([]json.RawMessage, bool) { - r, ok := b.kept[m] - return r, ok - }, + Kept: func() map[string][]json.RawMessage { return b.kept }, Trusted: func() []ed25519.PublicKey { return []ed25519.PublicKey{controllerPub, operatorPub} }, Say: func(s string) { t.Log(s) }, } @@ -556,3 +555,88 @@ func TestWithNoTrustedKeyNothingIsApplied(t *testing.T) { t.Fatalf("applied with no trusted key: %v", got) } } + +// What a module's apply must not take is every other module that runs, those the agent holds and those still running from +// the kept declaration (ADR 0429 §4): the resources of the kept declaration reach Apply as the others. +func TestApplyingOneModuleHoldsOutTheModulesStillRunningFromTheKeptDeclaration(t *testing.T) { + b := newBed(t) + b.kept = map[string][]json.RawMessage{"db": {file("db", "x", "1")}, NodeModule: {file("node", "n", "n")}} + stop := b.run() + defer stop() + fp := fpOf("a") + ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1")) + b.comps.put(ck, craw) + wk, wraw := wantFor(t, "web", fp, dig, "") + b.wants.put(wk, wraw) + eventually(t, "not applied", func() bool { return len(b.appliedList()) == 1 }) + b.mu.Lock() + defer b.mu.Unlock() + var names []string + for _, c := range b.others[0] { + names = append(names, c.Module) + } + if len(names) != 2 || !strings.Contains(strings.Join(names, ","), "db") { + t.Fatalf("the others of web were %v: db and the node's own run from the kept declaration", names) + } +} + +// Everything is what runs: the agent's compositions and the kept resources of each module it does not hold. +func TestEverythingIsWhatTheAgentRunsAndWhatIsKeptForTheRest(t *testing.T) { + b := newBed(t) + b.kept = map[string][]json.RawMessage{"db": {file("db", "x", "1")}, "web": {file("web", "x", "old")}} + a := New(b.env) + if got := a.Everything(); len(got) != 2 { + t.Fatalf("before the agent holds anything: %d compositions, want the 2 kept", len(got)) + } + a.running["web"] = delivery.Have{FP: fpOf("a"), Composed: "sha256:" + strings.Repeat("1", 64), Phase: delivery.PhaseRunning} + ck, craw, dig := composition(t, "web", fpOf("a"), false, file("web", "x", "new")) + a.running["web"] = delivery.Have{FP: fpOf("a"), Composed: dig, Phase: delivery.PhaseRunning} + a.takeComposition(Update{Key: ck, Value: craw, Revision: 1}) + got := a.Everything() + if len(got) != 2 { + t.Fatalf("%d compositions", len(got)) + } + for _, c := range got { + if c.Module == "web" && !strings.Contains(string(c.Resources[0]), "new") { + t.Fatal("web is the kept declaration's, not the composition the agent runs") + } + } + if a.HoldsNode() { + t.Fatal("the agent holds no node composition") + } +} + +// A failed apply is applied again after a restart even when the kept declaration holds the composition: once this +// node-engine wrote its haves, they are what it knows (the kept declaration is only for the first start after the switch). +func TestAFailedApplyIsAppliedAgainAfterARestartWhateverIsKept(t *testing.T) { + b := newBed(t) + fp := fpOf("a") + ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1")) + wk, wraw := wantFor(t, "web", fp, dig, "") + b.comps.before(ck, craw) + b.wants.before(wk, wraw) + b.kept = map[string][]json.RawMessage{"web": {file("web", "x", "1")}} + b.haves.stored = map[string]Have{"web": {Module: "web", FP: fp, Composed: dig, Phase: string(delivery.PhaseFailed)}} + stop := b.run() + defer stop() + eventually(t, "a failed apply was marked running from the kept declaration", func() bool { return len(b.appliedList()) == 1 }) +} + +// A have written before the switch carries no composition: then the kept declaration says what runs. +func TestAHaveWithNoCompositionDefersToTheKeptDeclaration(t *testing.T) { + b := newBed(t) + fp := fpOf("a") + ck, craw, dig := composition(t, "web", fp, false, file("web", "x", "1")) + wk, wraw := wantFor(t, "web", fp, dig, "") + b.comps.before(ck, craw) + b.wants.before(wk, wraw) + b.kept = map[string][]json.RawMessage{"web": {file("web", "x", "1")}} + b.haves.stored = map[string]Have{"web": {Module: "web", Phase: "running"}} + stop := b.run() + defer stop() + eventually(t, "the have was not written", func() bool { return b.haves.last("web").Composed == dig }) + time.Sleep(50 * time.Millisecond) + if got := b.appliedList(); len(got) != 0 { + t.Fatalf("applied %v", got) + } +}