Author SHA1 Message Date
jschoubben 2c5f818d15 Review round 1 of the bus want: a removal keeps the bus server, the have says a server made anew runs the want's fingerprint, the old build is read before the first write, and an unanswered inspect fails the apply rather than replacing the server (ADR 0594 §2, §4)
mesh/check failed: novox: not composed on novox: nats has not been built for this mesh: server is its "server" artifact… | 2c5f818d1522 on 798d73eb11e9
mesh/delivery-group group feat/774-bus-want rejected: composed together: the checker's verdict at the members' heads (a0ab6ce2e316): novox
mesh/delivery closed: the pull request closed unmerged
2026-11-03 09:18:31 +01:00
jschoubben a096abac92 The node agent's applies run through a runner its options name, and one integration test proves an unwarranted bus want keeps the server and a warranted one replaces it (ADR 0594 §2-§4, §8) 2026-11-03 09:06:19 +01:00
jschoubben 052004d1cf The node's own key says its agent keeps the bus server, so the assigner writes the bus want only to such a node (ADR 0594 §8) 2026-11-03 09:03:30 +01:00
jschoubben 9fd5429853 The node agent keeps the bus server unless the want carries the warrant's mark, and the bus's have says what the server runs (ADR 0594 §2-§4, §6) 2026-11-03 09:03:24 +01:00
jschoubben ffcd8f6d61 An apply leaves the resources its scope keeps as they run, so the bus server is not replaced by an unwarranted want (ADR 0594 §2) 2026-11-03 08:56:36 +01:00
jschoubben 089ac78007 Pin mesh-sdk at the bus-want group's head 959b157, so the node agent reads the warrant, the server's fingerprint and the server's list (issue 774) 2026-11-03 08:56:36 +01:00
13 changed files with 967 additions and 53 deletions
+13 -2
View File
@@ -156,6 +156,17 @@ type options struct {
file string
// out is where what a command says goes. Stdout, and a buffer under test.
out io.Writer
// runner is what the node agent's applies run commands through (issue 774, ADR 0594): nil is the machine's own,
// apply.ExecRunner; a test gives one that records what an apply asked of the container runtime.
runner apply.Runner
}
// run is the runner the node agent's applies use: the options' own, else the machine's.
func (o options) run() apply.Runner {
if o.runner != nil {
return o.runner
}
return apply.ExecRunner
}
// parseArgs takes the subcommand first, then its flags.
@@ -1637,11 +1648,11 @@ func applyModuleHeld(ctx context.Context, opts options, declared *declaration.De
func(known store.State) (apply.Report, store.State, error) {
if scope == nil {
return apply.ApplyMindingWindows(ctx, built, declared, known, store.OriginDeclared,
apply.ExecRunner, announceOr(say), sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)),
opts.run(), announceOr(say), sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)),
apply.WindowsIn(filepath.Dir(opts.state)))
}
return apply.ApplyModule(ctx, built, declared, *scope, known, store.OriginDeclared,
apply.ExecRunner, announceOr(say), sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)),
opts.run(), announceOr(say), sealOpener(opts.state), apply.KeepIn(filepath.Dir(opts.state)),
apply.WindowsIn(filepath.Dir(opts.state)))
}, announceOr(say))
+20 -17
View File
@@ -85,7 +85,7 @@ func (h sessionHaves) Read(ctx context.Context, module string) (agent.Have, bool
return agent.Have{}, false
}
return agent.Have{Module: module, FP: got.FP, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail,
Sealed: got.SealedAt}, true
Sealed: got.SealedAt, Server: got.Server, Build: got.Build}, true
}
// ReadErr is Read telling a have absent from one the bus could not read (agent.HavesReadErr).
@@ -103,7 +103,7 @@ func (h sessionHaves) ReadErr(ctx context.Context, module string) (agent.Have, b
return agent.Have{}, false, err
}
return agent.Have{Module: module, FP: got.FP, Composed: got.Composed, Phase: string(got.Phase), Detail: got.Detail,
Sealed: got.SealedAt}, true, nil
Sealed: got.SealedAt, Server: got.Server, Build: got.Build}, true, nil
}
func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
@@ -120,11 +120,11 @@ func (h sessionHaves) Write(ctx context.Context, x agent.Have) {
}
h.session.NoteModule(ctx, link.ModuleFact{Module: x.Module, FP: x.FP, Composed: x.Composed, Phase: phase,
Failed: phase == string(delivery.PhaseFailed), Detail: shorten(x.Detail), Judgement: x.Judgement, Waits: x.Waits,
Sealed: x.Sealed})
Sealed: x.Sealed, Server: x.Server})
}
func (h sessionHaves) Removed(ctx context.Context, module string) {
h.session.NoteRemoved(ctx, module, "")
func (h sessionHaves) Removed(ctx context.Context, module string, words ...string) {
h.session.NoteRemoved(ctx, module, "", words...)
}
// shorten keeps a have's detail within what a have value holds: its head, which says what failed, and its tail, which is
@@ -508,7 +508,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
// What this engine applied, kept on the machine and read at the start, so what runs is known before the node
// composes its own (issue 774).
Applied: appliedStoreFor(opts.state),
Inactive: inactiveIn(&liveness.Exec{Run: apply.ExecRunner, System: builtFor}, heldBy(apply.WindowsIn(filepath.Dir(opts.state)))),
Inactive: inactiveIn(&liveness.Exec{Run: liveness.Runner(opts.run()), System: builtFor}, heldBy(apply.WindowsIn(filepath.Dir(opts.state)))),
Revisions: func(ctx context.Context, key string) ([]agent.Update, error) {
got, err := session.KVHistory(ctx, composedBucket, key)
if err != nil {
@@ -538,7 +538,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
Say: say,
// A failure of the bus not answering for a moment is tried again: 2 s, 5 s, 15 s, 60 s, then every 5 min.
Backoff: link.Backoff,
Apply: func(ctx context.Context, c delivery.Composed, others []delivery.Composed) agent.Result {
Apply: func(ctx context.Context, c delivery.Composed, others []delivery.Composed, keep map[string]bool) agent.Result {
actx, done := applyContext(session.WithLink(ctx))
// While the grants part's list is followed, the controller's list in the bus module's composition is not
// written: the apply keeps the list on the machine (ADR 0547 clause 8, never both).
@@ -547,7 +547,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
if herr != nil {
res = failedWith(herr.Error())
} else {
res = applyComposition(actx, opts, held, others, ag.Placed(scopeName(c)), ag.RemovedByWant(), sched, say)
res = applyComposition(actx, opts, held, others, ag.Placed(scopeName(c)), ag.RemovedByWant(), keep, sched, say)
}
done()
// The mesh's own certificate it carries is kept for this machine's record (ADR 0555 §3).
@@ -564,9 +564,9 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
sayFacts(ctx, ag)
return res
},
Remove: func(ctx context.Context, module string, others []delivery.Composed) agent.Result {
Remove: func(ctx context.Context, module string, others []delivery.Composed, keep map[string]bool) agent.Result {
actx, done := applyContext(session.WithLink(ctx))
res := removeComposition(actx, opts, module, others, ag.Placed(module), sched, say)
res := removeComposition(actx, opts, module, others, ag.Placed(module), keep, sched, say)
done()
afterApply()
sayFacts(ctx, ag)
@@ -632,7 +632,10 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone
env.Composed, env.SelfComposed, env.Policy = composer, true, composer.Policy
env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does
session.NoteComposes(linked, delivery.ComposesOwn, "it composes its own modules, and applies no composition made elsewhere")
env.BusServer = composer.BusServer // the bus server an unwarranted want leaves as it runs (ADR 0594 §2)
// And its agent keeps the bus server as it runs unless a want carries the warrant's mark (ADR 0594 §8).
session.NoteComposes(linked, delivery.ComposesOwn, "it composes its own modules, and applies no composition made elsewhere",
delivery.KeepsBusServer)
ag = agent.New(env)
own.set(ag)
current.set(ag)
@@ -692,7 +695,7 @@ func (c *agentNow) get() *agent.Agent {
//
// removed is the modules whose want says removed: in the node's own apply, a resource of any other module goes only when
// its module is among them (issue 774).
func applyComposition(ctx context.Context, opts options, c delivery.Composed, others []delivery.Composed, placed, removed map[string]bool,
func applyComposition(ctx context.Context, opts options, c delivery.Composed, others []delivery.Composed, placed, removed, keep map[string]bool,
sched *apply.Scheduler, say link.Announce) agent.Result {
name := scopeName(c)
if c.Module == "" {
@@ -708,7 +711,7 @@ func applyComposition(ctx context.Context, opts options, c delivery.Composed, ot
if err := refuseEmptyWrites(d); err != nil {
return failedWith(err.Error())
}
scope := apply.Scope{Module: name, Others: resourcesOf(others), Placed: placed}
scope := apply.Scope{Module: name, Others: resourcesOf(others), Placed: placed, Keep: keep}
if name == agent.NodeModule {
scope.RemovedByWant = removed
if scope.RemovedByWant == nil {
@@ -729,13 +732,13 @@ func applyComposition(ctx context.Context, opts options, c delivery.Composed, ot
if got.Probe != nil {
return failedWith(got.Probe.partial().Error())
}
return agent.Result{}
return agent.Result{Kept: got.Outcome.Kept}
}
// removeComposition removes one module's own resources and nothing of another's.
func removeComposition(ctx context.Context, opts options, module string, others []delivery.Composed, placed map[string]bool,
func removeComposition(ctx context.Context, opts options, module string, others []delivery.Composed, placed, keep map[string]bool,
sched *apply.Scheduler, say link.Announce) agent.Result {
scope := apply.Scope{Module: module, Others: resourcesOf(others), Placed: placed}
scope := apply.Scope{Module: module, Others: resourcesOf(others), Placed: placed, Keep: keep}
got := applyModuleHeld(ctx, opts, &declaration.Declaration{Version: declaration.Version}, &scope, sched, say)
switch {
case got.Refused != "":
@@ -743,7 +746,7 @@ func removeComposition(ctx context.Context, opts options, module string, others
case got.Err != nil:
return agent.Result{Failed: true, Detail: shorten(got.Err.Error())}
}
return agent.Result{}
return agent.Result{Kept: got.Outcome.Kept}
}
// busyModules are the modules whose last apply left a container running because it said busy.
+216
View File
@@ -6,6 +6,7 @@ import (
"context"
"crypto/ed25519"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
@@ -14,13 +15,17 @@ import (
"time"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/declaration"
"git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-host/internal/agent"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/nodegrant"
"github.com/novox/mesh-host/internal/selfcompose"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
// The node agent whole, against a real bus with JetStream started in the test: the wants bucket, the node's composition
@@ -619,3 +624,214 @@ func mustComposedKey(t *testing.T, module, fp string) string {
}
return key
}
// recordingRuntime is a container runtime that keeps its containers in memory and records every command an apply asks
// of it (issue 774): what the bus server's container met is read from it, never from a real runtime.
type recordingRuntime struct {
mu sync.Mutex
containers map[string]string // name: the spec label it was made with
asked []string
}
func (r *recordingRuntime) run(_ context.Context, name string, args ...string) (string, error) {
r.mu.Lock()
defer r.mu.Unlock()
r.asked = append(r.asked, name+" "+strings.Join(args, " "))
if name != "docker" || len(args) == 0 {
return "", nil
}
switch args[0] {
case "info":
return "27.0\n", nil
case "container":
if len(args) > 1 && args[1] == "inspect" {
spec, ok := r.containers[args[len(args)-1]]
if !ok {
return "", errors.New("no such container")
}
return "true\t" + spec + "\n", nil
}
case "image":
if len(args) > 1 && args[1] == "inspect" {
return "2026-01-01T00:00:00Z\n", nil
}
case "rm":
delete(r.containers, args[len(args)-1])
case "run", "create":
var made, spec string
for i, a := range args {
if a == "--name" && i+1 < len(args) {
made = args[i+1]
}
if a == "--label" && i+1 < len(args) && strings.HasPrefix(args[i+1], "mesh-host.spec=") {
spec = strings.TrimPrefix(args[i+1], "mesh-host.spec=")
}
}
r.containers[made] = spec
return "made\n", nil
}
return "", nil
}
// moved is each command that removed, made, stopped or restarted the container named.
func (r *recordingRuntime) moved(container string) []string {
r.mu.Lock()
defer r.mu.Unlock()
var out []string
for _, a := range r.asked {
for _, verb := range []string{"docker rm ", "docker run ", "docker create ", "docker stop ", "docker restart ", "docker kill ",
"docker rename ", "docker start ", "docker container rm ", "docker container rename ", "docker container stop ",
"docker container restart ", "docker container start ", "docker container kill "} {
if strings.HasPrefix(a, verb) && strings.Contains(a, container) {
out = append(out, a)
}
}
}
return out
}
// Issue 774, ADR 0594 §2-§4: the bus server the node's own composition made while the bus module was folded is taken
// over by the bus module's first want, at a new-form fingerprint and without the warrant's mark, and left as it runs:
// the node agent started whole against a real bus composes the bus module, writes its file and records the server as
// the bus module's, asks the container runtime to remove, make, stop or restart nothing, and its have names the
// fingerprint applied beside the old-form build the server still runs; its own key says it keeps the bus server. The
// operator's warranted want at the same fingerprint then replaces the server once, and the have says it runs that
// fingerprint.
func TestNatsTheBusServerIsKeptUntilAWarrantedWantReplacesIt(t *testing.T) {
b := newItBed(t)
runtime := &recordingRuntime{containers: map[string]string{}}
b.opts.runner = runtime.run
const server = "mesh-broker-it"
conf := filepath.Join(b.dir, "nats.conf")
image := "registry.example/nats@sha256:" + strings.Repeat("a", 64)
// What the node's own composition made of the folded bus: the server and its file, recorded as the node's own.
built, err := system.For(builtFor)
if err != nil {
t.Fatal(err)
}
folded, err := declaration.Parse([]byte(`{"declaration":1,"resources":[` +
`{"id":"nats.server-conf","type":"file","path":"` + conf + `","content":"listen: old\n"},` +
`{"id":"nats.server","type":"container","name":"` + server + `","image":"` + image + `","env":{"FOLDED":"1"}}]}`))
if err != nil {
t.Fatal(err)
}
_, known, err := apply.ApplyModule(context.Background(), built, folded, apply.Scope{Module: store.NodeModule}, store.State{},
store.OriginDeclared, runtime.run, nil, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
if err := store.Save(b.opts.state, known); err != nil {
t.Fatal(err)
}
if moved := runtime.moved(server); len(moved) != 1 {
t.Fatalf("the folded server was not made once: %v", moved)
}
made := runtime.containers[server]
runtime.asked = nil
kvOf := func(bucket string) nats.KeyValue {
kv, err := b.js.CreateKeyValue(&nats.KeyValueConfig{Bucket: bucket, History: 5})
if err != nil {
t.Fatal(err)
}
return kv
}
placement, modules, manifests := kvOf(delivery.PlacementBucket), kvOf(delivery.ModulesBucket), kvOf(delivery.ManifestsBucket)
kvOf(selfcompose.ResolvedBucket)
kvOf(delivery.BuilderBuildsBucket)
kvOf(delivery.PlacementCandidatesBucket)
kvOf(link.PackagesBucket)
man := `{"module":"nats","version":"1","provides":[{"name":"mesh-bus","scope":"mesh"}],"resources":[` +
`{"id":"server-conf","type":"file","path":"` + conf + `","mode":"0644","content":"listen: new\n"},` +
`{"id":"server","type":"container","name":"` + server + `","image":"` + image + `"}]}`
digest := delivery.ManifestDigest([]byte(man))
mraw, _ := delivery.EncodeManifest(delivery.Manifest{Module: "nats", JSON: man})
mkey, _ := delivery.ManifestsKey(digest)
fp := fpIt("e")
modv, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: digest, Repository: "novox/mesh-catalog", Commit: strings.Repeat("3", 40)})
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: itNode,
Capabilities: []string{"service-manager", "package-manager", "container-runtime"}}})
hkey, _ := delivery.HaveNodeKey(itNode)
places, _ := delivery.PlacementValues(delivery.Placement{Node: itNode, Modules: []delivery.PlacedModule{{Module: "nats"}}})
// The have the engine before the keeper wrote of the folded bus: the old-form build, no fingerprint.
const took = "build-1792674595610176744"
natsHave, _ := delivery.HaveKey(itNode, "nats")
oldHave, _ := delivery.EncodeHave(delivery.Have{Build: took, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
for _, w := range []struct {
kv nats.KeyValue
key string
value []byte
}{{manifests, mkey, mraw}, {modules, "nats", modv}, {b.haves, hkey, rec}, {b.haves, natsHave, oldHave}} {
if _, err := w.kv.Put(w.key, w.value); err != nil {
t.Fatal(err)
}
}
busWant := func(warrant string) {
raw, err := delivery.EncodeWant(delivery.Want{FP: fp, Self: true, Warrant: warrant})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.WantKey(itNode, "nats")
if _, err := b.wants.Put(key, raw); err != nil {
t.Fatal(err)
}
}
busWant("")
stop := b.start()
defer stop()
for key, value := range places {
if _, err := placement.Put(key, value); err != nil {
t.Fatal(err)
}
}
within(t, 10*time.Second, "the bus module's want was not applied", func() bool {
h := b.have("nats")
got, _ := os.ReadFile(conf)
return h.FP == fp && h.Phase == delivery.PhaseRunning && string(got) == "listen: new\n"
})
if h := b.have("nats"); h.Server != took {
t.Errorf("the bus's have says its server runs %q, not the build it took over", h.Server)
}
if moved := runtime.moved(server); len(moved) != 0 || runtime.containers[server] != made {
t.Fatalf("an unwarranted want moved the bus server: %v", moved)
}
state, err := store.Load(b.opts.state)
if err != nil {
t.Fatal(err)
}
if rec, ok := state.Find("nats.server"); !ok || rec.Module != "nats" {
t.Errorf("the server is recorded %+v, not as the bus module's", rec)
}
// Said with the node's facts, as every start of its agent says them.
b.session.SayFacts(context.Background(), link.Report{Profile: map[string]any{"capabilities": []any{
map[string]any{"name": "container-runtime", "present": true}}}})
within(t, 5*time.Second, "the node's key does not say it keeps the bus server", func() bool {
e, err := b.haves.Get(hkey)
if err != nil {
return false
}
n, err := delivery.DecodeHaveNode(e.Value())
return err == nil && len(n.Keeps) == 1 && n.Keeps[0] == delivery.KeepsBusServer
})
// The operator's warrant at the fingerprint applied: the server is replaced once.
busWant(delivery.WarrantOperator)
within(t, 10*time.Second, "the warranted want did not replace the bus server", func() bool {
return b.have("nats").Server == fp
})
var runs int
for _, a := range runtime.moved(server) {
if strings.HasPrefix(a, "docker run ") && strings.Contains(a, " --name "+server+" ") {
runs++
}
}
if runs != 1 || runtime.containers[server] == made {
t.Errorf("the warranted want made the server %d times, its spec still the folded one %v: %v", runs,
runtime.containers[server] == made, runtime.moved(server))
}
if h := b.have("nats"); h.FP != fp || h.Phase != delivery.PhaseRunning {
t.Errorf("the bus's have after the warrant: %+v", h)
}
}
+1 -1
View File
@@ -3,7 +3,7 @@ module github.com/novox/mesh-host
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103075323-959b1575a8c1
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
golang.org/x/crypto v0.57.0
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128 h1:aI1DhZAuqVBXbybSqhLNOGX5rUOrn9T0cHNCDUQF7IQ=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102191058-f02f142af128/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103075323-959b1575a8c1 h1:lW1NzV8w3pLjH9ACAASYgtYVF3XdCQqIvK/BU81K34Y=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103075323-959b1575a8c1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
+199 -12
View File
@@ -62,6 +62,13 @@ type Have struct {
// slot (delivery.Have.SealedAt): kept in the have so the composer reuses them after a restart (ADR 0547 §3), each at
// its own slot alone (issue 1126, C1).
Sealed delivery.Seals
// Server is, in the bus module's have, what the bus server runs (delivery.Have.Server, novox/hq ADR 0594 §4): the
// fingerprint whose server this node agent last made, or the old-form build of the server it took over. Empty on
// every other have.
Server string
// Build is the old-form build a have from before the fingerprints names (delivery.Have.Build): read, never written
// by the agent; the server the bus module's first want takes over runs it.
Build string
}
// Haves is the node's have bucket as the agent uses it.
@@ -70,8 +77,9 @@ type Haves interface {
Read(ctx context.Context, module string) (Have, bool)
// Write writes a module's have where it differs from the bucket's.
Write(ctx context.Context, h Have)
// Removed writes the module's have with the phase removed, once, if a key of it exists.
Removed(ctx context.Context, module string)
// Removed writes the module's have with the phase removed, once, if a key of it exists; words, when given, are its
// detail.
Removed(ctx context.Context, module string, words ...string)
}
// Result is how one apply or removal came out.
@@ -81,6 +89,8 @@ type Result struct {
// JetStream API error while the bus reloads), not the module's: the apply is tried again without a new revision.
Transient bool
Detail string
// Kept is the resources of the keep set the apply left as they ran, by id (apply.Report.Kept, novox/hq ADR 0594 §4).
Kept []string
}
// Backup is the answer of the backup asked before a module moves.
@@ -103,10 +113,16 @@ type Env struct {
Haves Haves
// Apply applies one module's composition (scope NodeModule for the node's own), given the compositions the node's
// other modules run so a path, unit or group they hold is not removed or taken. It is called with one module at a
// time.
Apply func(ctx context.Context, c delivery.Composed, others []delivery.Composed) Result
// Remove removes the module's own resources and nothing of another's.
Remove func(ctx context.Context, module string, others []delivery.Composed) Result
// time. keep is the resources, by id, the apply leaves as they run when they exist (apply.Scope.Keep): the bus
// server's, on a want of the bus module without the warrant's mark (novox/hq ADR 0594 §2); nil keeps nothing.
Apply func(ctx context.Context, c delivery.Composed, others []delivery.Composed, keep map[string]bool) Result
// BusServer is, for a module that provides the mesh's bus, the ids its manifest declares of the bus server's
// resources (catalogue.BusServerOf, novox/hq ADR 0594 §7), empty for any other module; known says whether the
// module's manifest could be read. Nil names no bus module.
BusServer func(ctx context.Context, module string) (ids []string, known bool)
// Remove removes the module's own resources and nothing of another's, and leaves those of keep as they run (the bus
// server's, ADR 0594 §2).
Remove func(ctx context.Context, module string, others []delivery.Composed, keep map[string]bool) Result
// Backup backs a module up before a new composition of it is applied; nil backs up nothing.
Backup func(ctx context.Context, module string) Backup
// Kept is the resources the node last applied, by module, from the declaration it kept before the switch (ADR 0539
@@ -221,6 +237,9 @@ type Agent struct {
again chan againAsk
transient chan struct{}
stopped chan struct{}
// server is, by bus module, what its have says the bus server runs (Have.Server): read once from the have, then
// kept as this agent writes it, so every have written of the module carries it (novox/hq ADR 0594 §4).
server map[string]string
}
type againAsk struct {
@@ -261,7 +280,7 @@ func New(env Env) *Agent {
return &Agent{env: env, wants: map[string]wantNow{}, comps: map[string]compNow{}, running: map[string]delivery.Have{}, applied: map[string]compNow{}, had: map[string]compNow{}, keptData: map[string]bool{},
attempt: map[string]attempt{}, checked: map[string]bool{}, prior: map[string]bool{}, said: map[string]string{}, notComposed: map[string]notComposedMark{},
retry: make(chan []string, 8), reconcile: make(chan []string, 64), failedAgain: make(chan string, 8), tries: map[string]int{}, again: make(chan againAsk, 64),
transient: make(chan struct{}, 1), stopped: make(chan struct{})}
transient: make(chan struct{}, 1), stopped: make(chan struct{}), server: map[string]string{}}
}
// Run follows the wants until ctx ends. The keys there when it starts are taken first, the node's own composition
@@ -572,6 +591,13 @@ func (a *Agent) NotComposedEach(ctx context.Context, why string, held map[string
// said runs, its fingerprint, digest and sealed values, rather than a have naming nothing: what runs is not lost, and
// the boxes the composer reuses after a restart stay (review of L2, rounds 2 and 3).
func (a *Agent) writeHave(ctx context.Context, h Have) {
if h.Server == "" && h.Module != NodeModule {
h.Server = a.serverOf(ctx, h.Module)
} else if h.Server != "" {
a.mu.Lock()
a.server[h.Module] = h.Server
a.mu.Unlock()
}
if h.Composed == "" && h.Module != NodeModule {
if last, ok := a.env.Haves.Read(ctx, h.Module); ok && last.Composed != "" {
h.FP, h.Composed = last.FP, last.Composed
@@ -588,6 +614,69 @@ func (a *Agent) writeHave(ctx context.Context, h Have) {
a.env.Haves.Write(ctx, h)
}
// serverOf is what the module's have says the bus server runs (Have.Server): kept since this agent read or wrote it,
// else read from the have once. Empty for every module but the bus.
//
// A have written before this node agent kept the server names none: the server then runs what that have says the module
// runs, its fingerprint, else its old-form build (the take-over of a server the node's own composition made). Read so on
// the first read, before this agent writes the have, so every have it writes carries it.
func (a *Agent) serverOf(ctx context.Context, module string) string {
a.mu.Lock()
s, known := a.server[module]
a.mu.Unlock()
if known {
return s
}
if server, _ := a.busKeep(ctx, module); len(server) == 0 {
return "" // not the bus, or not known to be yet: nothing kept, so a later read asks again
}
if h, ok := a.env.Haves.Read(ctx, module); ok {
s = h.Server
if s == "" {
s = h.FP
}
if s == "" {
s = h.Build
}
}
a.mu.Lock()
if _, written := a.server[module]; !written {
a.server[module] = s
}
s = a.server[module]
a.mu.Unlock()
return s
}
// busKeep is, for a module that provides the mesh's bus, the ids of its server's resources (novox/hq ADR 0594 §7) as
// its composition names them (`<module>.<id>`), and true; nil and true for any other module. known is false when the
// module's manifest could not be read: whether it is the bus cannot be told, so nothing of it is applied.
func (a *Agent) busKeep(ctx context.Context, module string) (server map[string]bool, known bool) {
if module == NodeModule || a.env.BusServer == nil {
return nil, true
}
ids, known := a.env.BusServer(ctx, module)
if !known || len(ids) == 0 {
return nil, known
}
server = map[string]bool{}
for _, id := range ids {
server[module+"."+id] = true
}
return server, true
}
// serverOwed says a want of the bus module carries the warrant's mark at a fingerprint whose server does not run yet
// (novox/hq ADR 0594 §4): applied once more though its fingerprint and composition are what runs, so the server is
// replaced at the fingerprint already applied.
func (a *Agent) serverOwed(ctx context.Context, module string, w wantNow) bool {
if w.Want.Warrant == "" {
return false
}
server, _ := a.busKeep(ctx, module)
return len(server) > 0 && a.serverOf(ctx, module) != w.Want.FP
}
func (a *Agent) sealedOf(ctx context.Context, module, fp, composed string) delivery.Seals {
a.mu.Lock()
if c, ok := a.applied[module]; ok && c.Err == nil && c.Dig == composed {
@@ -1213,6 +1302,16 @@ func (a *Agent) settleOne(ctx context.Context, module string) {
a.remove(ctx, module, w)
return
}
if _, known := a.busKeep(ctx, module); !known {
// Whether it is the bus, whose server an unwarranted want never replaces (novox/hq ADR 0594 §2), cannot be told:
// nothing of it is applied until its manifest reads.
a.mu.Lock()
a.sayOnce("bus-unknown:"+module, fmt.Sprintf("%s at %s is not applied: its manifest cannot be read, so whether it "+
"is the bus, whose server only a warranted want replaces, cannot be told", module, w.Want.FP))
a.mu.Unlock()
return
}
owed := a.serverOwed(ctx, module, w)
a.mu.Lock()
comp, have := a.compositionFor(module, w.Want.FP)
a.mu.Unlock()
@@ -1247,14 +1346,15 @@ func (a *Agent) settleOne(ctx context.Context, module string) {
if !a.checked[module] {
a.checked[module] = true
a.mu.Unlock()
if a.baseline(ctx, module, w, comp) {
if !owed && a.baseline(ctx, module, w, comp) {
return
}
} else {
a.mu.Unlock()
}
a.mu.Lock()
if r, ok := a.running[module]; ok && r.FP == w.Want.FP && r.Composed == w.Want.Composed && r.Phase != delivery.PhaseRemoved {
// What runs is what the want names, and no warranted want owes the bus server's replacement (ADR 0594 §4).
if r, ok := a.running[module]; ok && !owed && r.FP == w.Want.FP && r.Composed == w.Want.Composed && r.Phase != delivery.PhaseRemoved {
a.mu.Unlock()
return
}
@@ -1405,6 +1505,27 @@ func (a *Agent) inactiveAtStart(ctx context.Context, module string, comp compNow
func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNow, others []delivery.Composed) {
fp, dig := w.Want.FP, w.Want.Composed
// The bus module's server is kept as it runs unless the want carries the warrant's mark (novox/hq ADR 0594 §2, §6):
// the mark alone decides. Its have says what the server runs (§4): before the apply what it ran, the fingerprint
// applied when the apply kept it a fingerprint or a build ago, and the want's fingerprint once a warranted apply
// replaced it.
keep, _ := a.busKeep(ctx, module)
ranServer, madeServer := "", ""
if len(keep) > 0 {
ranServer = a.serverOf(ctx, module)
if ranServer == "" {
ranServer = fp // a bus module this node never ran: its server is made at this fingerprint
}
madeServer = ranServer
a.mu.Lock()
a.server[module] = ranServer
a.mu.Unlock()
if w.Want.Warrant != "" {
keep, madeServer = nil, fp
a.env.Say(fmt.Sprintf("%s at %s carries the warrant %q: its server, running %s, is replaced", module, fp,
w.Want.Warrant, ranServer))
}
}
a.mu.Lock()
_, ranBefore := a.running[module]
ranBefore = ranBefore || a.prior[module]
@@ -1444,7 +1565,7 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
}
}
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseApplying)})
res := a.env.Apply(ctx, comp.C, others)
res := a.env.Apply(ctx, comp.C, others, keep)
a.mu.Lock()
what := fmt.Sprintf("%s at %s", module, fp)
if module == NodeModule {
@@ -1494,7 +1615,12 @@ func (a *Agent) apply(ctx context.Context, module string, w wantNow, comp compNo
if a.env.Settled != nil {
a.env.Settled(module)
}
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseRunning), Detail: res.Detail})
if len(keep) > 0 && len(res.Kept) == 0 {
// Nothing of the server was there to keep: the apply made it at this fingerprint (ADR 0594 §4).
madeServer = fp
}
a.writeHave(ctx, Have{Module: module, FP: fp, Composed: dig, Phase: string(delivery.PhaseRunning), Detail: res.Detail,
Server: madeServer})
}
func (a *Agent) remove(ctx context.Context, module string, w wantNow) {
@@ -1511,7 +1637,16 @@ func (a *Agent) remove(ctx context.Context, module string, w wantNow) {
}
others := a.heldOut(module)
a.mu.Unlock()
res := a.env.Remove(ctx, module, others)
// A removed want of the bus module never removes its server (novox/hq ADR 0594 §2): a removal carries no warrant, and
// removing the server is not built. One whose manifest cannot be read keeps what would be the server's, said.
keep, known := a.busKeep(ctx, module)
if !known {
keep = map[string]bool{}
for _, id := range catalogue.BusServer {
keep[module+"."+id] = true
}
}
res := a.env.Remove(ctx, module, others, keep)
a.mu.Lock()
if res.Failed {
a.attempt[module] = attempt{FP: "removed", WantRev: w.Rev}
@@ -1531,6 +1666,16 @@ func (a *Agent) remove(ctx context.Context, module string, w wantNow) {
if a.env.Settled != nil {
a.env.Settled(module)
}
if len(res.Kept) > 0 {
words := fmt.Sprintf("removed, but for %s: the bus server stays as it runs until a want carrying the operator's "+
"warrant removes it (novox/hq ADR 0594 §2)", strings.Join(res.Kept, ", "))
if !known {
words += "; its manifest could not be read, so what would be its server was left"
}
a.env.Say(module + " is " + words)
a.env.Haves.Removed(ctx, module, words)
return
}
a.env.Haves.Removed(ctx, module)
}
@@ -1599,6 +1744,13 @@ func (a *Agent) waitsForYes(ctx context.Context, module string, w wantNow, comp
if module == NodeModule || a.env.Policy == nil || a.env.Policy(ctx, module) != catalogue.PolicyRecord || w.Want.Yes == comp.Dig {
return false
}
// For the bus module the judgement covers its server alone (novox/hq ADR 0594 §3): a want with the warrant's mark is
// the operator's warrant, the put's ask after the streams' snapshot or the hand write at the terminal, and stands as
// its yes (ADR 0546 §5); one without it is judged with the server's resources as they run, which it leaves.
server, _ := a.busKeep(ctx, module)
if len(server) > 0 && w.Want.Warrant != "" {
return false
}
a.readKeptData(ctx, module) // the manifest may read now though it did not at the start
a.mu.Lock()
r, ran := a.running[module]
@@ -1632,6 +1784,11 @@ func (a *Agent) waitsForYes(ctx context.Context, module string, w wantNow, comp
}
var j judge.Judged
switch {
case len(server) > 0 && !known:
// What the bus runs cannot be read, and its server is left as it runs whatever it is: nothing to hold.
return false
case len(server) > 0:
j = judge.JudgeMove(&from, withServerAsItRuns(from, comp.C, server))
case known:
j = judge.JudgeMove(&from, comp.C)
case prior:
@@ -1653,6 +1810,36 @@ func (a *Agent) waitsForYes(ctx context.Context, module string, w wantNow, comp
return true
}
// withServerAsItRuns is c with each resource of server taken as from runs it (novox/hq ADR 0594 §3): what an apply that
// keeps the bus server makes of the machine, so its judgement covers the rest of the module alone. A server resource
// from does not run stays as c declares it: an apply makes one that does not exist.
func withServerAsItRuns(from, c delivery.Composed, server map[string]bool) delivery.Composed {
running := map[string]json.RawMessage{}
for _, raw := range from.Resources {
if id := resourceID(raw); server[id] {
running[id] = raw
}
}
out := c
out.Resources = make([]json.RawMessage, 0, len(c.Resources))
for _, raw := range c.Resources {
if was, ok := running[resourceID(raw)]; ok {
raw = was
}
out.Resources = append(out.Resources, raw)
}
return out
}
// resourceID is a composed resource's id, empty when it names none.
func resourceID(raw json.RawMessage) string {
var r struct {
ID string `json:"id"`
}
_ = json.Unmarshal(raw, &r)
return r.ID
}
// runningComposition is what the module runs now: the composition last applied, else its resources in the declaration
// the node kept from before the switch. Called with a.mu held.
//
+185 -10
View File
@@ -17,6 +17,15 @@ import (
const node = "n1"
func mustComposedKey(t *testing.T, module, fp string) string {
t.Helper()
key, err := delivery.ComposedKey(node, module, fp)
if err != nil {
t.Fatal(err)
}
return key
}
func fpOf(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) }
// A watcher the test writes to; it says Caught once the keys written before the watch began are out.
@@ -56,10 +65,11 @@ func (f *feed) put(key string, value []byte) {
}
type haves struct {
mu sync.Mutex
stored map[string]Have
log []Have
removed []string
mu sync.Mutex
stored map[string]Have
log []Have
removed []string
removedWords []string
// unreadable is each module whose have the bus cannot read now (ReadErr).
unreadable map[string]bool
}
@@ -90,10 +100,11 @@ func (h *haves) Write(_ context.Context, x Have) {
h.stored[x.Module] = x
h.log = append(h.log, x)
}
func (h *haves) Removed(_ context.Context, m string) {
func (h *haves) Removed(_ context.Context, m string, words ...string) {
h.mu.Lock()
defer h.mu.Unlock()
h.removed = append(h.removed, m)
h.removedWords = append(h.removedWords, strings.Join(words, ""))
}
func (h *haves) last(module string) Have {
h.mu.Lock()
@@ -117,7 +128,10 @@ type bed struct {
backup func(module string) Backup
kept map[string][]json.RawMessage
others [][]delivery.Composed
said []string
keeps []map[string]bool // what each apply was told to keep as it runs (ADR 0594 §2)
// serverGone is a bus server not on the machine: an apply keeping it keeps nothing and makes it.
serverGone bool
said []string
// settled is each module Settled was told, and settledRan whether what runs held it then.
settled []string
settledRan []bool
@@ -128,10 +142,18 @@ type bed struct {
func newBed(t *testing.T) *bed {
b := &bed{t: t, wants: newFeed(), comps: newFeed(), haves: &haves{}}
b.env = Env{Node: node, Wants: b.wants, Composed: b.comps, Haves: b.haves,
Apply: func(_ context.Context, c delivery.Composed, others []delivery.Composed) Result {
Apply: func(_ context.Context, c delivery.Composed, others []delivery.Composed, keep map[string]bool) Result {
b.mu.Lock()
defer b.mu.Unlock()
b.others = append(b.others, others)
b.keeps = append(b.keeps, keep)
var kept []string // what of keep was there to keep: all of it, unless the test took the server away
for id := range keep {
if !b.serverGone {
kept = append(kept, id)
}
}
slices.Sort(kept)
name := c.Module
if name == "" {
name = NodeModule
@@ -140,13 +162,19 @@ func newBed(t *testing.T) *bed {
if b.failWith != "" && (b.failTimes == 0 || len(b.applied) <= b.failTimes) {
return Result{Failed: true, Transient: b.transient, Detail: b.failWith}
}
return Result{}
return Result{Kept: kept}
},
Remove: func(_ context.Context, m string, _ []delivery.Composed) Result {
Remove: func(_ context.Context, m string, _ []delivery.Composed, keep map[string]bool) Result {
b.mu.Lock()
defer b.mu.Unlock()
b.removed = append(b.removed, m)
return Result{}
b.keeps = append(b.keeps, keep)
var kept []string
for id := range keep {
kept = append(kept, id)
}
slices.Sort(kept)
return Result{Kept: kept}
},
Backup: func(_ context.Context, m string) Backup {
b.mu.Lock()
@@ -1529,6 +1557,153 @@ func TestTheNodesOwnCompositionIsAppliedWhenItDiffersAndARecordMoveWaitsForAYes(
return len(b.appliedList()) == applied+2 && h.Phase == string(delivery.PhaseRunning) && h.FP == fpB &&
r.Phase == string(delivery.PhaseRunning) && r.FP == fpB && r.Composed == rdig
})
// Issue 774, ADR 0594 §2-§4: the bus module's server is kept as it runs on every want without the warrant's mark (a
// new fingerprint whose server recreates, a revision of its user list), never held for a yes, and its have names the
// fingerprint applied and, beside it, what the server runs: at the take-over the old-form build. A warranted want at
// the fingerprint already applied replaces the server once (the warrant is its yes), and the same want again applies
// nothing. A module that is not the bus keeps nothing, marked or not.
t.Run("the bus module", func(t *testing.T) {
b := newBed(t)
b.env.SelfComposed = true
b.env.Policy = func(context.Context, string) string { return catalogue.PolicyRecord }
b.env.BusServer = func(_ context.Context, m string) ([]string, bool) {
if m == "nats" {
return []string{"program-entrypoint", "program-snapshot", "server"}, true
}
return nil, true
}
const took = "build-1792674595610176744"
b.haves.Write(context.Background(), Have{Module: "nats", Build: took, Phase: string(delivery.PhaseRunning)})
stop := b.run()
defer stop()
bus := func(fp, image, users string) (string, []byte, string) {
server, _ := json.Marshal(map[string]any{"id": "nats.server", "type": "container", "module": "nats",
"name": "mesh-broker-nats", "image": "nats@sha256:" + strings.Repeat(image, 64)})
c := delivery.Composed{Node: node, Module: "nats", FP: fp, Resources: []json.RawMessage{server,
file("nats", "server-conf", "listen"), file("nats", "bus-users", users)}}
key, _ := delivery.ComposedKey(node, "nats", fp)
return encodedAs(t, c, key)
}
want := func(module, fp, warrant string) {
key, _ := delivery.WantKey(node, module)
raw, err := delivery.EncodeWant(delivery.Want{FP: fp, Self: true, Warrant: warrant})
if err != nil {
t.Fatal(err)
}
b.wants.put(key, raw)
}
server := map[string]bool{"nats.server": true, "nats.program-snapshot": true, "nats.program-entrypoint": true}
lastKeep := func() map[string]bool {
b.mu.Lock()
defer b.mu.Unlock()
return b.keeps[len(b.keeps)-1]
}
fpA, fpB := fpOf("a"), fpOf("b")
// A have written before the first apply (the node cannot compose yet) keeps the old build's name as what the
// server runs: the writer names no build, so it is read before the first write.
want("nats", fpA, "")
eventually(t, "the bus's want was not taken", func() bool {
b.ag.mu.Lock()
defer b.ag.mu.Unlock()
return b.ag.wants["nats"].Present
})
b.ag.NotComposed(context.Background(), "a stand-in for the bus's secret")
if h := b.haves.last("nats"); h.Build != "" || h.Server != took {
t.Fatalf("a have written before the first apply: %+v", h)
}
b.ag.NotComposed(context.Background(), "")
for i, row := range []struct {
name, fp, image, users, warrant string
gone bool // the server is not on the machine: made, at the want's fingerprint
applies bool
keep map[string]bool
server string
}{
{name: "the take-over of the folded server", fp: fpA, image: "1", users: "u1", applies: true, keep: server, server: took},
{name: "a new fingerprint whose server recreates", fp: fpB, image: "2", users: "u1", applies: true, keep: server, server: took},
{name: "a revision of the user list", fp: fpB, image: "2", users: "u2", applies: true, keep: server, server: took},
{name: "the operator's warrant at the fingerprint applied", fp: fpB, image: "2", users: "u2", warrant: delivery.WarrantOperator,
applies: true, server: fpB},
{name: "the same warrant again", fp: fpB, image: "2", users: "u2", warrant: delivery.WarrantOperator, server: fpB},
{name: "a build after the warrant", fp: fpA, image: "3", users: "u2", applies: true, keep: server, server: fpB},
{name: "a build whose server is not there", fp: fpOf("e"), image: "4", users: "u2", gone: true, applies: true,
keep: server, server: fpOf("e")},
} {
b.mu.Lock()
b.serverGone = row.gone
b.mu.Unlock()
before := len(b.appliedList())
key, raw, dig := bus(row.fp, row.image, row.users)
b.comps.put(key, raw)
want("nats", row.fp, row.warrant)
if row.applies {
eventually(t, row.name+": not applied", func() bool {
h := b.haves.last("nats")
return len(b.appliedList()) == before+1 && h.Composed == dig && h.Phase == string(delivery.PhaseRunning)
})
if got := lastKeep(); !maps.Equal(got, row.keep) {
t.Errorf("%s: the apply kept %v, want %v", row.name, got, row.keep)
}
} else {
time.Sleep(100 * time.Millisecond)
if got := b.appliedList(); len(got) != before {
t.Errorf("%s: applied again: %v", row.name, got)
}
}
if h := b.haves.last("nats"); h.FP != row.fp || h.Server != row.server || h.Waits != "" {
t.Errorf("row %d, %s: the bus's have is %+v, its server %s", i, row.name, h, row.server)
}
}
// Every have written of the bus says what its server runs, applying included: one that did not would lose it.
b.haves.mu.Lock()
for _, h := range b.haves.log[1:] {
if h.Module == "nats" && h.Server == "" {
t.Errorf("a have of the bus names no server: %+v", h)
}
}
b.haves.mu.Unlock()
for _, warrant := range []string{"", delivery.WarrantOperator} {
before := len(b.appliedList())
fp := fpOf("c")
if warrant != "" {
fp = fpOf("d")
}
key, raw, _ := encodedAs(t, delivery.Composed{Node: node, Module: "web", FP: fp,
Resources: []json.RawMessage{file("web", "x", warrant)}}, mustComposedKey(t, "web", fp))
b.comps.put(key, raw)
want("web", fp, warrant)
eventually(t, "web was not applied", func() bool { return len(b.appliedList()) == before+1 })
if got := lastKeep(); got != nil {
t.Errorf("a module that is not the bus kept %v", got)
}
if h := b.haves.last("web"); h.Server != "" {
t.Errorf("web's have names a server: %+v", h)
}
}
time.Sleep(100 * time.Millisecond)
if got := b.appliedList(); len(got) != 8 {
t.Errorf("applied %v, not once for each change", got)
}
// A removed want removes the rest of the bus and never its server: it is kept, said in the journal and in the
// removed have's words.
rkey, _ := delivery.WantKey(node, "nats")
rraw, _ := delivery.EncodeWant(delivery.Want{Removed: true})
b.wants.put(rkey, rraw)
eventually(t, "the bus's removal was not asked", func() bool {
b.haves.mu.Lock()
defer b.haves.mu.Unlock()
return slices.Contains(b.haves.removed, "nats")
})
if got := lastKeep(); !maps.Equal(got, server) {
t.Errorf("the bus's removal kept %v, not its server %v", got, server)
}
b.haves.mu.Lock()
words := b.haves.removedWords[len(b.haves.removedWords)-1]
b.haves.mu.Unlock()
if !strings.Contains(words, "the bus server stays") || !strings.Contains(words, "nats.server") {
t.Errorf("the removed have does not say the server stays: %q", words)
}
})
}
// 1957/AC1 and AC4 (unit; ADR 0568 §1, §4): an engine start takes a module from what the machine shows, not only from the
+112 -4
View File
@@ -120,6 +120,9 @@ type Report struct {
MovedOut []MovedOut `json:"moved-out,omitempty"`
// Left is every socket, pipe or device a move-out directory holds and did not move (ADR 0422 §4).
Left []LeftInPlace `json:"left,omitempty"`
// Kept is every resource the scope keeps (Scope.Keep) that this apply left as it runs, by id (novox/hq ADR 0594
// §2, §4): what says whether the bus server still runs what it ran, or was made by this apply.
Kept []string `json:"kept,omitempty"`
}
// Waiting says whether a container of this apply waits to be replaced because it is busy: what makes the
@@ -143,8 +146,9 @@ func (r Report) Changed() bool {
// (novox/hq issue 224).
// waits busy is a container left running because it said busy (novox/hq ADR 0335), which nothing
// moved either.
// Kept as it runs is a bus server a want without the warrant's mark left alone (novox/hq ADR 0594 §2).
if o.Action != "unchanged" && o.Action != "held" && o.Action != "waiting" && o.Action != heldStill &&
o.Action != waitsBusy {
o.Action != waitsBusy && o.Action != keptAsItRuns {
return true
}
}
@@ -321,6 +325,11 @@ type Scope struct {
// in its own composition has no want, so Placed does not name it (issue 774, review of the re-land, round 1: nats on
// the bus node has none, and its nats.conf was guessed as nats's and removed). Nil guards by Placed alone.
RemovedByWant map[string]bool
// Keep are the resources, by id, the apply leaves as they run when they exist (novox/hq ADR 0594 §2): the bus
// server's container and what restarts it, on a want of the bus module without the warrant's mark. A kept resource
// that exists is neither recreated nor rewritten and is recorded as the module's with what it was last recorded
// reading; one that does not exist is made. Nil keeps nothing.
Keep map[string]bool
}
// placedOwner is the module a want still places on the node that the record belongs to, other than the scope's own:
@@ -410,6 +419,11 @@ func applyScoped(
if err := refuseRootIdentities(d); err != nil {
return report, known, &Error{Resource: "the declaration's identities", Err: err, Done: report}
}
// What of the bus server is kept as it runs, asked of the runtime before anything moves (ADR 0594 §2).
keepNow, err := keptServer(ctx, d, scope, run)
if err != nil {
return report, known, &Error{Resource: "the bus server", Err: err, Done: report}
}
declared := map[string]bool{}
for _, r := range d.Resources {
@@ -538,6 +552,17 @@ func applyScoped(
removeOrphan := func(orphan store.Applied) error {
var action, detail string
var err error
if keepNow[orphan.ID] {
// A removed want of the bus module removes the rest of it and never its server (ADR 0594 §2): only a
// warranted want replaces or removes the server. Its record stays, the module's.
detail := "no longer declared; the bus server is left as it runs: only a want carrying the operator's " +
"warrant removes it (novox/hq ADR 0594 §2)"
report.Outcomes = append(report.Outcomes, Outcome{ID: orphan.ID, Type: orphan.Type, Target: orphan.Target,
Action: keptAsItRuns, Detail: detail})
report.Kept = append(report.Kept, orphan.ID)
log(fmt.Sprintf(" %s %s (%s): %s", keptAsItRuns, orphan.ID, orphan.Target, detail))
return nil
}
if module, held := heldByOthers[orphan.ID]; held && !declared[orphan.ID] {
if module == "" {
module = "another module"
@@ -1065,7 +1090,12 @@ func applyScoped(
if from, handed := handedFiles[resource.Identity()]; handed && was.ID == "" {
previous.Wrote = from.Wrote
}
outcome, err = applyOne(ctx, sys, resource, run, changed, in, previous, unseal, keepFound)
if o, kept := keepAsItRuns(keepNow, resource, previous); kept {
outcome = o
report.Kept = append(report.Kept, o.ID)
} else {
outcome, err = applyOne(ctx, sys, resource, run, changed, in, previous, unseal, keepFound)
}
}
if err != nil {
// **What was found is kept whatever the apply then did** (novox/hq ADR 0118). The
@@ -1272,7 +1302,7 @@ func applyScoped(
if outcome.busy != nil {
report.Busy = append(report.Busy, *outcome.busy)
}
if outcome.Action == heldStill || outcome.Action == waitsBusy {
if outcome.Action == heldStill || outcome.Action == waitsBusy || outcome.Action == keptAsItRuns {
// Not changed: nothing about it moved, so nothing that restarts on it restarts.
log(fmt.Sprintf(" %s %s (%s): %s", outcome.Action, outcome.ID, outcome.Target, outcome.Detail))
} else if moved(outcome) {
@@ -1429,7 +1459,8 @@ func recordApplied(known *store.State, origin string, resource declaration.Resou
// moved reports whether applying a resource moved it, so what restarts or reloads on it does: anything but
// unchanged, and not a container held still or waiting for its busy one to say idle.
func moved(outcome Outcome) bool {
return outcome.Action != "unchanged" && outcome.Action != heldStill && outcome.Action != waitsBusy
return outcome.Action != "unchanged" && outcome.Action != heldStill && outcome.Action != waitsBusy &&
outcome.Action != keptAsItRuns
}
// afterWhatTheyRead is the resources in declared order, except that **a service comes after every
@@ -3019,6 +3050,83 @@ func applyNetwork(ctx context.Context, r *declaration.Network, run Runner) (Outc
return out, nil
}
// keptAsItRuns is the action of a resource the scope keeps (Scope.Keep, novox/hq ADR 0594 §2), found on the machine
// and left as it runs: the bus server on a want of the bus module without the warrant's mark. Its own word, beside
// heldStill: nothing about it moved, so nothing that restarts on it restarts, and a person reading the apply sees why
// a changed server was not replaced.
const keptAsItRuns = "kept-as-it-runs"
// keepAsItRuns is the outcome of a resource the scope keeps that exists on the machine, true; false for any other,
// which is applied as always. A container exists when the runtime knows it by its name; an archive when it was
// unpacked at its path before. The outcome carries what the resource was last recorded reading and holding, so its
// record is the module's from here on (taken over from the node's own composition at the bus module's first want) and
// a later warranted apply still sees what changed under it.
func keepAsItRuns(keep map[string]bool, r declaration.Resource, previous store.Applied) (Outcome, bool) {
if !keep[r.Identity()] {
return Outcome{}, false
}
out := begin(r)
out.Action = keptAsItRuns
why := "the bus server is left as it runs: only a want carrying the operator's warrant replaces it (novox/hq ADR 0594 §2)"
switch res := r.(type) {
case *declaration.Container:
if res.Schedule != "" || res.RunOnce {
return Outcome{}, false
}
// Found running by keptServer before the loop: it is left whatever its declaration says now.
out.reads = previous.Reads
out.busySince = previous.BusySince
out.Detail = why
return out, true
case *declaration.Archive:
if previous.Wrote == "" || previous.Target != res.Path {
return Outcome{}, false
}
if _, err := os.Stat(res.Path); err != nil {
return Outcome{}, false
}
out.wrote = previous.Wrote
out.unpacked = previous.Unpacked
if previous.Wrote != res.Digest {
out.Detail = why + "; the declared " + res.Digest + " waits for the warrant"
} else {
out.Detail = why
}
return out, true
}
return Outcome{}, false
}
// keptServer is what of the scope's Keep this apply keeps (novox/hq ADR 0594 §2), decided once before anything is
// applied, as the archives that restart the server come before it: every kept container the declaration names is
// asked of the runtime once. All found: the Keep stands. One the runtime says is not there: nothing is kept, so the
// server is made at what the declaration says now and its archives are applied with it. A runtime that does not
// answer, or answers anything else, is an error, never taken as an absent server: replacing a running bus server
// because one inspect failed is the harm the Keep exists to prevent.
func keptServer(ctx context.Context, d *declaration.Declaration, scope *Scope, run Runner) (map[string]bool, error) {
if scope == nil || len(scope.Keep) == 0 {
return nil, nil
}
for _, r := range d.Resources {
c, ok := r.(*declaration.Container)
if !ok || !scope.Keep[r.Identity()] || c.Schedule != "" || c.RunOnce {
continue
}
cri, err := containerRuntime(ctx, run)
if err != nil {
return nil, fmt.Errorf("%w, so whether the bus server %s runs cannot be told; nothing was applied", err, c.Name)
}
if _, err := run(ctx, cri, "container", "inspect", "--format", "{{.State.Running}}", c.Name); err != nil {
if absent(err) {
return nil, nil // made, at what the declaration says now
}
return nil, fmt.Errorf("the container runtime did not say whether the bus server %s runs (%v); nothing was "+
"applied, so it is not replaced on an unanswered look", c.Name, err)
}
}
return scope.Keep, nil
}
// heldStill is the action of a container left alone because a maintenance window holds it stopped
// (novox/hq issue 224). Its own word rather than "unchanged": a person reading a machine that looks
// half-stopped needs to see why, and "unchanged" beside a stopped server says the opposite.
+146
View File
@@ -2,7 +2,10 @@ package apply
import (
"context"
"errors"
"os"
"path/filepath"
"slices"
"strings"
"testing"
@@ -219,6 +222,149 @@ func TestAResourceThatMovedToItsModulesCompositionIsLeftByTheNodesOwnApply(t *te
}
})
}
// Issue 774, ADR 0594 §2: the bus module's first own apply takes over the server the node's own composition made
// while the bus was folded. Kept (a want without the warrant's mark), the server's container and its program archive
// are left as they run, whatever the build changed in them, and become nats's; its files and the tools' account file
// are written. Not kept, the same apply replaces the container.
// Review of 774, round 1: the keep is decided once before anything moves, by the runtime's answer for the server. A
// server not there is made and its archives applied with it, none said kept; a runtime that does not answer fails the
// apply, nothing written and nothing replaced; a removal of the bus removes the rest and leaves the server.
for _, tc := range []struct {
name string
keep bool
gone bool // the server is not on the machine
mute bool // the runtime does not answer the server's inspect
remove bool // the bus module's want is removed
}{
{name: "kept", keep: true},
{name: "not kept"},
{name: "kept, the server not there", keep: true, gone: true},
{name: "kept, the runtime does not answer", keep: true, mute: true},
{name: "kept, the bus removed", keep: true, remove: true},
} {
t.Run("the bus server "+tc.name, func(t *testing.T) {
keep := tc.keep && !tc.gone
dir := t.TempDir()
conf, account, program := filepath.Join(dir, "nats.conf"), filepath.Join(dir, "tools", "account"), filepath.Join(dir, "program")
bus := func(env, confBody, programBody, module string) *declaration.Declaration {
archive, _ := archiveDecl(t, "nats.program-snapshot", program, map[string]string{"snapshot": programBody}, "")
if module != "" {
archive = strings.Replace(archive, `"id":"nats.program-snapshot"`, `"id":"nats.program-snapshot","module":"nats"`, 1)
module = `,"module":"nats"`
}
return declare(t, archive+`,
{"id":"nats.server-conf","type":"file","path":"`+conf+`","content":"`+confBody+`"`+module+`},
{"id":"nats.server","type":"container","name":"mesh-broker-nats","image":"`+pinned+`","env":{"V":"`+env+
`"},"restart-on":["nats.program-snapshot","nats.server-conf"]`+module+`}`)
}
m := &machine{containers: map[string]*fakeContainer{}}
_, state, err := ApplyModule(context.Background(), archHost(t), bus("1", "a", "one", ""), Scope{Module: store.NodeModule},
state0(), store.OriginDeclared, m.run, nil, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
made := m.containers["mesh-broker-nats"]
if made == nil {
t.Fatalf("the node's own made no server: %v", m.asked)
}
if tc.gone {
delete(m.containers, "mesh-broker-nats")
}
m.asked = nil
run := m.run
if tc.mute {
run = func(ctx context.Context, name string, args ...string) (string, error) {
if name == "docker" && len(args) > 1 && args[0] == "container" && args[1] == "inspect" {
m.asked = append(m.asked, "docker "+strings.Join(args, " "))
return "", errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock")
}
return m.run(ctx, name, args...)
}
}
next := bus("2", "b", "two", "nats")
next.Resources = append(next.Resources, declare(t, `{"id":"nats.tools-account","type":"file","path":"`+account+
`","content":"tools","module":"nats"}`).Resources...)
scope := Scope{Module: "nats", Placed: map[string]bool{"nats": true}}
if tc.keep {
scope.Keep = map[string]bool{"nats.server": true, "nats.program-snapshot": true}
}
moved := func() []string {
var out []string
for _, a := range m.asked {
for _, verb := range []string{"docker rm", "docker run", "docker create", "docker stop", "docker restart", "docker rename"} {
if strings.HasPrefix(a, verb+" ") {
out = append(out, a)
}
}
}
return out
}
report, after, err := ApplyModule(context.Background(), archHost(t), next, scope, state,
store.OriginDeclared, run, nil, nil, nil, nil)
if tc.mute {
if err == nil || !strings.Contains(err.Error(), "did not say whether the bus server") {
t.Fatalf("an unanswered inspect did not fail the apply: %v", err)
}
if got, _ := os.ReadFile(conf); string(got) != "a" || len(moved()) > 0 {
t.Errorf("an apply that could not tell whether the server runs wrote %q and moved %v", got, moved())
}
return
}
if err != nil {
t.Fatal(err)
}
if keep == (len(moved()) > 0) {
t.Fatalf("kept %v, and the server was moved by %v", keep, moved())
}
if got, _ := os.ReadFile(conf); string(got) != "b" {
t.Errorf("the server's file holds %q", got)
}
if got, _ := os.ReadFile(account); string(got) != "tools" {
t.Errorf("the tools' account file holds %q", got)
}
if got, _ := os.ReadFile(filepath.Join(program, "snapshot")); keep != (string(got) == "one") {
t.Errorf("kept %v, and the server's program is %q", keep, got)
}
var kept []string
for _, id := range []string{"nats.program-snapshot", "nats.server"} {
rec, ok := after.Find(id)
if !ok || rec.Module != "nats" {
t.Errorf("%s is recorded %+v, not as nats's", id, rec)
}
if o := outcomeOf(report, id); keep != (o.Action == keptAsItRuns) {
t.Errorf("kept %v, and %s was %s: %s", keep, id, o.Action, o.Detail)
}
if keep {
kept = append(kept, id)
}
}
if !slices.Equal(report.Kept, kept) && !(len(kept) == 0 && len(report.Kept) == 0) {
t.Errorf("the report says it kept %v, want %v", report.Kept, kept)
}
if tc.keep && report.Changed() == false {
t.Error("an apply that wrote the server's files says nothing changed")
}
if !tc.remove {
return
}
m.asked = nil
gone, last, err := ApplyModule(context.Background(), archHost(t), &declaration.Declaration{}, scope, after,
store.OriginDeclared, run, nil, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
if len(moved()) > 0 || m.containers["mesh-broker-nats"] == nil || !exists(filepath.Join(program, "snapshot")) {
t.Errorf("the bus's removal removed its server: %v", moved())
}
if exists(conf) || exists(account) {
t.Errorf("the bus's removal left the rest of it: conf %v, account %v", exists(conf), exists(account))
}
if _, ok := last.Find("nats.server"); !ok || !slices.Equal(gone.Kept, []string{"nats.program-snapshot", "nats.server"}) &&
!slices.Equal(gone.Kept, []string{"nats.server", "nats.program-snapshot"}) {
t.Errorf("the removal kept %v, the server recorded %v", gone.Kept, ok)
}
})
}
}
func state0() store.State { return store.State{} }
+29 -5
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"slices"
"sort"
"strings"
"sync"
@@ -55,6 +56,11 @@ type haveState struct {
// nodeComposes and nodeComposesWhy are whose compositions the node applies and why (HaveNode.Composes), written into
// its key.
nodeComposes, nodeComposesWhy string
// nodeKeeps is what the node agent keeps as it runs on a want without the warrant's mark (HaveNode.Keeps, novox/hq
// ADR 0594 §8), written into its key.
nodeKeeps []string
// removedWords is, by module in removed, what its removed have says when it is more than that it was removed.
removedWords map[string]string
// written is the build of each module this node-engine wrote a have key for and that the declaration it applied
// still holds; removed is each such module the latest declaration no longer holds, with the build its key last
// said, until the key is written once with the phase removed (issue 1369, novox/hq ADR 0408 §4: never deleted).
@@ -384,6 +390,7 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
h.Phase = delivery.PhaseFailed
}
h.SealedAt = f.Sealed
h.Server = f.Server
h.Health = s.healthOf(module, f)
if h.Detail == "" {
h.Detail = s.wordsOf(module, h.Health)
@@ -427,6 +434,7 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
facts.SealedAt = s.nodeSealed
facts.Candidates = s.nodeCandidates
facts.Composes, facts.ComposesWhy = s.nodeComposes, s.nodeComposesWhy
facts.Keeps = s.nodeKeeps
if raw, err = delivery.EncodeHaveNode(facts); err == nil {
want := facts
if !q.putHave(write, bucket, key, raw, func(current []byte) bool {
@@ -502,6 +510,9 @@ func (q *Session) writeRemoved(ctx context.Context, b HaveBucket, node string) b
}
h := delivery.Have{V: delivery.Version, Build: s.removed[module], Phase: delivery.PhaseRemoved,
Health: delivery.HealthUnknown, Detail: "the applied declaration no longer holds the module"}
if words := s.removedWords[module]; words != "" {
h.Detail = words
}
raw, err := delivery.EncodeHave(h)
if err != nil {
q.sayHave(fmt.Sprintf("cannot encode %s: %v", key, err))
@@ -693,13 +704,23 @@ func (q *Session) NoteUserList(ctx context.Context, module string, w delivery.Us
// NoteRemoved takes that a module's own resources were removed, and writes its have with the phase removed once (issue
// 1369): the key is never deleted.
func (q *Session) NoteRemoved(ctx context.Context, module, build string) {
//
// words, when given, are the removed have's detail (the bus server left as it runs, novox/hq ADR 0594 §2).
func (q *Session) NoteRemoved(ctx context.Context, module, build string, words ...string) {
q.init()
s := &q.haves
s.mu.Lock()
if s.removed == nil {
s.removed = map[string]string{}
}
if said := strings.Join(words, ""); said != "" {
if s.removedWords == nil {
s.removedWords = map[string]string{}
}
s.removedWords[module] = said
} else {
delete(s.removedWords, module)
}
if f, ok := s.modules[module]; ok && build == "" {
build = f.Build
}
@@ -756,14 +777,17 @@ func (q *Session) NoteCandidates(ctx context.Context, answers map[string]deliver
q.writeHaves(ctx, bus)
}
// NoteComposes says, in the node's own key, whose compositions the node applies (delivery.ComposesOwn) and why: written
// at each start of its agent, where every have is read (issue 1126).
func (q *Session) NoteComposes(ctx context.Context, composes, why string) {
// NoteComposes says, in the node's own key, whose compositions the node applies (delivery.ComposesOwn) and why, and what
// its agent keeps as it runs on a want without the warrant's mark (delivery.KeepsBusServer, novox/hq ADR 0594 §8): written
// at each start of its agent, where every have is read (issue 1126). The assigner writes the bus module's want only to
// a node whose key says it keeps the bus server.
func (q *Session) NoteComposes(ctx context.Context, composes, why string, keeps ...string) {
q.init()
s := &q.haves
s.mu.Lock()
changed := s.nodeComposes != composes || s.nodeComposesWhy != why
changed := s.nodeComposes != composes || s.nodeComposesWhy != why || !slices.Equal(s.nodeKeeps, keeps)
s.nodeComposes, s.nodeComposesWhy = composes, why
s.nodeKeeps = slices.Clone(keeps)
known := s.node != nil
s.mu.Unlock()
if !known || !changed {
+18
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"sync"
"testing"
@@ -579,6 +580,23 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth(
if n, _ := delivery.DecodeHaveNode(l.kv["n1.node"].value); n.Candidates != nil {
t.Fatalf("an answer left after its group went: %+v", n.Candidates)
}
// Issue 774, ADR 0594 §4, §8: the bus module's have names what its server runs beside the fingerprint applied, and the
// node's key says its agent keeps the bus server.
q.NoteModule(context.Background(), ModuleFact{Module: "nats", FP: fp, Composed: dig, Server: "build-1792674595610176744"})
if got := l.have(t, "n1.nats"); got.FP != fp || got.Server != "build-1792674595610176744" {
t.Fatalf("the bus's have: %+v", got)
}
// Its removal leaves the server, and its removed have says so (review of 774, round 1).
q.NoteRemoved(context.Background(), "nats", "", "removed, but for nats.server: the bus server stays as it runs")
if got := l.have(t, "n1.nats"); got.Phase != delivery.PhaseRemoved || !strings.Contains(got.Detail, "the bus server stays") {
t.Fatalf("the bus's removed have: %+v", got)
}
q.NoteModule(context.Background(), ModuleFact{Module: "nats", FP: fp, Composed: dig, Server: "build-1792674595610176744"})
q.NoteComposes(context.Background(), delivery.ComposesOwn, "it composes its own", delivery.KeepsBusServer)
if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || !slices.Equal(n.Keeps, []string{delivery.KeepsBusServer}) ||
n.Composes != delivery.ComposesOwn {
t.Fatalf("the node key does not say it keeps the bus server: %+v %v", n, err)
}
}
// A module applied and running that has nothing long-running to judge is ok once the engine has looked: there is nothing
+2
View File
@@ -641,4 +641,6 @@ type ModuleFact struct {
Waits string
// Sealed is the values the composition it names carries sealed to this node's key, by slot (delivery.Have.SealedAt).
Sealed delivery.Seals
// Server is, for the bus module, what its server runs (delivery.Have.Server, novox/hq ADR 0594 §4).
Server string
}
+24
View File
@@ -1163,6 +1163,30 @@ func (c *Composer) Policy(ctx context.Context, module string) string {
return policy
}
// BusServer is, for a module whose manifest provides the mesh's bus, the ids it declares of the bus server's resources
// (catalogue.BusServerOf, novox/hq ADR 0594 §7), none for any other; known says whether the manifest could be read.
func (c *Composer) BusServer(ctx context.Context, module string) (ids []string, known bool) {
c.mu.Lock()
m, ok := c.modules[module]
c.mu.Unlock()
if !ok || m.Manifest == "" {
return nil, false
}
raw, err := reader{c: c}.Manifest(ctx, m.Manifest)
if err != nil {
return nil, false
}
parsed, held, err := c.taken(module, m, raw)
if err != nil || held != "" {
return nil, false
}
for id := range catalogue.BusServerOf(parsed) {
ids = append(ids, id)
}
sort.Strings(ids)
return ids, true
}
// HoldsData says whether a module's manifest declares data (catalogue.Manifest.DataItems), as a composition's Data says
// it, and whether the manifest could be read: the agent's Data for what runs of a module from the kept declaration
// (agent.Env.HoldsData, issue 924).