From 38d666f223902d4cd421a85a592f4d1dae2943e3 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 2 Nov 2026 10:40:50 +0100 Subject: [PATCH] Apply only the node's own compositions, never the controller's The controller ended by its unassignment, so a node that falls back to the compositions in its bucket follows what nobody writes (issue 1126, A7). The node agent now always runs its own composer: without a placement key it composes nothing and keeps what runs, a placement it cannot watch stops the agent and is said, and a candidate taking its placement away is refused instead of answered as a return to the controller. The start-time placement read, the end-when-placed watchers and the returning answer go, with their tests and the three integration tests of the controller-composed path. Claude-Session: https://claude.ai/code/session_01NcbgfJ9Gum47zkFKWRaovL --- cmd/mesh-host/nodeagent.go | 172 ++------------ cmd/mesh-host/nodeagent_integration_test.go | 245 ++------------------ internal/selfcompose/selfcompose.go | 72 +----- internal/selfcompose/selfcompose_test.go | 111 ++------- internal/selfcompose/shadow.go | 82 +------ 5 files changed, 61 insertions(+), 621 deletions(-) diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index a823c4d..0a12e7e 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -552,8 +552,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus } return out, nil }, - Trusted: trustedKeys(opts, noControllerKey), - Backup: backupNode(session), + Backup: backupNode(session), // What the filling refused before this apply may be named now: the provisioner's client is told to fill // again (ace, 2026-10-25 21:17). Settled: func(module string) { @@ -604,8 +603,7 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus return res }, } - // The node composes its own modules once the catalogue places it (novox/hq ADR 0547 §2, ADR 0549, issue 1126): its - // placement key is read at the agent's start; until then it applies the controller's compositions as before. + // The node composes its own modules (novox/hq ADR 0547 §2, ADR 0549, issue 1126). run, stopRun := context.WithCancel(linked) // What the node runs and how it opens a sealed value, for the composer to keep the boxes it runs; and the agent // told when the node cannot compose, so each wanted module's have says it. @@ -650,8 +648,8 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus shadowHooks := hooks shadowHooks.refused, shadowHooks.certify = nil, nil shadowEnv := composerEnv(session, node, say, shadowHooks) - // And, after each answer, what runs of an own secret the answer says the vault holds another value of, handed to - // the vault while the node applies the controller's compositions (issue 1126, cause C4; ownresync). + // And, after each answer, what runs of an own secret the answer says the vault holds another value of: handed to + // the vault only while a node applied the controller's compositions (issue 1126, cause C4; ownresync), so nothing now. composes := &composesNow{} resync := ownResync(session, node, say, shadowEnv, composes) shadow := selfcompose.NewShadow(shadowEnv, @@ -660,33 +658,16 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus go resync.take(ctx, answers) }) go shadow.Keep(run, link.Backoff) // this run's alone: a switch begins a run, and its shadow, again - composer, why := selfComposer(linked, session, node, say, hooks) - if linked.Err() != nil { - // The link ended before the node knew whose compositions to apply: it chose neither, and waits for the next. - stopRun() - cancel() - continue - } - if composer != nil { - machinesNow.Store(composer) // the machines a failing network part points at, by the placements (issue 774) - composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone - composes.set(delivery.ComposesOwn) - env.Composed, env.SelfComposed, env.Trusted, env.Policy = composer, true, nil, composer.Policy - env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does - session.NoteComposes(linked, delivery.ComposesOwn, why) - // And once the catalogue no longer places it, the agent begins again on the controller's compositions: the - // placement key deleted ends this run, the way back (issue 1126, the seed's revert). - go endWhenUnplaced(run, session, node, stopRun, say) - } else { - composes.set(delivery.ComposesController) - go resync.again(run) - session.NoteComposes(linked, delivery.ComposesController, why) - // And once the catalogue places it, the agent begins again composing its own: the placement key appearing ends - // this run, so a node does not wait for a restart to follow the seed of its placement (ADR 0549 §4). - go endWhenPlaced(run, session, node, stopRun, say, func(reason string) { - session.NoteComposes(linked, delivery.ComposesController, why+"; "+reason) - }) - } + // Its own composer, always (issue 1126, A7): a composition made elsewhere (the controller's, in the node's + // composition bucket) is never applied. While the seat's placement holds no key for this node it composes + // nothing and keeps running what it runs; a placement it cannot watch stops the agent, said, and it begins again. + composer := selfComposer(session, node, hooks, say) + machinesNow.Store(composer) // the machines a failing network part points at, by the placements (issue 774) + composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone + composes.set(delivery.ComposesOwn) + env.Composed, env.SelfComposed, env.Policy = composer, true, composer.Policy + env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does + session.NoteComposes(linked, delivery.ComposesOwn, "it composes its own modules, and applies no composition made elsewhere") ag = agent.New(env) own.set(ag) current.set(ag) @@ -1002,8 +983,6 @@ func applyContext(ctx context.Context) (context.Context, context.CancelFunc) { return context.WithTimeout(context.WithoutCancel(ctx), applyBound) } -// selfComposer is the node's own composer when the catalogue places the node (its key `placement.` in the -// indexer's state), and nil, said once, while it does not or the key cannot be read. // composerHooks are what the composer reads of the agent and the machine (selfcompose.Env Running, Unseal, Refused). type composerHooks struct { running func() []delivery.Composed @@ -1015,12 +994,9 @@ type composerHooks struct { certify func(ctx context.Context) error } -func selfComposer(ctx context.Context, session *link.Session, node string, say func(string), hooks composerHooks) (*selfcompose.Composer, string) { - key, err := delivery.PlacementNodeKey(node) - if err != nil { - say("this node applies the controller's compositions: " + err.Error()) - return nil, err.Error() - } +// selfComposer is the node's own composer: it composes what the seat's placement places on the node, and nothing while +// the placement holds no key for it (issue 1126). +func selfComposer(session *link.Session, node string, hooks composerHooks, say func(string)) *selfcompose.Composer { env := composerEnv(session, node, say, hooks) // And it keeps each placed module's data items in its own state data, which the node's disk-load watches (novox/hq // ADR 0577 clause 1); the shadow composes for candidates and keeps none. @@ -1028,27 +1004,7 @@ func selfComposer(ctx context.Context, session *link.Session, node string, say f // And the packages its compositions declare in its own key of the state packages, which pacman reads (novox/hq ADR // 0565 §6). env.Packages = link.EnginePackages{Session: session} - unreadable := "" - placed, err := selfcompose.Placed(ctx, env.Get, node, link.Backoff, func(m string) { - if strings.Contains(m, "cannot be read") { - unreadable = m - } - say(m) - }, selfcompose.ComposedItsOwn(ctx, env.Get, node)) - if err != nil { - return nil, "its placement could not be read before the link ended: " + err.Error() - } - if !placed { - why := "the seat's placement state holds no key for it" - if unreadable != "" { - why = "its placement cannot be read, and it never composed its own: " + unreadable - } - say("this node applies the controller's compositions: " + why) - return nil, why - } - why := "the seat's placement state holds its key (" + key + ")" - say("this node composes its own modules: " + why) - return selfcompose.New(env), why + return selfcompose.New(env) } // composerEnv is what the node's composers read of the bus, the vault and the agent: its own composer and its answer @@ -1085,95 +1041,3 @@ func composerEnv(session *link.Session, node string, say func(string), hooks com Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held, VaultAuthority: hooks.vaultAuthority, Certify: hooks.certify} } - -// placementKeysOf is a node's node key and the pattern of the keys that say it is placed: its node key and its legacy -// key `placement.`, read while the holder may not have converted it (research 072 row 37). Its module keys say -// nothing of whether it is placed. -func placementKeysOf(node string) (key, keys string, err error) { - key, err = delivery.PlacementNodeKey(node) - if err != nil { - return "", "", err - } - legacy, _ := delivery.LegacyPlacementKey(node) - return key, link.WatchPatterns(key, legacy), nil -} - -// anyHeld says some key holds a value. -func anyHeld(held map[string]bool) bool { - for _, h := range held { - if h { - return true - } - } - return false -} - -// endWhenUnplaced calls stop once this node's placement key is deleted from the seat's placement (after the watch -// caught up, or gone already), so the agent begins again and applies the controller's compositions, as a node the -// catalogue does not place does from its start: the way back (issue 1126, the seed's revert). A watch that cannot be -// made is said; the node then returns at its next start. -func endWhenUnplaced(ctx context.Context, session *link.Session, node string, stop func(), say link.Announce) { - key, keys, err := placementKeysOf(node) - if err != nil { - return - } - updates, err := bucketWatch{session, delivery.PlacementBucket}.Watch(ctx, keys) - if err != nil { - say("whether this node's key in the seat's placement state still exists is not followed (" + err.Error() + "): it returns to the controller's compositions at its next start") - return - } - // Its node key or, before the holder converted it, its legacy key: placed while either holds a value (research - // 072 row 37); the holder deletes the legacy key before the node key, so the node key's delete is the last. - held, caught := map[string]bool{}, false - for u := range updates { - if u.Caught { - caught = true - } else { - held[u.Key] = !u.Deleted && len(u.Value) > 0 - } - if caught && !anyHeld(held) { - say("this node's key in the seat's placement state is gone (" + key + "): the node agent begins again, applying the controller's compositions") - stop() - return - } - } - if ctx.Err() == nil { - say("the watch of " + key + " ended while the link stays: whether this node's key in the seat's placement state still exists is followed again at the node agent's next start") - } -} - -// endWhenPlaced calls stop once this node's placement key appears in the seat's placement (a value after the watch -// caught up, or one already there), so the agent begins again and composes its own. A watch that cannot be made is said; -// the node then switches at its next start. -func endWhenPlaced(ctx context.Context, session *link.Session, node string, stop func(), say link.Announce, note func(string)) { - key, keys, err := placementKeysOf(node) - if err != nil { - return - } - // The seat's placement state is made when its holder starts; the delivery brings the grants part and the holder first, - // so it is normally there. A node-engine that starts before it waits for the bus's word that the bucket was created - // (no timer), says so once in the journal and in the node's own have, where the watchman reads it, and stays on the - // controller's compositions until its key is written. A watch the account may not make is said the same way, and the - // node stays until its next start. - updates, err := bucketWatch{session, delivery.PlacementBucket}.WatchAwait(ctx, keys, func(err error) { - why := "this node's key in the seat's placement state is waited for, the state is not on the bus yet (" + err.Error() + "): it stays on the controller's compositions and is watched when the bus says the state was created" - say(why) - note(why) - }) - if err != nil { - why := "this node's key in the seat's placement state cannot be followed (" + err.Error() + "): it stays on the controller's compositions until its next start" - say(why) - note(why) - return - } - for u := range updates { - if !u.Caught && !u.Deleted && len(u.Value) > 0 { - say("this node's key in the seat's placement state is written now (" + u.Key + "): the node agent begins again, composing its own modules") - stop() - return - } - } - if ctx.Err() == nil { - say("the watch of " + key + " ended while the link stays: whether this node's key in the seat's placement state is written is followed again at the node agent's next start") - } -} diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index ed731ee..8a958f3 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -21,7 +21,6 @@ import ( "github.com/novox/mesh-host/internal/agent" "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/selfcompose" - "github.com/novox/mesh-host/internal/store" ) // The node agent whole, against a real bus with JetStream started in the test: the wants bucket, the node's composition @@ -237,126 +236,6 @@ func within(t *testing.T, d time.Duration, what string, ok func() bool) { } } -// R6-A AC1: given a running agent on a bus, when a want naming a digest and that composition are written, the module is -// applied within one second and its have names the fingerprint and the digest; a composition whose digest no want names -// is not applied. -func TestNatsAWantAndItsCompositionAreAppliedWithinASecond(t *testing.T) { - t.Run("1957: an engine start starts a declared-running unit", testNatsAnEngineStartStartsADeclaredRunningUnitTheHaveSaysRuns) - b := newItBed(t) - stop := b.start(false) - defer stop() - time.Sleep(300 * time.Millisecond) // the watches are up - - fp := fpIt("a") - dig := b.compose("web", fp, "one\n", false, b.controll) - time.Sleep(300 * time.Millisecond) - if b.file("web") != "" { - t.Fatal("a composition no want names was applied") - } - began := time.Now() - b.want("web", fp, dig, "") - within(t, time.Second, "the module was not applied within one second of its want", func() bool { return b.file("web") == "one\n" }) - t.Logf("applied %s after the want", time.Since(began)) - within(t, 2*time.Second, "the have does not name the fingerprint and the digest", func() bool { - h := b.have("web") - return h.FP == fp && h.Composed == dig && h.Phase == delivery.PhaseRunning - }) - - // A want naming a digest the composition at its fingerprint does not have applies nothing. - fp2 := fpIt("b") - b.compose("web", fp2, "two\n", false, b.controll) - b.want("web", fp2, "sha256:"+strings.Repeat("0", 64), "") - time.Sleep(300 * time.Millisecond) - if b.file("web") != "one\n" { - t.Fatalf("a want naming another digest changed the module: %q", b.file("web")) - } -} - -// R6-A AC5: a data module that ran before asks its backup before any resource changes; a backup that fails holds the -// module (nothing applied, the have says held with the backup's words); when its want then says skip, it is applied. -func TestNatsADataModuleAsksItsBackupFirstAndIsHeldWhenItFails(t *testing.T) { - b := newItBed(t) - var mu sync.Mutex - var asked []string - var onDiskWhenAsked string - if _, err := b.conn.Subscribe("mesh.seat.node-backup.tool.now."+itNode, func(m *nats.Msg) { - mu.Lock() - asked = append(asked, "now") - onDiskWhenAsked = b.file("db") - mu.Unlock() - _ = m.Respond([]byte(`{"result":{"started":"db"}}`)) - // The holder says the backup ended on its event. - ev, _ := json.Marshal(map[string]any{"node": itNode, "module": "db", "ok": false, "at": time.Now().UTC(), - "error": "restic: the repository is locked"}) - _ = b.conn.Publish(backedUpEvent, ev) - }); err != nil { - t.Fatal(err) - } - stop := b.start(false) - defer stop() - time.Sleep(300 * time.Millisecond) - - fp1, fp2 := fpIt("a"), fpIt("b") - b.want("db", fp1, b.compose("db", fp1, "v1\n", true, b.controll), "") - within(t, 3*time.Second, "the first composition was not applied", func() bool { return b.file("db") == "v1\n" }) - mu.Lock() - if len(asked) != 0 { - t.Fatalf("a module that never ran was backed up: %v", asked) - } - mu.Unlock() - - dig2 := b.compose("db", fp2, "v2\n", true, b.controll) - b.want("db", fp2, dig2, "") - within(t, 3*time.Second, "the module was not held behind the failed backup", func() bool { - h := b.have("db") - return h.Phase == delivery.PhaseHeld && strings.Contains(h.Detail, "repository is locked") - }) - mu.Lock() - if len(asked) != 1 || onDiskWhenAsked != "v1\n" { - t.Fatalf("the backup was asked %v, with %q on disk: it is asked before any resource changes", asked, onDiskWhenAsked) - } - mu.Unlock() - if b.file("db") != "v1\n" { - t.Fatalf("something was applied behind a failed backup: %q", b.file("db")) - } - if h := b.have("db"); h.FP != fp1 { - t.Fatalf("a held module should still name what it runs: %+v", h) - } - - b.want("db", fp2, dig2, delivery.BackupSkip) - within(t, 3*time.Second, "the module was not applied once its want said skip", func() bool { return b.file("db") == "v2\n" }) - within(t, 2*time.Second, "the have does not say running at the new fingerprint", func() bool { - h := b.have("db") - return h.Phase == delivery.PhaseRunning && h.FP == fp2 && h.Composed == dig2 - }) -} - -// R6-A AC9, the hand path end to end: with no controller running, a composition signed by the operator's key and a want -// naming it are applied and the have written; the same signed by a key nobody trusts is not. -func TestNatsTheHandPathAppliesACompositionTheOperatorSigned(t *testing.T) { - b := newItBed(t) - stop := b.start(true) - defer stop() - time.Sleep(300 * time.Millisecond) - - _, stranger, _ := ed25519.GenerateKey(nil) - fpS := fpIt("c") - b.want("evil", fpS, b.compose("evil", fpS, "no\n", false, stranger), "") - time.Sleep(300 * time.Millisecond) - if b.file("evil") != "" { - t.Fatal("a composition signed by a key nobody trusts was applied") - } - - fp := fpIt("d") - dig := b.compose("rescue", fp, "back\n", false, b.operator) - b.want("rescue", fp, dig, "") - within(t, 2*time.Second, "the operator's composition was not applied", func() bool { return b.file("rescue") == "back\n" }) - within(t, 2*time.Second, "the have was not written", func() bool { - h := b.have("rescue") - return h.FP == fp && h.Composed == dig && h.Phase == delivery.PhaseRunning - }) -} - // Issue 1126 (ADR 0547 §2, the milestone's H8): the node agent started whole against a real bus, once the catalogue // places its node (the placement written after it started), composes a module from its want, the indexer's manifest, its record and its mesh-config value, and applies it, // with no key in the controller's compositions; a changed config value is composed again on its event and applied as a @@ -478,9 +357,10 @@ func h8once(t *testing.T, b *itBed, resolved, candidates nats.KeyValue, place [] } } -// h8back is the way back (issue 1126, the seed's revert): a candidate deleting the node's placement is compared with -// the controller's compositions at its wants (here none: refused, naming web), and the placement key deleted returns the agent to them without a -// restart, said in the journal and the node's have, keeping what runs. +// h8back is A7 of issue 1126: no composition made elsewhere is applied. A candidate deleting the node's placement is +// refused, never answered as a return to the controller's compositions; the placement key deleted leaves the agent on +// its own composer, composing nothing and keeping what runs, said in the journal, the node's have still saying it +// composes its own. func h8back(t *testing.T, b *itBed, placement, resolved, candidates nats.KeyValue, hkey string) { t.Helper() node := func() delivery.HaveNode { @@ -502,65 +382,38 @@ func h8back(t *testing.T, b *itBed, placement, resolved, candidates nats.KeyValu if _, err := candidates.Put(ckey, craw); err != nil { t.Fatal(err) } - // The controller holds no composition at web's want (the node composed its own): returning would leave web with - // nothing to apply, and the answer says so rather than "same" (review of the shadow, round 1). - within(t, 10*time.Second, "the node did not refuse returning to compositions the controller does not hold", func() bool { + within(t, 10*time.Second, "the node did not refuse a candidate taking its placement away", func() bool { a := node().Candidates["revert-x"] return a.Proposal == "revert-x" && a.Returns == "" && a.Verdict == delivery.CandidateRefused && - strings.Contains(a.Refused, "web: the controller holds no composition") + strings.Contains(a.Refused, "composes nothing") }) ran := b.have("web").Composed pkey, _ := delivery.PlacementNodeKey(itNode) - // A pass in flight at the switch (issue 1126, 2026-10-28: the last pass of the own run, failing on its run's end, - // marked the next agent's modules failed): a config value changed starts one, and the key goes at once. + // A pass in flight when the key goes (issue 1126, 2026-10-28): a config value changed starts one, and the key goes at once. if _, err := resolved.Put(itNode+".web.greeting", []byte(`{"value":"salut","source":"module"}`)); err != nil { t.Fatal(err) } if err := placement.Delete(pkey); err != nil { t.Fatal(err) } - stopWatch := make(chan struct{}) - marked := make(chan Have1126, 1) - go func() { - for { - select { - case <-stopWatch: - return - case <-time.After(20 * time.Millisecond): - } - if h := b.have("web"); h.Phase == delivery.PhaseFailed { - select { - case marked <- Have1126{h.Phase, h.Detail}: - default: - } - return - } - } - }() - defer close(stopWatch) - within(t, 15*time.Second, "the node did not return to the controller's compositions when its placement went", func() bool { + within(t, 15*time.Second, "the node did not say it composes nothing once its placement went", func() bool { b.mu.Lock() said := strings.Join(b.said, "\n") b.mu.Unlock() - return strings.Contains(said, "key in the seat's placement state is gone") && node().Composes == delivery.ComposesController + return strings.Contains(said, "key in the seat's placement state is gone: it composes nothing of its own") }) time.Sleep(3 * time.Second) // what a pass in flight would mark lands within it - select { - case h := <-marked: - t.Errorf("a pass in flight at the switch marked web failed: %+v", h) - default: + b.mu.Lock() + said := strings.Join(b.said, "\n") + b.mu.Unlock() + if n := node(); n.Composes != delivery.ComposesOwn || strings.Contains(said, "applying the controller's compositions") { + t.Errorf("without its placement the node turned to the controller's compositions: %q", n.Composes) } if h := b.have("web"); h.Composed != ran || h.Phase == delivery.PhaseFailed || b.file("web") != "greeting=salut\n" { - t.Errorf("returning to the controller's compositions moved what runs: %+v, %q", h, b.file("web")) + t.Errorf("without its placement what runs moved: %+v, %q", h, b.file("web")) } } -// Have1126 is what H8's way back watches of web's have. -type Have1126 struct { - Phase delivery.Phase - Detail string -} - // h8state is what h8 set up that the rest of H8 writes to. type h8state struct { placement, resolved, candidates nats.KeyValue @@ -730,71 +583,3 @@ func h8rest(t *testing.T, b *itBed, resolved, candidates nats.KeyValue, hkey str t.Errorf("a candidate moved what runs: %+v, %q", h, b.file("web")) } } - -// 1957/AC1 (integration; ADR 0568 §1): a node agent starting on a bus that holds a `running` have for a module at its want, -// with a service manager (a systemctl on the PATH) that shows the module's unit inactive, starts the unit once and writes -// the have running; the same start with the unit active applies nothing. -func testNatsAnEngineStartStartsADeclaredRunningUnitTheHaveSaysRuns(t *testing.T) { - for _, active := range []bool{false, true} { - b := newItBed(t) - bin := t.TempDir() - stateFile, calls := filepath.Join(bin, "state"), filepath.Join(bin, "calls") - if active { - os.WriteFile(stateFile, []byte("active"), 0o644) - } else { - os.WriteFile(stateFile, []byte("inactive"), 0o644) - } - script := "#!/bin/sh\necho \"$@\" >> " + calls + "\ncase \"$1\" in\n" + - "show) printf 'LoadState=loaded\\nActiveState=%s\\nType=simple\\n' \"$(cat " + stateFile + ")\" ;;\n" + - "start) printf active > " + stateFile + " ;;\nis-enabled) echo enabled ;;\nesac\nexit 0\n" - if err := os.WriteFile(filepath.Join(bin, "systemctl"), []byte(script), 0o755); err != nil { - t.Fatal(err) - } - t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) - - // The engine's own record of the unit, as a previous run left it, and the have it wrote then. - res, _ := json.Marshal(map[string]any{"id": "web.unit", "type": "service", "unit": "web-it.service", "state": "running", "module": "web"}) - var known store.State - known.Record(store.Applied{ID: "web.unit", Type: "service", Target: "web-it.service"}) - if err := store.Save(b.opts.state, known); err != nil { - t.Fatal(err) - } - fp := fpIt("a") - c := delivery.Composed{Node: itNode, Module: "web", FP: fp, Resources: []json.RawMessage{res}} - signed, _ := delivery.SignComposed(c, b.controll) - raw, _ := delivery.EncodeComposed(signed) - dig, _ := delivery.ComposedDigestOf(signed) - key, _ := delivery.ComposedKey(itNode, "web", fp) - if _, err := b.composed.Put(key, raw); err != nil { - t.Fatal(err) - } - b.want("web", fp, dig, "") - hkey, _ := delivery.HaveKey(itNode, "web") - have, err := delivery.EncodeHave(delivery.Have{FP: fp, Composed: dig, Phase: delivery.PhaseRunning, Health: delivery.HealthUnknown}) - if err != nil { - t.Fatal(err) - } - if _, err := b.haves.Put(hkey, have); err != nil { - t.Fatal(err) - } - - stop := b.start(false) - within(t, 5*time.Second, "the unit was not started and the have not running", func() bool { - got, _ := os.ReadFile(stateFile) - h := b.have("web") - return string(got) == "active" && h.Phase == delivery.PhaseRunning && h.Composed == dig - }) - time.Sleep(300 * time.Millisecond) - stop() - starts := 0 - log, _ := os.ReadFile(calls) - for _, l := range strings.Split(string(log), "\n") { - if strings.HasPrefix(l, "start") { - starts++ - } - } - if want := map[bool]int{false: 1, true: 0}[active]; starts != want { - t.Fatalf("unit active=%v: started %d times, want %d", active, starts, want) - } - } -} diff --git a/internal/selfcompose/selfcompose.go b/internal/selfcompose/selfcompose.go index 101a511..67e9d66 100644 --- a/internal/selfcompose/selfcompose.go +++ b/internal/selfcompose/selfcompose.go @@ -798,8 +798,8 @@ func (c *Composer) compose(ctx context.Context) []agent.Update { r := reader{c: c} c.mu.Unlock() if !placed { - // The catalogue no longer places it: the agent returns to the controller's compositions (endWhenUnplaced), and - // this is no refusal of what it runs (issue 1126, the seed's revert). + // The catalogue does not place it: it composes nothing, applies no composition made elsewhere, and this is no + // refusal of what it runs (issue 1126). c.sayOnce("this node's key in the seat's placement state is gone: it composes nothing of its own, and keeps running what it runs") c.paid() // nothing of its own to compose, so no pass is owed and no timer armed again return nil @@ -1496,74 +1496,6 @@ func (r reader) Credentials(ctx context.Context, node string, asConsumer, asProv return creds, issued, nil } -// Placed reads whether the catalogue places node now, for the node agent choosing whose compositions it applies at -// each start: true when its placement key holds a value, false only when a read succeeds and finds no key or a delete -// marker. A read that fails chooses neither (review of the shadow, round 1: a timeout or a bus not ready at a restart -// flipped a self-composing node to the controller's compositions for a whole run): it is tried again after wait(try), -// each distinct failure said once, until ctx ends, which is then its error. -// -// switched says the node composed from its own placement key before (its have says it composes its own, -// delivery.ComposesOwn, and that is kept on the bus, so it survives a restart). Only a node that never switched takes a -// read that fails (a timeout, or a state its account may not read yet, as the seat's before its grant arrives) as "not -// placed": it stays on the controller's compositions, says why once, and switches on a positive read of its key (the -// watch endWhenPlaced keeps). A node that switched never goes back on a failed read, however long it lasts. -func Placed(ctx context.Context, get func(ctx context.Context, bucket, key string) ([]byte, error), node string, - wait func(try int) time.Duration, say func(string), switched bool) (bool, error) { - key, err := delivery.PlacementNodeKey(node) - if err != nil { - return false, err - } - legacy, _ := delivery.LegacyPlacementKey(node) - said := "" - for try := 1; ; try++ { - // Its node key, or, before the seat's holder converted it, its legacy key (research 072 row 37). - raw, err := get(ctx, delivery.PlacementBucket, key) - if err == nil && len(raw) == 0 { - raw, err = get(ctx, delivery.PlacementBucket, legacy) - } - if errors.Is(err, nats.ErrBucketNotFound) { - return false, nil // the seat has written no placement state yet: nothing places it, read and known - } - if err == nil { - return len(raw) > 0, nil - } - if ctx.Err() != nil { - return false, ctx.Err() - } - if !switched { - say(fmt.Sprintf("this node's key in the seat's placement state cannot be read (%s): it has never composed its own, so it applies the controller's compositions and switches when its key is read: %v", key, err)) - return false, nil - } - if why := err.Error(); why != said { - say(fmt.Sprintf("this node's key in the seat's placement state cannot be read (%s): the node agent waits, choosing neither its own compositions nor the controller's: %v", key, err)) - said = why - } - select { - case <-ctx.Done(): - return false, ctx.Err() - case <-time.After(wait(try)): - } - } -} - -// ComposedItsOwn says whether the node's own have, kept on the bus, says it composed from its placement key at its last -// start. A have that cannot be read says yes: the safe side, a node waits and does not fall back. -func ComposedItsOwn(ctx context.Context, get func(ctx context.Context, bucket, key string) ([]byte, error), node string) bool { - key, err := delivery.HaveNodeKey(node) - if err != nil { - return true - } - raw, err := get(ctx, delivery.HaveBucket, key) - if err != nil { - return true - } - if len(raw) == 0 { - return false - } - n, err := delivery.DecodeHaveNode(raw) - return err != nil || n.Composes == delivery.ComposesOwn -} - // composeAgain are the waits before a pass that met a bus not answering is tried again. var composeAgain = []time.Duration{time.Second, 2 * time.Second, 4 * time.Second} diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index 1daff41..802b4d4 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -1162,9 +1162,11 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { return a.Verdict == delivery.CandidateSame && a.Returns == "" && a.Revision == 8 && len(a.Removed) == 0 && fmt.Sprint(a.Same) == "[lemurs node]" }, + // Issue 1126, A7: a candidate deleting the node's placement is refused, never answered as a return to the + // controller's compositions: no composition made elsewhere is applied, so the node would compose nothing. "deletes": func(a delivery.CandidateAnswer) bool { - return a.Verdict == delivery.CandidateSame && a.Returns == delivery.ReturnsToController && a.Revision == 7 && - len(a.Removed) == 0 && fmt.Sprint(a.Same) == "[lemurs node]" + return a.Verdict == delivery.CandidateRefused && a.Returns == "" && a.Revision == 7 && + strings.Contains(a.Refused, "composes nothing") }, } { if !want(answers[group]) { @@ -1286,61 +1288,21 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { return !ok && len(a) == 4 }) - // Whose compositions a node applies at a start (review of the shadow, round 1): only a read that succeeds chooses; a - // failing read is tried again, said once, and chooses neither, even when the link ends. - for name, row := range map[string]struct { - answers []error // per try; nil then the value - value []byte - end bool - placed bool - tries int - quiet bool // read and known: nothing to say - // switched: the node's have says it composed its own before (it never falls back on a failed read). - switched bool - }{ - "switched: placed after two failures": {answers: []error{fmt.Errorf("nats: timeout"), fmt.Errorf("nats: timeout")}, value: []byte(`{}`), placed: true, tries: 3, switched: true}, - "never switched: a timeout is not placed, said once": {answers: []error{fmt.Errorf("nats: timeout")}, placed: false, tries: 1}, - "no key": {placed: false, tries: 2}, // its node key, then its legacy key (research 072 row 37) - "no placement state yet": {answers: []error{fmt.Errorf("the bucket %s: %w", delivery.PlacementBucket, nats.ErrBucketNotFound)}, placed: false, tries: 1, quiet: true}, - "switched: failing until the link ends": {answers: []error{fmt.Errorf("jetstream not ready"), fmt.Errorf("jetstream not ready"), - fmt.Errorf("jetstream not ready")}, end: true, tries: 3, switched: true}, - } { - lctx, lcancel := context.WithCancel(ctx) - tries := 0 - var said []string - get := func(context.Context, string, string) ([]byte, error) { - tries++ - if tries <= len(row.answers) { - if row.end && tries == len(row.answers) { - lcancel() - } - return nil, row.answers[tries-1] + // Issue 1126, A7 AC1: a placement that cannot be read composes nothing and says why; the node never applies a + // composition made elsewhere instead. (A placement read without the node's key composes nothing either: above.) + { + refused := newFakeBus() + env := refused.env() + watch := env.Watch + env.Watch = func(ctx context.Context, bucket, pattern string) (<-chan agent.Update, error) { + if bucket == delivery.PlacementBucket { + return nil, fmt.Errorf("nats: Permissions Violation for Subscription to %s", bucket) } - return row.value, nil + return watch(ctx, bucket, pattern) } - placed, err := Placed(lctx, get, "n1", func(int) time.Duration { return time.Millisecond }, func(s string) { said = append(said, s) }, row.switched) - lcancel() - if placed != row.placed || tries != row.tries || (err != nil) != row.end || (len(row.answers) > 0 && !row.quiet) != (len(said) == 1) { - t.Errorf("%s: placed %v, err %v after %d reads, said %q", name, placed, err, tries, said) - } - } - - // Whether the node composed its own before is read from its own have: own says yes, the controller's or no have says - // no, and a have that cannot be read says yes (the safe side: a node waits and does not fall back). - for name, row := range map[string]struct { - raw []byte - err error - want bool - }{ - "own": {raw: []byte(`{"v":1,"composes":"own"}`), want: true}, - "the controller's": {raw: []byte(`{"v":1,"composes":"controller"}`), want: false}, - "no have yet": {want: false}, - "a have that cannot be read": {err: fmt.Errorf("nats: timeout"), want: true}, - "a have that is no have": {raw: []byte(`{`), want: true}, - } { - get := func(context.Context, string, string) ([]byte, error) { return row.raw, row.err } - if got := ComposedItsOwn(ctx, get, "n1"); got != row.want { - t.Errorf("composed its own, %s: %v, want %v", name, got, row.want) + out, err := New(env).Watch(ctx, "") + if out != nil || err == nil || !strings.Contains(err.Error(), delivery.PlacementBucket) || !strings.Contains(err.Error(), "cannot be watched") { + t.Errorf("an unreadable placement: composed %v, said %v", out != nil, err) } } @@ -1450,45 +1412,6 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { } } - // The way back, row by row (review of the shadow, round 1): the controller's compositions at n1's wants compared - // with what runs; one differing is answered so; one the controller holds none of refuses, without returning. - var runsList []delivery.Composed - for _, c := range running { - runsList = append(runsList, c) - } - lemursKey, _ = delivery.ComposedKey("n1", "lemurs", fp) - changed := running[lemursKey] - changed.Resources = []json.RawMessage{json.RawMessage(`{"id":"lemurs.service","type":"service","module":"lemurs","unit":"lemurs.service","state":"stopped","boot":"disabled"}`)} - changedRaw, _ := delivery.EncodeComposed(changed) - for name, row := range map[string]struct { - lemurs []byte - check func(delivery.CandidateAnswer) bool - }{ - "the controller's differs": {changedRaw, func(a delivery.CandidateAnswer) bool { - return a.Verdict == delivery.CandidateDiffers && a.Returns == delivery.ReturnsToController && - len(a.Differs) == 1 && a.Differs[0].Module == "lemurs" && fmt.Sprint(a.Same) == "[node]" - }}, - "the controller holds none": {nil, func(a delivery.CandidateAnswer) bool { - return a.Verdict == delivery.CandidateRefused && a.Returns == "" && strings.Contains(a.Refused, "lemurs: the controller holds no composition") - }}, - } { - get := func(_ context.Context, bucket, key string) ([]byte, error) { - if key == lemursKey { - return row.lemurs, nil - } - readOnly.mu.Lock() - defer readOnly.mu.Unlock() - return readOnly.kv[bucket+"/"+key], nil - } - a := returning(ctx, get, "n1", map[string]delivery.Want{"lemurs": {FP: fp}, agent.NodeModule: {}}, runsList, "pro", 1) - if !row.check(a) { - t.Errorf("%s: %+v", name, a) - } - if _, err := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"pro": a}}); err != nil { - t.Errorf("%s: the answer is not one a have holds: %v", name, err) - } - } - // Issue 1126, the seed's review: a module n1 runs inside its own composition (nats, at a build of the old form) is // composed there again, not at the trunk's build as one added; lemurs at the build it runs, keeper (added) at the // trunk's. diff --git a/internal/selfcompose/shadow.go b/internal/selfcompose/shadow.go index 5627274..254699f 100644 --- a/internal/selfcompose/shadow.go +++ b/internal/selfcompose/shadow.go @@ -23,8 +23,7 @@ import ( // // **Event-driven, like the Composer.** It watches every candidate key, the trunk's placement, every node's record, the // indexer's modules, this node's config values, and this node's module haves (what it runs moved); a burst of changes -// answers once. It runs whether or not the node is placed yet: a node that applies the controller's compositions answers -// against what it runs from them. An answer is handed only when it changes, and a proposal's answer is dropped when its +// answers once. It runs whether or not the node is placed yet, against what it runs. An answer is handed only when it changes, and a proposal's answer is dropped when its // keys go. type Shadow struct { c *Composer @@ -33,8 +32,7 @@ type Shadow struct { mu sync.Mutex candidates map[string]map[string]candidateKey // by proposal, by node answers map[string]delivery.CandidateAnswer - // wants are the node's wants by module (agent.NodeModule for its own): what a proposal deleting its placement returns - // it to, the controller's compositions at them. + // wants are the node's wants by module (agent.NodeModule for its own); a change of one answers again. wants map[string]delivery.Want // ran is, by module, the digest the module's have says runs: what runs, where the agent's list holds only the copy // from the declaration kept before the switch (issue 1126, probe #532 on shanks). @@ -161,7 +159,7 @@ func (s *Shadow) Run(ctx context.Context) error { {ResolvedBucket, node + ".>", (*Composer).takeValue, nil}, // What this node runs moved: an apply ends in its module's have. {delivery.HaveBucket, node + ".>", nil, nil}, - // Its wants: a proposal deleting its placement returns it to the controller's compositions at them. + // Its wants. {delivery.AssignerWantsBucket, "want." + node + ".>", nil, nil}, } type tagged struct { @@ -530,16 +528,12 @@ func (s *Shadow) answerProposal(ctx context.Context, proposal string) (delivery. running = c.env.Running() } if p, ok := given[node]; ok && p == nil && !own.c.Reached { - // The proposal deletes this node's placement key: the node then returns to the controller's compositions, as one the - // catalogue does not place (endWhenUnplaced). Those at its wants are compared with what runs (issue 1126, the - // seed's revert, whose check read "would stop running node"). - s.mu.Lock() - wants := make(map[string]delivery.Want, len(s.wants)) - for m, w := range s.wants { - wants[m] = w - } - s.mu.Unlock() - return returning(ctx, c.get, node, wants, running, proposal, rev), true + // The proposal deletes this node's placement key: the node would compose nothing and keep running what it runs, + // since no composition made elsewhere is applied (issue 1126: the controller ended by its unassignment). Refused, + // so the catalogue does not take a node's placement away by a merge. + return delivery.CandidateAnswer{Proposal: proposal, Revision: rev, Verdict: delivery.CandidateRefused, + Refused: "the proposal takes this node's placement away: it then composes nothing, and no composition made " + + "elsewhere is applied, so what runs here would be followed by nothing"}, true } placement := given[node] @@ -718,64 +712,6 @@ func (s *Shadow) takeWant(u agent.Update) bool { return !had || old.FP != w.FP } -// returning is the node's answer to a proposal deleting its placement: it returns to the controller's compositions, each -// read at the fingerprint its want names in the node's composition bucket and compared with what runs -// (delivery.CompareComposed). A want the controller holds no composition for, or one that cannot be read, refuses the -// answer, naming each: returning then would leave that module with nothing to apply. -func returning(ctx context.Context, get func(ctx context.Context, bucket, key string) ([]byte, error), node string, - wants map[string]delivery.Want, running []delivery.Composed, proposal string, rev uint64) delivery.CandidateAnswer { - a := delivery.CandidateAnswer{Proposal: proposal, Revision: rev} - bucket, err := delivery.ComposedBucketFor(node) - if err != nil { - a.Refused, a.Verdict = err.Error(), delivery.CandidateRefused - return a - } - modules := make([]string, 0, len(wants)) - for m := range wants { - modules = append(modules, m) - } - sort.Strings(modules) - comps := map[string]delivery.Composed{} - var missing []string - for _, m := range modules { - var key string - if m == agent.NodeModule { - key, err = delivery.ComposedNodeKey(node) - } else { - key, err = delivery.ComposedKey(node, m, wants[m].FP) - } - if err != nil { - missing = append(missing, m+": "+err.Error()) - continue - } - raw, err := get(ctx, bucket, key) - switch { - case err != nil: - missing = append(missing, fmt.Sprintf("%s: the controller's composition %s cannot be read: %v", m, key, err)) - continue - case raw == nil: - missing = append(missing, fmt.Sprintf("%s: the controller holds no composition %s, which its want names", m, key)) - continue - } - comp, err := delivery.DecodeComposed(raw) - if err != nil { - missing = append(missing, fmt.Sprintf("%s: the controller's composition %s: %v", m, key, err)) - continue - } - comps[key] = comp - } - if len(missing) > 0 { - a.Refused = "returning to the controller's compositions, " + strings.Join(missing, "; ") - a.Verdict = delivery.CandidateRefused - return a - } - cmp := delivery.CompareComposed(comps, running) - a.Returns = delivery.ReturnsToController - a.Same, a.Differs, a.Added, a.Removed = cmp.Same, cmp.Differs, cmp.Added, cmp.Removed - a.Verdict = delivery.VerdictOf(a) - return a -} - // sameValue says whether two sealed values open, with this node's key, to the same value; unknown when either does not // open (or the node holds no opener). Only the comparison leaves: never a value (issue 1126, the shadow shows its work). func (c *Composer) sameValue(a, b string) (same, known bool) {