diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index b4e16bd..879d80c 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -535,6 +535,19 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess } }, } + // And its answer to every placement candidate, placed or not, applying nothing (novox/hq ADR 0559 §4): written into + // its own key, `.node`, under candidates. + shadowHooks := hooks + shadowHooks.refused = nil + shadow := selfcompose.NewShadow(composerEnv(session, node, say, shadowHooks), + func(ctx context.Context, answers map[string]delivery.CandidateAnswer) { + session.NoteCandidates(ctx, answers) + }) + go func() { + if err := shadow.Run(linked); err != nil && linked.Err() == nil { + say("this node answers no placement candidate on this link: " + err.Error()) + } + }() if composer := selfComposer(linked, session, node, say, hooks); composer != nil { env.Composed, env.SelfComposed, env.Trusted, env.Policy = composer, true, nil, composer.Policy } else { @@ -850,7 +863,13 @@ func selfComposer(ctx context.Context, session *link.Session, node string, say f return nil } say("this node composes its own modules: the catalogue places it (" + key + ")") - return selfcompose.New(selfcompose.Env{Node: node, + return selfcompose.New(composerEnv(session, node, say, hooks)) +} + +// composerEnv is what the node's composers read of the bus, the vault and the agent: its own composer and its answer +// to placement candidates (selfcompose.Shadow) alike. +func composerEnv(session *link.Session, node string, say func(string), hooks composerHooks) selfcompose.Env { + return selfcompose.Env{Node: node, Watch: func(ctx context.Context, bucket, pattern string) (<-chan agent.Update, error) { return bucketWatch{session, bucket}.Watch(ctx, pattern) }, @@ -869,7 +888,7 @@ func selfComposer(ctx context.Context, session *link.Session, node string, say f }, // And the vault's word that a secret changed, on which it composes again (ADR 0547 §2). Hear: session.Hear, - Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held}) + Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held} } // endWhenPlaced calls stop once this node's placement key appears in the indexer's placement (a value after the watch diff --git a/go.mod b/go.mod index 6c08518..dee61a9 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012259-a37d2622ed56 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index 25c0557..34f6ef5 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa h1:tT6sl3vY git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef h1:uMqybf/sTAjapyllVjy8mjCi8evpcOmgnjmqRdJRsFo= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012259-a37d2622ed56 h1:PUc5s3oaWEg0uyVmXhJSSK3x0IXZBXWGqq30U/CSx30= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012259-a37d2622ed56/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/internal/link/have.go b/internal/link/have.go index 50211f6..d923ee8 100644 --- a/internal/link/have.go +++ b/internal/link/have.go @@ -50,6 +50,8 @@ type haveState struct { // nodeSealed is, on a node composing its own, each value its own composition carries sealed to its key (ADR 0547 §3, // issue 1126): kept in its key so a restart reuses them and applies nothing again. nodeSealed []string + // nodeCandidates is the node's answer to each placement candidate group (novox/hq ADR 0559 §4), written into its key. + nodeCandidates map[string]delivery.CandidateAnswer // written is the build of each module this node-engine wrote a have key for and that the declaration it applied // still holds; removed is each such module the latest declaration no longer holds, with the build its key last // said, until the key is written once with the phase removed (issue 1369, novox/hq ADR 0408 §4: never deleted). @@ -420,6 +422,7 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) { facts := *s.node facts.Composed = s.nodeComposed facts.Sealed = s.nodeSealed + facts.Candidates = s.nodeCandidates if raw, err = delivery.EncodeHaveNode(facts); err == nil { want := facts if !q.putHave(write, bucket, key, raw, func(current []byte) bool { @@ -726,6 +729,29 @@ func (q *Session) NoteNode(ctx context.Context, composed string, sealed []string q.writeHaves(ctx, bus) } +// NoteCandidates takes the node's answers to the placement candidates, by group (its shadow composer's, novox/hq ADR +// 0559 §4), and writes the node's key where it differs; a group left out is no longer answered. +func (q *Session) NoteCandidates(ctx context.Context, answers map[string]delivery.CandidateAnswer) { + q.init() + s := &q.haves + s.mu.Lock() + s.nodeCandidates = answers + if len(answers) == 0 { + s.nodeCandidates = nil + } + // Before the node's facts are first said, its key is not written for an answer alone: it would carry no record. The + // answers ride with the first facts. + known := s.node != nil + s.mu.Unlock() + if !known { + return + } + q.mu.Lock() + bus := q.bus + q.mu.Unlock() + q.writeHaves(ctx, bus) +} + // ReadNodeHave is the node's own key as the bucket holds it, and whether it holds one: what the node's own composition // it last ran is and carries sealed, read after a restart (ADR 0547 §3). func (q *Session) ReadNodeHave(ctx context.Context) (delivery.HaveNode, bool) { diff --git a/internal/link/have_test.go b/internal/link/have_test.go index ee263ac..309fb4d 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -589,6 +589,16 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth( if n, ok := q.ReadNodeHave(context.Background()); !ok || n.Composed != dig || len(n.Sealed) != 1 { t.Fatalf("the node key read back: %+v %v", n, ok) } + // The node's answers to placement candidates ride in the same key (ADR 0559 §4), and go when no group is answered. + answers := map[string]delivery.CandidateAnswer{"feat/x": {Head: "h", Verdict: delivery.CandidateSame, Same: []string{"web"}}} + q.NoteCandidates(context.Background(), answers) + if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Candidates["feat/x"].Head != "h" || n.Composed != dig { + t.Fatalf("the node key with its answers: %+v %v", n, err) + } + q.NoteCandidates(context.Background(), nil) + if n, _ := delivery.DecodeHaveNode(l.kv["n1.node"].value); n.Candidates != nil { + t.Fatalf("an answer left after its group went: %+v", n.Candidates) + } } // A module applied and running that has nothing long-running to judge is ok once the engine has looked: there is nothing diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index a7d220d..913335a 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -10,6 +10,7 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/compose/frombus" "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-host/internal/agent" @@ -317,4 +318,216 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { if asked, _ := json.Marshal(sent["credentials"]); !strings.Contains(string(asked), `"role":"consumer"`) || !strings.Contains(string(asked), `"role":"provider"`) { t.Errorf("the vault was asked %s", asked) } + // ADR 0559 §4: the node answers placement candidates in shadow, placed or not, applying nothing and handing no + // composition: the same; a module the group adds, its secret not yet issued; a module removed; a placement it cannot + // compose; a group whose keys went is no longer answered. What runs is never composed from a candidate: the + // Composer does not read the state. + for _, in := range New(env).inputs() { + if in.bucket == delivery.PlacementCandidatesBucket { + t.Fatal("the node's own composer reads placement candidates") + } + } + sb := newPatternBus() + trunk := delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"}, Modules: []delivery.PlacedModule{{Module: "lemurs"}}} + mods := map[string]delivery.Module{"lemurs": {FP2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"}, + "keeper": {FP2: fp, Manifest: kdigest, Repository: "novox/k", Commit: "c"}} + readOnly := &fakeBus{kv: map[string][]byte{delivery.ManifestsBucket + "/" + key: man, delivery.ManifestsBucket + "/" + kkey: kman}} + senv := readOnly.env() + senv.Watch = sb.watch + senv.Own = func(context.Context, map[string]any) (json.RawMessage, error) { + return json.RawMessage(`{"node":"n1","key":"sk","own":[]}`), nil // nothing issued for keeper: it is not placed here + } + // What n1 runs: lemurs and its own composition, as its composer composes them from the trunk. + running, _, err := frombus.Compose(ctx, fixedReader{placements: map[string]delivery.Placement{"n1": trunk}, + records: map[string]delivery.NodeRecord{"n1": {Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}}, + modules: mods, manifests: map[string][]byte{digest: []byte(lemurs), kdigest: []byte(keeper)}}, frombus.Mesh{}, frombus.Scope{Node: "n1"}, map[string]string{"lemurs": fp}) + if err != nil { + t.Fatal(err) + } + senv.Running = func() []delivery.Composed { + var out []delivery.Composed + for _, c := range running { + out = append(out, c) + } + return out + } + answered := make(chan map[string]delivery.CandidateAnswer, 8) + sh := NewShadow(senv, func(_ context.Context, a map[string]delivery.CandidateAnswer) { answered <- a }) + sctx, scancel := context.WithCancel(ctx) + defer scancel() + go func() { _ = sh.Run(sctx) }() + sb.made(6) + rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}}) + tp, _ := delivery.EncodePlacement(trunk) + sb.put(delivery.PlacementBucket, "placement.n1", tp, 1) + sb.put(delivery.HaveBucket, "n1.node", rec, 1) + for name, m := range mods { + raw, _ := delivery.EncodeModule(m) + sb.put(delivery.ModulesBucket, name, raw, 1) + } + candidate := func(group string, rev uint64, modules ...string) { + p := trunk + p.Modules = nil + for _, m := range modules { + p.Modules = append(p.Modules, delivery.PlacedModule{Module: m}) + } + raw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Group: group, + Pull: delivery.Head{Repository: delivery.CatalogueRepository, Number: 1, Head: "h-" + group}, Placement: &p}) + if err != nil { + t.Fatal(err) + } + k, _ := delivery.PlacementCandidateKey(group, "n1") + sb.put(delivery.PlacementCandidatesBucket, k, raw, rev) + } + candidate("same", 3, "lemurs") + candidate("adds", 4, "lemurs", "keeper") + candidate("drops", 5) + candidate("ghost", 6, "lemurs", "ghost") + sb.caught() + var answers map[string]delivery.CandidateAnswer + waitFor := func(why string, ok func(map[string]delivery.CandidateAnswer) bool) { + t.Helper() + deadline := time.After(10 * time.Second) + for answers == nil || !ok(answers) { + select { + case answers = <-answered: + case <-deadline: + t.Fatalf("%s: last answers %+v", why, answers) + } + } + } + waitFor("every group answered", func(a map[string]delivery.CandidateAnswer) bool { return len(a) == 4 }) + for group, want := range map[string]func(delivery.CandidateAnswer) bool{ + "same": func(a delivery.CandidateAnswer) bool { + return a.Verdict == delivery.CandidateSame && a.Head == "h-same" && a.Revision == 3 && len(a.Same) == 2 + }, + "adds": func(a delivery.CandidateAnswer) bool { + return a.Verdict == delivery.CandidateDiffers && len(a.Added) == 1 && a.Added[0].Module == "keeper" && + len(a.Added[0].Unissued) == 1 && len(a.Refusals) == 0 + }, + "drops": func(a delivery.CandidateAnswer) bool { + return a.Verdict == delivery.CandidateDiffers && len(a.Removed) == 1 && a.Removed[0] == "lemurs" + }, + "ghost": func(a delivery.CandidateAnswer) bool { + return a.Verdict == delivery.CandidateRefused && a.Refused != "" + }, + } { + if !want(answers[group]) { + t.Errorf("%s: %+v", group, answers[group]) + } + } + // The node's own key changing (its answers written) composes nothing again; a group's key deleted drops its answer. + sb.put(delivery.HaveBucket, "n1.node", rec, 2) + select { + case a := <-answered: + t.Errorf("answers handed again for nothing changed: %+v", a) + case <-time.After(100 * time.Millisecond): + } + k, _ := delivery.PlacementCandidateKey("same", "n1") + sb.del(delivery.PlacementCandidatesBucket, k) + waitFor("the deleted group's answer stays", func(a map[string]delivery.CandidateAnswer) bool { _, ok := a["same"]; return !ok && len(a) == 3 }) +} + +// patternBus is a bus whose every watch gets the updates of its bucket matching its pattern, as the bus's own watches +// do: two watches of one bucket each see what matches theirs. +type patternBus struct { + mu sync.Mutex + watches []patternWatch +} + +type patternWatch struct { + bucket, pattern string + ch chan agent.Update +} + +func newPatternBus() *patternBus { return &patternBus{} } + +func (b *patternBus) watch(_ context.Context, bucket, pattern string) (<-chan agent.Update, error) { + b.mu.Lock() + defer b.mu.Unlock() + w := patternWatch{bucket: bucket, pattern: pattern, ch: make(chan agent.Update, 64)} + b.watches = append(b.watches, w) + return w.ch, nil +} + +func matches(pattern, key string) bool { + p, k := strings.Split(pattern, "."), strings.Split(key, ".") + for i, tok := range p { + if tok == ">" { + return len(k) > i + } + if i >= len(k) || (tok != "*" && tok != k[i]) { + return false + } + } + return len(p) == len(k) +} + +func (b *patternBus) send(bucket string, u agent.Update) { + b.mu.Lock() + defer b.mu.Unlock() + for _, w := range b.watches { + if w.bucket == bucket && (u.Caught || matches(w.pattern, u.Key)) { + w.ch <- u + } + } +} + +func (b *patternBus) put(bucket, key string, value []byte, rev uint64) { + b.send(bucket, agent.Update{Key: key, Value: value, Revision: rev}) +} + +func (b *patternBus) del(bucket, key string) { b.send(bucket, agent.Update{Key: key, Deleted: true}) } + +// made waits until n watches are made, so nothing put is lost. +func (b *patternBus) made(n int) { + for { + b.mu.Lock() + got := len(b.watches) + b.mu.Unlock() + if got >= n { + return + } + time.Sleep(5 * time.Millisecond) + } +} + +// caught says every watch caught up. +func (b *patternBus) caught() { + b.mu.Lock() + defer b.mu.Unlock() + for _, w := range b.watches { + w.ch <- agent.Update{Caught: true} + } +} + +// fixedReader is frombus's Reader over values held, for composing what a node runs. +type fixedReader struct { + placements map[string]delivery.Placement + records map[string]delivery.NodeRecord + modules map[string]delivery.Module + manifests map[string][]byte +} + +func (r fixedReader) Placements(context.Context) (map[string]delivery.Placement, error) { + return r.placements, nil +} +func (r fixedReader) Records(context.Context) (map[string]delivery.NodeRecord, error) { + return r.records, nil +} +func (r fixedReader) Modules(context.Context) (map[string]delivery.Module, error) { + return r.modules, nil +} +func (r fixedReader) Manifest(_ context.Context, d string) ([]byte, error) { + if raw, ok := r.manifests[d]; ok { + return raw, nil + } + return nil, fmt.Errorf("no manifest %s", d) +} +func (r fixedReader) Values(context.Context, string, string) (map[string]any, error) { return nil, nil } +func (r fixedReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) { + return nil, nil +} +func (r fixedReader) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) { + return nil, nil, nil } diff --git a/internal/selfcompose/shadow.go b/internal/selfcompose/shadow.go new file mode 100644 index 0000000..20b4ce3 --- /dev/null +++ b/internal/selfcompose/shadow.go @@ -0,0 +1,318 @@ +package selfcompose + +import ( + "context" + "fmt" + "sort" + "sync" + + "git.novox.be/novox/mesh-sdk/go/compose/frombus" + "git.novox.be/novox/mesh-sdk/go/delivery" + + "github.com/novox/mesh-host/internal/agent" +) + +// Shadow is the node's answer to placement candidates (novox/hq ADR 0559 §4, issue 1126): for every group with a key in +// the indexer's state `placement-candidates`, it composes this node in shadow from the trunk's placement with the group's +// keys laid over it and its own inputs (the sealed values and boxes it holds among them), compares the result with what +// the node runs, and hands the answers to be written into the node's own have key, `.node`, under `candidates`. +// It applies nothing and hands no composition: what runs is still composed from `placement` alone, by the Composer. +// +// **Event-driven, like the Composer.** It watches every candidate key, the trunk's placement, every node's record, the +// indexer's modules, this node's config values, and this node's module haves (what it runs moved); a burst of changes +// answers once. It runs whether or not the node is placed yet: a node that applies the controller's compositions answers +// against what it runs from them. An answer is handed only when it changes, and a group's answer is dropped when its +// keys go. +type Shadow struct { + c *Composer + answer func(ctx context.Context, answers map[string]delivery.CandidateAnswer) + + mu sync.Mutex + candidates map[string]map[string]candidateKey // by group, by node + answers map[string]delivery.CandidateAnswer +} + +// candidateKey is one candidate key as last read: its value and revision. +type candidateKey struct { + c delivery.PlacementCandidate + rev uint64 +} + +// NewShadow is the shadow composer of one node; answer writes the node's answers into its own have key. +func NewShadow(env Env, answer func(ctx context.Context, answers map[string]delivery.CandidateAnswer)) *Shadow { + return &Shadow{c: New(env), answer: answer, candidates: map[string]map[string]candidateKey{}, + answers: map[string]delivery.CandidateAnswer{}} +} + +// Run answers until ctx ends or an input's watch ends. +func (s *Shadow) Run(ctx context.Context) error { + c := s.c + node := c.env.Node + ins := []input{ + {delivery.PlacementCandidatesBucket, ">", nil}, + {delivery.PlacementBucket, ">", (*Composer).takePlacement}, + {delivery.HaveBucket, "*." + delivery.HaveNodeWord, nil}, + {delivery.ModulesBucket, ">", (*Composer).takeModule}, + {ResolvedBucket, node + ".>", (*Composer).takeValue}, + // What this node runs moved: an apply ends in its module's have. + {delivery.HaveBucket, node + ".*", nil}, + } + type tagged struct { + in int + u agent.Update + ok bool + } + merged := make(chan tagged, 64) + for i, in := range ins { + ch, err := c.env.Watch(ctx, in.bucket, in.pattern) + if err != nil { + return fmt.Errorf("the node answers placement candidates from %s, which cannot be watched: %w", in.bucket, err) + } + go func(i int, ch <-chan agent.Update) { + for u := range ch { + select { + case merged <- tagged{in: i, u: u, ok: true}: + case <-ctx.Done(): + return + } + } + select { + case merged <- tagged{in: i}: + case <-ctx.Done(): + } + }(i, ch) + } + caught := make([]bool, len(ins)) + ready, dirty := false, false + for { + select { + case <-ctx.Done(): + return nil + case t := <-merged: + if !t.ok { + return fmt.Errorf("the watch of %s ended; the node answers no placement candidate until its agent starts again", ins[t.in].bucket) + } + switch { + case t.u.Caught: + caught[t.in] = true + case t.in == 0: + dirty = s.takeCandidate(t.u) || dirty + case t.in == 2: + dirty = s.takeRecord(t.u) || dirty + case t.in == 5: + dirty = !isNodeKey(t.u.Key) || dirty + default: + ins[t.in].take(c, t.u) + dirty = true + } + if len(merged) > 0 { + continue + } + } + if !ready { + if ready = all(caught); !ready { + continue + } + dirty = true + } + if dirty { + dirty = false + s.answerAll(ctx) + } + } +} + +func isNodeKey(key string) bool { + _, _, isNode, err := delivery.ParseHaveKey(key) + return err == nil && isNode +} + +// takeCandidate takes one candidate key, and says whether it changed anything. +func (s *Shadow) takeCandidate(u agent.Update) bool { + group, node, err := delivery.ParsePlacementCandidateKey(u.Key) + if err != nil { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + if u.Deleted { + if _, ok := s.candidates[group][node]; !ok { + return false + } + delete(s.candidates[group], node) + if len(s.candidates[group]) == 0 { + delete(s.candidates, group) + } + return true + } + c, err := delivery.DecodePlacementCandidate(u.Value) + if err != nil || c.Group != group || (c.Placement != nil && c.Placement.Node != node) { + // Not one: read as gone. + if _, ok := s.candidates[group][node]; ok { + delete(s.candidates[group], node) + return true + } + return false + } + if s.candidates[group] == nil { + s.candidates[group] = map[string]candidateKey{} + } + s.candidates[group][node] = candidateKey{c: c, rev: u.Revision} + return true +} + +// takeRecord takes a node's record, and says whether it changed: the node's own key changes with each answer written, +// which moves nothing composed. +func (s *Shadow) takeRecord(u agent.Update) bool { + node, _, isNode, err := delivery.ParseHaveKey(u.Key) + if err != nil || !isNode { + return false + } + c := s.c + c.mu.Lock() + before, had := c.records[node] + c.mu.Unlock() + c.takeRecord(u) + c.mu.Lock() + after, has := c.records[node] + c.mu.Unlock() + return had != has || (has && !delivery.SameNodeRecord(before, after)) +} + +// answerAll answers every group and hands the answers when any changed. +func (s *Shadow) answerAll(ctx context.Context) { + c := s.c + node := c.env.Node + s.mu.Lock() + groups := make([]string, 0, len(s.candidates)) + for g := range s.candidates { + groups = append(groups, g) + } + s.mu.Unlock() + sort.Strings(groups) + next := map[string]delivery.CandidateAnswer{} + for _, group := range groups { + if a, ok := s.answerGroup(ctx, group); ok { + next[group] = a + } + } + s.mu.Lock() + same := len(next) == len(s.answers) + for g, a := range next { + if old, ok := s.answers[g]; !ok || !sameAnswer(old, a) { + same = false + } + } + if !same { + s.answers = next + } + s.mu.Unlock() + if !same && s.answer != nil { + out := make(map[string]delivery.CandidateAnswer, len(next)) + for g, a := range next { + out[g] = a + } + s.answer(ctx, out) + c.env.Say(fmt.Sprintf("%s answered %d placement candidate group(s)", node, len(out))) + } +} + +// answerGroup is the node's answer to one group, and false when the node has nothing to answer: neither the trunk nor +// the group places it. +func (s *Shadow) answerGroup(ctx context.Context, group string) (delivery.CandidateAnswer, bool) { + c := s.c + node := c.env.Node + s.mu.Lock() + keys := s.candidates[group] + given := make(map[string]*delivery.Placement, len(keys)) + var head string + for n, k := range keys { + given[n] = k.c.Placement + head = k.c.Pull.Head + } + // The revision answered is the node's own key's, 0 when the group has none for it (judged by the head alone). + own, mine := keys[node] + rev := own.rev + s.mu.Unlock() + + c.mu.Lock() + _, placed := c.placements[node] + modules := make(map[string]delivery.Module, len(c.modules)) + for k, v := range c.modules { + modules[k] = v + } + c.mu.Unlock() + if !placed && !mine { + return delivery.CandidateAnswer{}, false + } + + var running []delivery.Composed + if c.env.Running != nil { + running = c.env.Running() + } + if p, ok := given[node]; ok && p == nil { + // The head removes this node's placement file: everything it runs is removed. + a := delivery.CandidateAnswer{Head: head, Revision: rev} + for _, r := range running { + if r.Module == "" { + a.Removed = append(a.Removed, delivery.HaveNodeWord) + } else { + a.Removed = append(a.Removed, r.Module) + } + } + sort.Strings(a.Removed) + a.Verdict = delivery.VerdictOf(a) + return a, true + } + + // Each module at the build it runs; one the candidate adds at the trunk's build. + fps := map[string]string{} + kept := map[string]string{} + for _, r := range running { + if r.Module != "" && r.FP != "" { + fps[r.Module] = r.FP + } + } + placement := given[node] + if placement == nil { + c.mu.Lock() + p := c.placements[node] + c.mu.Unlock() + placement = &p + } + for _, m := range placement.Modules { + if _, runs := fps[m.Module]; !runs { + if mod, ok := modules[m.Module]; ok && mod.FP2 != "" { + fps[m.Module] = mod.FP2 + } + } + } + for module, fp := range fps { + if mod, ok := modules[module]; ok && mod.FP2 == fp { + continue + } + digest, err := c.builtManifest(ctx, module, fp) + if err != nil { + return delivery.CandidateAnswer{Head: head, Revision: rev, Verdict: delivery.CandidateRefused, + Refused: fmt.Sprintf("%s runs at %s, whose manifest cannot be read: %v", module, fp, err)}, true + } + kept[module] = digest + } + // The boxes it runs, by what each module's have and its own key say after a restart (Held), are reused for the + // modules it runs: a value sealed again is no difference. + held := map[string]delivery.Want{agent.NodeModule: {}} + for module := range fps { + held[module] = delivery.Want{FP: fps[module]} + } + c.mu.Lock() + c.wants = held + c.mu.Unlock() + a := frombus.Answer(ctx, reader{c: c}, c.env.Mesh, frombus.Scope{Node: node, Kept: kept, Placements: given}, fps, running, head, rev) + return a, true +} + +func sameAnswer(a, b delivery.CandidateAnswer) bool { + x, errA := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"g": a}}) + y, errB := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"g": b}}) + return errA == nil && errB == nil && string(x) == string(y) +}