Answer every placement candidate in shadow in the node's own key, applying nothing, so a placement change is checked by the nodes it places before it merges (issue 1126, ADR 0559 §4)
This commit is contained in:
@@ -535,6 +535,19 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, sess
|
||||
}
|
||||
},
|
||||
}
|
||||
// And its answer to every placement candidate, placed or not, applying nothing (novox/hq ADR 0559 §4): written into
|
||||
// its own key, `<node>.node`, under candidates.
|
||||
shadowHooks := hooks
|
||||
shadowHooks.refused = nil
|
||||
shadow := selfcompose.NewShadow(composerEnv(session, node, say, shadowHooks),
|
||||
func(ctx context.Context, answers map[string]delivery.CandidateAnswer) {
|
||||
session.NoteCandidates(ctx, answers)
|
||||
})
|
||||
go func() {
|
||||
if err := shadow.Run(linked); err != nil && linked.Err() == nil {
|
||||
say("this node answers no placement candidate on this link: " + err.Error())
|
||||
}
|
||||
}()
|
||||
if composer := selfComposer(linked, session, node, say, hooks); composer != nil {
|
||||
env.Composed, env.SelfComposed, env.Trusted, env.Policy = composer, true, nil, composer.Policy
|
||||
} else {
|
||||
@@ -850,7 +863,13 @@ func selfComposer(ctx context.Context, session *link.Session, node string, say f
|
||||
return nil
|
||||
}
|
||||
say("this node composes its own modules: the catalogue places it (" + key + ")")
|
||||
return selfcompose.New(selfcompose.Env{Node: node,
|
||||
return selfcompose.New(composerEnv(session, node, say, hooks))
|
||||
}
|
||||
|
||||
// composerEnv is what the node's composers read of the bus, the vault and the agent: its own composer and its answer
|
||||
// to placement candidates (selfcompose.Shadow) alike.
|
||||
func composerEnv(session *link.Session, node string, say func(string), hooks composerHooks) selfcompose.Env {
|
||||
return selfcompose.Env{Node: node,
|
||||
Watch: func(ctx context.Context, bucket, pattern string) (<-chan agent.Update, error) {
|
||||
return bucketWatch{session, bucket}.Watch(ctx, pattern)
|
||||
},
|
||||
@@ -869,7 +888,7 @@ func selfComposer(ctx context.Context, session *link.Session, node string, say f
|
||||
},
|
||||
// And the vault's word that a secret changed, on which it composes again (ADR 0547 §2).
|
||||
Hear: session.Hear,
|
||||
Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held})
|
||||
Running: hooks.running, Unseal: hooks.unseal, Refused: hooks.refused, Held: hooks.held}
|
||||
}
|
||||
|
||||
// endWhenPlaced calls stop once this node's placement key appears in the indexer's placement (a value after the watch
|
||||
|
||||
@@ -3,7 +3,7 @@ module github.com/novox/mesh-host
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012259-a37d2622ed56
|
||||
github.com/nats-io/nats-server/v2 v2.11.17
|
||||
github.com/nats-io/nats.go v1.54.0
|
||||
golang.org/x/crypto v0.57.0
|
||||
|
||||
@@ -4,6 +4,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa h1:tT6sl3vY
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011211-82f10c3bc9fa/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef h1:uMqybf/sTAjapyllVjy8mjCi8evpcOmgnjmqRdJRsFo=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027011927-ba398e2460ef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012259-a37d2622ed56 h1:PUc5s3oaWEg0uyVmXhJSSK3x0IXZBXWGqq30U/CSx30=
|
||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261027012259-a37d2622ed56/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||
|
||||
@@ -50,6 +50,8 @@ type haveState struct {
|
||||
// nodeSealed is, on a node composing its own, each value its own composition carries sealed to its key (ADR 0547 §3,
|
||||
// issue 1126): kept in its key so a restart reuses them and applies nothing again.
|
||||
nodeSealed []string
|
||||
// nodeCandidates is the node's answer to each placement candidate group (novox/hq ADR 0559 §4), written into its key.
|
||||
nodeCandidates map[string]delivery.CandidateAnswer
|
||||
// written is the build of each module this node-engine wrote a have key for and that the declaration it applied
|
||||
// still holds; removed is each such module the latest declaration no longer holds, with the build its key last
|
||||
// said, until the key is written once with the phase removed (issue 1369, novox/hq ADR 0408 §4: never deleted).
|
||||
@@ -420,6 +422,7 @@ func (q *Session) writeHaves(ctx context.Context, bus Bus) {
|
||||
facts := *s.node
|
||||
facts.Composed = s.nodeComposed
|
||||
facts.Sealed = s.nodeSealed
|
||||
facts.Candidates = s.nodeCandidates
|
||||
if raw, err = delivery.EncodeHaveNode(facts); err == nil {
|
||||
want := facts
|
||||
if !q.putHave(write, bucket, key, raw, func(current []byte) bool {
|
||||
@@ -726,6 +729,29 @@ func (q *Session) NoteNode(ctx context.Context, composed string, sealed []string
|
||||
q.writeHaves(ctx, bus)
|
||||
}
|
||||
|
||||
// NoteCandidates takes the node's answers to the placement candidates, by group (its shadow composer's, novox/hq ADR
|
||||
// 0559 §4), and writes the node's key where it differs; a group left out is no longer answered.
|
||||
func (q *Session) NoteCandidates(ctx context.Context, answers map[string]delivery.CandidateAnswer) {
|
||||
q.init()
|
||||
s := &q.haves
|
||||
s.mu.Lock()
|
||||
s.nodeCandidates = answers
|
||||
if len(answers) == 0 {
|
||||
s.nodeCandidates = nil
|
||||
}
|
||||
// Before the node's facts are first said, its key is not written for an answer alone: it would carry no record. The
|
||||
// answers ride with the first facts.
|
||||
known := s.node != nil
|
||||
s.mu.Unlock()
|
||||
if !known {
|
||||
return
|
||||
}
|
||||
q.mu.Lock()
|
||||
bus := q.bus
|
||||
q.mu.Unlock()
|
||||
q.writeHaves(ctx, bus)
|
||||
}
|
||||
|
||||
// ReadNodeHave is the node's own key as the bucket holds it, and whether it holds one: what the node's own composition
|
||||
// it last ran is and carries sealed, read after a restart (ADR 0547 §3).
|
||||
func (q *Session) ReadNodeHave(ctx context.Context) (delivery.HaveNode, bool) {
|
||||
|
||||
@@ -589,6 +589,16 @@ func TestAnAppliedModulesHaveNamesTheFingerprintAndTheDigestAndFollowsItsHealth(
|
||||
if n, ok := q.ReadNodeHave(context.Background()); !ok || n.Composed != dig || len(n.Sealed) != 1 {
|
||||
t.Fatalf("the node key read back: %+v %v", n, ok)
|
||||
}
|
||||
// The node's answers to placement candidates ride in the same key (ADR 0559 §4), and go when no group is answered.
|
||||
answers := map[string]delivery.CandidateAnswer{"feat/x": {Head: "h", Verdict: delivery.CandidateSame, Same: []string{"web"}}}
|
||||
q.NoteCandidates(context.Background(), answers)
|
||||
if n, err := delivery.DecodeHaveNode(l.kv["n1.node"].value); err != nil || n.Candidates["feat/x"].Head != "h" || n.Composed != dig {
|
||||
t.Fatalf("the node key with its answers: %+v %v", n, err)
|
||||
}
|
||||
q.NoteCandidates(context.Background(), nil)
|
||||
if n, _ := delivery.DecodeHaveNode(l.kv["n1.node"].value); n.Candidates != nil {
|
||||
t.Fatalf("an answer left after its group went: %+v", n.Candidates)
|
||||
}
|
||||
}
|
||||
|
||||
// A module applied and running that has nothing long-running to judge is ok once the engine has looked: there is nothing
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
@@ -317,4 +318,216 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
||||
if asked, _ := json.Marshal(sent["credentials"]); !strings.Contains(string(asked), `"role":"consumer"`) || !strings.Contains(string(asked), `"role":"provider"`) {
|
||||
t.Errorf("the vault was asked %s", asked)
|
||||
}
|
||||
// ADR 0559 §4: the node answers placement candidates in shadow, placed or not, applying nothing and handing no
|
||||
// composition: the same; a module the group adds, its secret not yet issued; a module removed; a placement it cannot
|
||||
// compose; a group whose keys went is no longer answered. What runs is never composed from a candidate: the
|
||||
// Composer does not read the state.
|
||||
for _, in := range New(env).inputs() {
|
||||
if in.bucket == delivery.PlacementCandidatesBucket {
|
||||
t.Fatal("the node's own composer reads placement candidates")
|
||||
}
|
||||
}
|
||||
sb := newPatternBus()
|
||||
trunk := delivery.Placement{Node: "n1", Decided: delivery.NodeDecided{Hub: true, Address: "10.77.0.1"}, Modules: []delivery.PlacedModule{{Module: "lemurs"}}}
|
||||
mods := map[string]delivery.Module{"lemurs": {FP2: fp, Manifest: digest, Repository: "novox/x", Commit: "c"},
|
||||
"keeper": {FP2: fp, Manifest: kdigest, Repository: "novox/k", Commit: "c"}}
|
||||
readOnly := &fakeBus{kv: map[string][]byte{delivery.ManifestsBucket + "/" + key: man, delivery.ManifestsBucket + "/" + kkey: kman}}
|
||||
senv := readOnly.env()
|
||||
senv.Watch = sb.watch
|
||||
senv.Own = func(context.Context, map[string]any) (json.RawMessage, error) {
|
||||
return json.RawMessage(`{"node":"n1","key":"sk","own":[]}`), nil // nothing issued for keeper: it is not placed here
|
||||
}
|
||||
// What n1 runs: lemurs and its own composition, as its composer composes them from the trunk.
|
||||
running, _, err := frombus.Compose(ctx, fixedReader{placements: map[string]delivery.Placement{"n1": trunk},
|
||||
records: map[string]delivery.NodeRecord{"n1": {Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}},
|
||||
modules: mods, manifests: map[string][]byte{digest: []byte(lemurs), kdigest: []byte(keeper)}}, frombus.Mesh{}, frombus.Scope{Node: "n1"}, map[string]string{"lemurs": fp})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
senv.Running = func() []delivery.Composed {
|
||||
var out []delivery.Composed
|
||||
for _, c := range running {
|
||||
out = append(out, c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
answered := make(chan map[string]delivery.CandidateAnswer, 8)
|
||||
sh := NewShadow(senv, func(_ context.Context, a map[string]delivery.CandidateAnswer) { answered <- a })
|
||||
sctx, scancel := context.WithCancel(ctx)
|
||||
defer scancel()
|
||||
go func() { _ = sh.Run(sctx) }()
|
||||
sb.made(6)
|
||||
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}})
|
||||
tp, _ := delivery.EncodePlacement(trunk)
|
||||
sb.put(delivery.PlacementBucket, "placement.n1", tp, 1)
|
||||
sb.put(delivery.HaveBucket, "n1.node", rec, 1)
|
||||
for name, m := range mods {
|
||||
raw, _ := delivery.EncodeModule(m)
|
||||
sb.put(delivery.ModulesBucket, name, raw, 1)
|
||||
}
|
||||
candidate := func(group string, rev uint64, modules ...string) {
|
||||
p := trunk
|
||||
p.Modules = nil
|
||||
for _, m := range modules {
|
||||
p.Modules = append(p.Modules, delivery.PlacedModule{Module: m})
|
||||
}
|
||||
raw, err := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Group: group,
|
||||
Pull: delivery.Head{Repository: delivery.CatalogueRepository, Number: 1, Head: "h-" + group}, Placement: &p})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k, _ := delivery.PlacementCandidateKey(group, "n1")
|
||||
sb.put(delivery.PlacementCandidatesBucket, k, raw, rev)
|
||||
}
|
||||
candidate("same", 3, "lemurs")
|
||||
candidate("adds", 4, "lemurs", "keeper")
|
||||
candidate("drops", 5)
|
||||
candidate("ghost", 6, "lemurs", "ghost")
|
||||
sb.caught()
|
||||
var answers map[string]delivery.CandidateAnswer
|
||||
waitFor := func(why string, ok func(map[string]delivery.CandidateAnswer) bool) {
|
||||
t.Helper()
|
||||
deadline := time.After(10 * time.Second)
|
||||
for answers == nil || !ok(answers) {
|
||||
select {
|
||||
case answers = <-answered:
|
||||
case <-deadline:
|
||||
t.Fatalf("%s: last answers %+v", why, answers)
|
||||
}
|
||||
}
|
||||
}
|
||||
waitFor("every group answered", func(a map[string]delivery.CandidateAnswer) bool { return len(a) == 4 })
|
||||
for group, want := range map[string]func(delivery.CandidateAnswer) bool{
|
||||
"same": func(a delivery.CandidateAnswer) bool {
|
||||
return a.Verdict == delivery.CandidateSame && a.Head == "h-same" && a.Revision == 3 && len(a.Same) == 2
|
||||
},
|
||||
"adds": func(a delivery.CandidateAnswer) bool {
|
||||
return a.Verdict == delivery.CandidateDiffers && len(a.Added) == 1 && a.Added[0].Module == "keeper" &&
|
||||
len(a.Added[0].Unissued) == 1 && len(a.Refusals) == 0
|
||||
},
|
||||
"drops": func(a delivery.CandidateAnswer) bool {
|
||||
return a.Verdict == delivery.CandidateDiffers && len(a.Removed) == 1 && a.Removed[0] == "lemurs"
|
||||
},
|
||||
"ghost": func(a delivery.CandidateAnswer) bool {
|
||||
return a.Verdict == delivery.CandidateRefused && a.Refused != ""
|
||||
},
|
||||
} {
|
||||
if !want(answers[group]) {
|
||||
t.Errorf("%s: %+v", group, answers[group])
|
||||
}
|
||||
}
|
||||
// The node's own key changing (its answers written) composes nothing again; a group's key deleted drops its answer.
|
||||
sb.put(delivery.HaveBucket, "n1.node", rec, 2)
|
||||
select {
|
||||
case a := <-answered:
|
||||
t.Errorf("answers handed again for nothing changed: %+v", a)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
k, _ := delivery.PlacementCandidateKey("same", "n1")
|
||||
sb.del(delivery.PlacementCandidatesBucket, k)
|
||||
waitFor("the deleted group's answer stays", func(a map[string]delivery.CandidateAnswer) bool { _, ok := a["same"]; return !ok && len(a) == 3 })
|
||||
}
|
||||
|
||||
// patternBus is a bus whose every watch gets the updates of its bucket matching its pattern, as the bus's own watches
|
||||
// do: two watches of one bucket each see what matches theirs.
|
||||
type patternBus struct {
|
||||
mu sync.Mutex
|
||||
watches []patternWatch
|
||||
}
|
||||
|
||||
type patternWatch struct {
|
||||
bucket, pattern string
|
||||
ch chan agent.Update
|
||||
}
|
||||
|
||||
func newPatternBus() *patternBus { return &patternBus{} }
|
||||
|
||||
func (b *patternBus) watch(_ context.Context, bucket, pattern string) (<-chan agent.Update, error) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
w := patternWatch{bucket: bucket, pattern: pattern, ch: make(chan agent.Update, 64)}
|
||||
b.watches = append(b.watches, w)
|
||||
return w.ch, nil
|
||||
}
|
||||
|
||||
func matches(pattern, key string) bool {
|
||||
p, k := strings.Split(pattern, "."), strings.Split(key, ".")
|
||||
for i, tok := range p {
|
||||
if tok == ">" {
|
||||
return len(k) > i
|
||||
}
|
||||
if i >= len(k) || (tok != "*" && tok != k[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(p) == len(k)
|
||||
}
|
||||
|
||||
func (b *patternBus) send(bucket string, u agent.Update) {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
for _, w := range b.watches {
|
||||
if w.bucket == bucket && (u.Caught || matches(w.pattern, u.Key)) {
|
||||
w.ch <- u
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (b *patternBus) put(bucket, key string, value []byte, rev uint64) {
|
||||
b.send(bucket, agent.Update{Key: key, Value: value, Revision: rev})
|
||||
}
|
||||
|
||||
func (b *patternBus) del(bucket, key string) { b.send(bucket, agent.Update{Key: key, Deleted: true}) }
|
||||
|
||||
// made waits until n watches are made, so nothing put is lost.
|
||||
func (b *patternBus) made(n int) {
|
||||
for {
|
||||
b.mu.Lock()
|
||||
got := len(b.watches)
|
||||
b.mu.Unlock()
|
||||
if got >= n {
|
||||
return
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// caught says every watch caught up.
|
||||
func (b *patternBus) caught() {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
for _, w := range b.watches {
|
||||
w.ch <- agent.Update{Caught: true}
|
||||
}
|
||||
}
|
||||
|
||||
// fixedReader is frombus's Reader over values held, for composing what a node runs.
|
||||
type fixedReader struct {
|
||||
placements map[string]delivery.Placement
|
||||
records map[string]delivery.NodeRecord
|
||||
modules map[string]delivery.Module
|
||||
manifests map[string][]byte
|
||||
}
|
||||
|
||||
func (r fixedReader) Placements(context.Context) (map[string]delivery.Placement, error) {
|
||||
return r.placements, nil
|
||||
}
|
||||
func (r fixedReader) Records(context.Context) (map[string]delivery.NodeRecord, error) {
|
||||
return r.records, nil
|
||||
}
|
||||
func (r fixedReader) Modules(context.Context) (map[string]delivery.Module, error) {
|
||||
return r.modules, nil
|
||||
}
|
||||
func (r fixedReader) Manifest(_ context.Context, d string) ([]byte, error) {
|
||||
if raw, ok := r.manifests[d]; ok {
|
||||
return raw, nil
|
||||
}
|
||||
return nil, fmt.Errorf("no manifest %s", d)
|
||||
}
|
||||
func (r fixedReader) Values(context.Context, string, string) (map[string]any, error) { return nil, nil }
|
||||
func (r fixedReader) OwnSecrets(context.Context, string) ([]compose.OwnSecretRow, error) {
|
||||
return nil, nil
|
||||
}
|
||||
func (r fixedReader) Credentials(context.Context, string, []compose.CredentialKey, []compose.CredentialKey) ([]compose.Credential, []compose.Issued, error) {
|
||||
return nil, nil, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,318 @@
|
||||
package selfcompose
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/compose/frombus"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"github.com/novox/mesh-host/internal/agent"
|
||||
)
|
||||
|
||||
// Shadow is the node's answer to placement candidates (novox/hq ADR 0559 §4, issue 1126): for every group with a key in
|
||||
// the indexer's state `placement-candidates`, it composes this node in shadow from the trunk's placement with the group's
|
||||
// keys laid over it and its own inputs (the sealed values and boxes it holds among them), compares the result with what
|
||||
// the node runs, and hands the answers to be written into the node's own have key, `<node>.node`, under `candidates`.
|
||||
// It applies nothing and hands no composition: what runs is still composed from `placement` alone, by the Composer.
|
||||
//
|
||||
// **Event-driven, like the Composer.** It watches every candidate key, the trunk's placement, every node's record, the
|
||||
// indexer's modules, this node's config values, and this node's module haves (what it runs moved); a burst of changes
|
||||
// answers once. It runs whether or not the node is placed yet: a node that applies the controller's compositions answers
|
||||
// against what it runs from them. An answer is handed only when it changes, and a group's answer is dropped when its
|
||||
// keys go.
|
||||
type Shadow struct {
|
||||
c *Composer
|
||||
answer func(ctx context.Context, answers map[string]delivery.CandidateAnswer)
|
||||
|
||||
mu sync.Mutex
|
||||
candidates map[string]map[string]candidateKey // by group, by node
|
||||
answers map[string]delivery.CandidateAnswer
|
||||
}
|
||||
|
||||
// candidateKey is one candidate key as last read: its value and revision.
|
||||
type candidateKey struct {
|
||||
c delivery.PlacementCandidate
|
||||
rev uint64
|
||||
}
|
||||
|
||||
// NewShadow is the shadow composer of one node; answer writes the node's answers into its own have key.
|
||||
func NewShadow(env Env, answer func(ctx context.Context, answers map[string]delivery.CandidateAnswer)) *Shadow {
|
||||
return &Shadow{c: New(env), answer: answer, candidates: map[string]map[string]candidateKey{},
|
||||
answers: map[string]delivery.CandidateAnswer{}}
|
||||
}
|
||||
|
||||
// Run answers until ctx ends or an input's watch ends.
|
||||
func (s *Shadow) Run(ctx context.Context) error {
|
||||
c := s.c
|
||||
node := c.env.Node
|
||||
ins := []input{
|
||||
{delivery.PlacementCandidatesBucket, ">", nil},
|
||||
{delivery.PlacementBucket, ">", (*Composer).takePlacement},
|
||||
{delivery.HaveBucket, "*." + delivery.HaveNodeWord, nil},
|
||||
{delivery.ModulesBucket, ">", (*Composer).takeModule},
|
||||
{ResolvedBucket, node + ".>", (*Composer).takeValue},
|
||||
// What this node runs moved: an apply ends in its module's have.
|
||||
{delivery.HaveBucket, node + ".*", nil},
|
||||
}
|
||||
type tagged struct {
|
||||
in int
|
||||
u agent.Update
|
||||
ok bool
|
||||
}
|
||||
merged := make(chan tagged, 64)
|
||||
for i, in := range ins {
|
||||
ch, err := c.env.Watch(ctx, in.bucket, in.pattern)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the node answers placement candidates from %s, which cannot be watched: %w", in.bucket, err)
|
||||
}
|
||||
go func(i int, ch <-chan agent.Update) {
|
||||
for u := range ch {
|
||||
select {
|
||||
case merged <- tagged{in: i, u: u, ok: true}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
select {
|
||||
case merged <- tagged{in: i}:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
}(i, ch)
|
||||
}
|
||||
caught := make([]bool, len(ins))
|
||||
ready, dirty := false, false
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case t := <-merged:
|
||||
if !t.ok {
|
||||
return fmt.Errorf("the watch of %s ended; the node answers no placement candidate until its agent starts again", ins[t.in].bucket)
|
||||
}
|
||||
switch {
|
||||
case t.u.Caught:
|
||||
caught[t.in] = true
|
||||
case t.in == 0:
|
||||
dirty = s.takeCandidate(t.u) || dirty
|
||||
case t.in == 2:
|
||||
dirty = s.takeRecord(t.u) || dirty
|
||||
case t.in == 5:
|
||||
dirty = !isNodeKey(t.u.Key) || dirty
|
||||
default:
|
||||
ins[t.in].take(c, t.u)
|
||||
dirty = true
|
||||
}
|
||||
if len(merged) > 0 {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if !ready {
|
||||
if ready = all(caught); !ready {
|
||||
continue
|
||||
}
|
||||
dirty = true
|
||||
}
|
||||
if dirty {
|
||||
dirty = false
|
||||
s.answerAll(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isNodeKey(key string) bool {
|
||||
_, _, isNode, err := delivery.ParseHaveKey(key)
|
||||
return err == nil && isNode
|
||||
}
|
||||
|
||||
// takeCandidate takes one candidate key, and says whether it changed anything.
|
||||
func (s *Shadow) takeCandidate(u agent.Update) bool {
|
||||
group, node, err := delivery.ParsePlacementCandidateKey(u.Key)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if u.Deleted {
|
||||
if _, ok := s.candidates[group][node]; !ok {
|
||||
return false
|
||||
}
|
||||
delete(s.candidates[group], node)
|
||||
if len(s.candidates[group]) == 0 {
|
||||
delete(s.candidates, group)
|
||||
}
|
||||
return true
|
||||
}
|
||||
c, err := delivery.DecodePlacementCandidate(u.Value)
|
||||
if err != nil || c.Group != group || (c.Placement != nil && c.Placement.Node != node) {
|
||||
// Not one: read as gone.
|
||||
if _, ok := s.candidates[group][node]; ok {
|
||||
delete(s.candidates[group], node)
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
if s.candidates[group] == nil {
|
||||
s.candidates[group] = map[string]candidateKey{}
|
||||
}
|
||||
s.candidates[group][node] = candidateKey{c: c, rev: u.Revision}
|
||||
return true
|
||||
}
|
||||
|
||||
// takeRecord takes a node's record, and says whether it changed: the node's own key changes with each answer written,
|
||||
// which moves nothing composed.
|
||||
func (s *Shadow) takeRecord(u agent.Update) bool {
|
||||
node, _, isNode, err := delivery.ParseHaveKey(u.Key)
|
||||
if err != nil || !isNode {
|
||||
return false
|
||||
}
|
||||
c := s.c
|
||||
c.mu.Lock()
|
||||
before, had := c.records[node]
|
||||
c.mu.Unlock()
|
||||
c.takeRecord(u)
|
||||
c.mu.Lock()
|
||||
after, has := c.records[node]
|
||||
c.mu.Unlock()
|
||||
return had != has || (has && !delivery.SameNodeRecord(before, after))
|
||||
}
|
||||
|
||||
// answerAll answers every group and hands the answers when any changed.
|
||||
func (s *Shadow) answerAll(ctx context.Context) {
|
||||
c := s.c
|
||||
node := c.env.Node
|
||||
s.mu.Lock()
|
||||
groups := make([]string, 0, len(s.candidates))
|
||||
for g := range s.candidates {
|
||||
groups = append(groups, g)
|
||||
}
|
||||
s.mu.Unlock()
|
||||
sort.Strings(groups)
|
||||
next := map[string]delivery.CandidateAnswer{}
|
||||
for _, group := range groups {
|
||||
if a, ok := s.answerGroup(ctx, group); ok {
|
||||
next[group] = a
|
||||
}
|
||||
}
|
||||
s.mu.Lock()
|
||||
same := len(next) == len(s.answers)
|
||||
for g, a := range next {
|
||||
if old, ok := s.answers[g]; !ok || !sameAnswer(old, a) {
|
||||
same = false
|
||||
}
|
||||
}
|
||||
if !same {
|
||||
s.answers = next
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if !same && s.answer != nil {
|
||||
out := make(map[string]delivery.CandidateAnswer, len(next))
|
||||
for g, a := range next {
|
||||
out[g] = a
|
||||
}
|
||||
s.answer(ctx, out)
|
||||
c.env.Say(fmt.Sprintf("%s answered %d placement candidate group(s)", node, len(out)))
|
||||
}
|
||||
}
|
||||
|
||||
// answerGroup is the node's answer to one group, and false when the node has nothing to answer: neither the trunk nor
|
||||
// the group places it.
|
||||
func (s *Shadow) answerGroup(ctx context.Context, group string) (delivery.CandidateAnswer, bool) {
|
||||
c := s.c
|
||||
node := c.env.Node
|
||||
s.mu.Lock()
|
||||
keys := s.candidates[group]
|
||||
given := make(map[string]*delivery.Placement, len(keys))
|
||||
var head string
|
||||
for n, k := range keys {
|
||||
given[n] = k.c.Placement
|
||||
head = k.c.Pull.Head
|
||||
}
|
||||
// The revision answered is the node's own key's, 0 when the group has none for it (judged by the head alone).
|
||||
own, mine := keys[node]
|
||||
rev := own.rev
|
||||
s.mu.Unlock()
|
||||
|
||||
c.mu.Lock()
|
||||
_, placed := c.placements[node]
|
||||
modules := make(map[string]delivery.Module, len(c.modules))
|
||||
for k, v := range c.modules {
|
||||
modules[k] = v
|
||||
}
|
||||
c.mu.Unlock()
|
||||
if !placed && !mine {
|
||||
return delivery.CandidateAnswer{}, false
|
||||
}
|
||||
|
||||
var running []delivery.Composed
|
||||
if c.env.Running != nil {
|
||||
running = c.env.Running()
|
||||
}
|
||||
if p, ok := given[node]; ok && p == nil {
|
||||
// The head removes this node's placement file: everything it runs is removed.
|
||||
a := delivery.CandidateAnswer{Head: head, Revision: rev}
|
||||
for _, r := range running {
|
||||
if r.Module == "" {
|
||||
a.Removed = append(a.Removed, delivery.HaveNodeWord)
|
||||
} else {
|
||||
a.Removed = append(a.Removed, r.Module)
|
||||
}
|
||||
}
|
||||
sort.Strings(a.Removed)
|
||||
a.Verdict = delivery.VerdictOf(a)
|
||||
return a, true
|
||||
}
|
||||
|
||||
// Each module at the build it runs; one the candidate adds at the trunk's build.
|
||||
fps := map[string]string{}
|
||||
kept := map[string]string{}
|
||||
for _, r := range running {
|
||||
if r.Module != "" && r.FP != "" {
|
||||
fps[r.Module] = r.FP
|
||||
}
|
||||
}
|
||||
placement := given[node]
|
||||
if placement == nil {
|
||||
c.mu.Lock()
|
||||
p := c.placements[node]
|
||||
c.mu.Unlock()
|
||||
placement = &p
|
||||
}
|
||||
for _, m := range placement.Modules {
|
||||
if _, runs := fps[m.Module]; !runs {
|
||||
if mod, ok := modules[m.Module]; ok && mod.FP2 != "" {
|
||||
fps[m.Module] = mod.FP2
|
||||
}
|
||||
}
|
||||
}
|
||||
for module, fp := range fps {
|
||||
if mod, ok := modules[module]; ok && mod.FP2 == fp {
|
||||
continue
|
||||
}
|
||||
digest, err := c.builtManifest(ctx, module, fp)
|
||||
if err != nil {
|
||||
return delivery.CandidateAnswer{Head: head, Revision: rev, Verdict: delivery.CandidateRefused,
|
||||
Refused: fmt.Sprintf("%s runs at %s, whose manifest cannot be read: %v", module, fp, err)}, true
|
||||
}
|
||||
kept[module] = digest
|
||||
}
|
||||
// The boxes it runs, by what each module's have and its own key say after a restart (Held), are reused for the
|
||||
// modules it runs: a value sealed again is no difference.
|
||||
held := map[string]delivery.Want{agent.NodeModule: {}}
|
||||
for module := range fps {
|
||||
held[module] = delivery.Want{FP: fps[module]}
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.wants = held
|
||||
c.mu.Unlock()
|
||||
a := frombus.Answer(ctx, reader{c: c}, c.env.Mesh, frombus.Scope{Node: node, Kept: kept, Placements: given}, fps, running, head, rev)
|
||||
return a, true
|
||||
}
|
||||
|
||||
func sameAnswer(a, b delivery.CandidateAnswer) bool {
|
||||
x, errA := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"g": a}})
|
||||
y, errB := delivery.EncodeHaveNode(delivery.HaveNode{Candidates: map[string]delivery.CandidateAnswer{"g": b}})
|
||||
return errA == nil && errB == nil && string(x) == string(y)
|
||||
}
|
||||
Reference in New Issue
Block a user