diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go index 32069148..1d746b2a 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -12,6 +12,7 @@ import ( "path/filepath" "slices" "strings" + "sync" "testing" "time" @@ -30,6 +31,9 @@ import ( "git.novox.be/novox/mesh-sdk/go/provisions" "mesh-delivery-assigner/internal/judge" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/sdk/enginereads" + "mesh-delivery-assigner/internal/assign" ) @@ -92,7 +96,14 @@ func startBed(t *testing.T, leaveOut string) *bed { Subjects: []string{buscompose.LogSubject(delivery.BuilderModule, delivery.BuilderLog) + ".>"}, AllowDirect: true}); err != nil { t.Fatal(err) } - if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil { + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>", "mesh.seat.*.event.>"}}); err != nil { + t.Fatal(err) + } + // The assigner's durable consumer on EVENTS, as the grants part makes it from the manifest's consumes: a node-engine seat's + // event names the engine's machine last, so the filter is the event with every machine (mesh-grants buscompose + // machineEvent; the grants part's own test holds the derivation). + if _, err := js.CreateOrUpdateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-assigner", + FilterSubjects: []string{enginereads.SubjectPrefix + "*"}, AckPolicy: jetstream.AckExplicitPolicy}); err != nil { t.Fatal(err) } // The operator-channel seat's stream of conditions, which the assigner says its conditions on. @@ -490,6 +501,116 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) { if w := b.wantNow(t, "ace", "app", "the consumer's want once its provision is provided", func(w delivery.Want) bool { return w.FP == fpOf('3') }); w.FP != fpOf('3') { t.Errorf("app's want = %+v", w) } + + // ADR 0618 §1, §3, §4, ADR 0624 §2: ace's engine answers the node-engine seat's manifest-reads; a build written only in a + // form it does not state is held on ace, naming the form, and moves once the engine says manifest-reads-changed on its + // subject and states the form; no timer moves it. g14's engine answers nothing and has no record: read as the baseline. + stated := catalogue.ManifestReads() + var statedMu sync.Mutex + nc, err := nats.Connect(b.srv.ClientURL()) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + if _, err := nc.Subscribe("mesh.seat.node-engine.tool.manifest-reads.ace", func(m *nats.Msg) { + statedMu.Lock() + answer, _ := json.Marshal(nodeengine.ManifestReadsAnswer{V: 1, Node: "ace", ManifestReads: stated, StatedAt: time.Now().Format(time.RFC3339Nano)}) + statedMu.Unlock() + reply, _ := json.Marshal(map[string]any{"result": json.RawMessage(answer), "node": "ace"}) + _ = m.Respond(reply) + }); err != nil { + t.Fatal(err) + } + say := func(why string) { + body, _ := json.Marshal(nodeengine.ManifestReadsChangedEvent{V: 1, Node: "ace", ManifestReads: stated, At: time.Now().Format(time.RFC3339Nano), Why: why}) + if _, err := b.js.Publish(b.ctx, enginereads.SubjectPrefix+"ace", body); err != nil { + t.Fatal(err) + } + } + say(enginereads.ReadsStarted) + formsOld, formsNew := fpOf('5'), fpOf('6') + formsDigest := b.manifest(t, "forms", `{"module":"forms","version":"1"}`) + formsAt := func(fp string) { + mod, _ := delivery.EncodeModule(delivery.Module{FP2: fp, Manifest: formsDigest, Repository: "novox/forms", Commit: strings.Repeat("3", 40), Path: delivery.RootPath}) + b.putJSON(t, delivery.ModulesBucket, "forms", mod) + } + for _, fp := range []string{formsOld, formsNew} { + build := delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: formsDigest} + if fp == formsNew { + build.Needs = []string{"form:new"} + } + built, err := delivery.EncodeBuilderBuild(build) + if err != nil { + t.Fatal(err) + } + bkey, _ := delivery.BuilderBuildKey("forms", fp) + b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built) + gkey, _ := delivery.PartGrantKey("forms", fp) + grant, _ := delivery.EncodeGrant(delivery.Grant{}) + b.putJSON(t, delivery.PartGrantsBucket, gkey, grant) + } + runsFormsOld, _ := delivery.EncodeHave(delivery.Have{FP: formsOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) + b.putJSON(t, delivery.HaveBucket, "ace.forms", runsFormsOld) + b.place(t, "ace", "forms") + formsAt(formsOld) + b.wantNow(t, "ace", "forms", "the want at what ace runs, a build in the first form", func(w delivery.Want) bool { return w.FP == formsOld }) + formsAt(formsNew) + statement := func(node string, ok func(enginereads.Statement) bool) enginereads.Statement { + var last enginereads.Statement + for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if e, err := b.buckets[engineReadsBucket].Get(b.ctx, node); err == nil { + if last, err = enginereads.Decode(e.Value()); err == nil && ok(last) { + return last + } + } + } + t.Fatalf("%s's engine reads are %+v", node, last) + return last + } + statement("ace", func(s enginereads.Statement) bool { return s.From == enginereads.FromVerb && s.Answered }) + if s := statement("g14", func(s enginereads.Statement) bool { return !s.Answered }); s.From != enginereads.FromNone { + t.Errorf("g14, whose engine answers nothing and which has no record, reads %+v", s) + } + if w := b.wantNow(t, "ace", "forms", "the want held on ace, naming the form", func(w delivery.Want) bool { + return w.Newest == formsNew && strings.Contains(w.Words, "form:new") + }); w.FP != formsOld || w.Rule != delivery.RuleComposed { + t.Errorf("forms' want = %+v, want it held at what ace runs under rule 7", w) + } + statedMu.Lock() + stated = append(slices.Clone(stated), "form:new") + statedMu.Unlock() + say(enginereads.ReadsStarted) + b.wantNow(t, "ace", "forms", "the want moved once ace's engine said it reads the form", func(w delivery.Want) bool { return w.FP == formsNew }) + // assigner_engine_reads asks ace's engine again through the state: the assigner answers there with what it states now. + again, err := enginereads.AskAgainOf(bedState{b, engineReadsBucket}, "ace", "n1.ada", "a person asks", time.Now(), 15*time.Second) + if err != nil || again.From != enginereads.FromVerb || !slices.Contains(again.ManifestReads, "form:new") || again.AskAgain != nil { + t.Errorf("the ask again of ace answered %+v, %v", again, err) + } +} + +// bedState is one of the bed's buckets as the engine reads' tool reads it. +type bedState struct { + b *bed + bucket string +} + +func (s bedState) Keys() ([]string, error) { return s.b.buckets[s.bucket].Keys(s.b.ctx) } +func (s bedState) Get(key string) ([]byte, uint64, bool, error) { + e, err := s.b.buckets[s.bucket].Get(s.b.ctx, key) + if err != nil { + return nil, 0, false, nil + } + return e.Value(), e.Revision(), true, nil +} +func (s bedState) Put(key string, value []byte) (uint64, error) { return s.b.buckets[s.bucket].Put(s.b.ctx, key, value) } +func (s bedState) Delete(key string) error { return s.b.buckets[s.bucket].Delete(s.b.ctx, key) } +func (s bedState) Await(key string, after uint64, done func([]byte) bool, bound time.Duration) ([]byte, error) { + for deadline := time.Now().Add(bound); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if v, rev, found, _ := s.Get(key); found && rev > after && done(v) { + return v, nil + } + } + return nil, fmt.Errorf("no answer at %s within %s", key, bound) } // 1297/B9 (the assigner's half, ADR 0528 clause 6): started whole, the assigner holds a fingerprint whose health check diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go index ec894b3b..679199e8 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -28,6 +28,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/process" "git.novox.be/novox/mesh-sdk/go/provisions" + "git.novox.be/novox/mesh-delivery/sdk/enginereads" "git.novox.be/novox/mesh-delivery/sdk/graph" "mesh-delivery-assigner/internal/assign" @@ -75,7 +76,15 @@ var readFamilies = []string{delivery.ComposedRead} // (ADR 0567 §1), named after the assigner as every module state is. var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerDifferencesBucket, delivery.AssignerPutsBucket, delivery.AssignerWantsBucket, delivery.PlacementBucket, delivery.PlacementProposalsBucket, - delivery.PlacementCandidatesBucket, delivery.PlacementPulsesBucket, assign.FindingsBucket, assign.PoliciesBucket, delivery.SeatHoldersBucket} + delivery.PlacementCandidatesBucket, delivery.PlacementPulsesBucket, assign.FindingsBucket, assign.PoliciesBucket, delivery.SeatHoldersBucket, + engineReadsBucket} + +// engineReadsState is the assigner's state of what each node's engine states it reads (ADR 0618 §3), one key ``, as +// the verb manifest-reads last answered it or the fallback read it: what assigner_engine_reads lists and asks again. +const engineReadsState = "engine-reads" + +// engineReadsBucket is that state's bucket. +const engineReadsBucket = delivery.AssignerModule + "_" + engineReadsState // seatBuckets are the seat's states, new with it: until the controller has made them from the manifest and the bus grants // them, the assigner starts without them and binds each once the bus says it was created (issue 1790's kind). @@ -99,6 +108,7 @@ var keptStates = []announce.State{ {Name: health.FindingsState, Readable: true}, {Name: delivery.AssignerPoliciesState, Readable: true}, {Name: delivery.SeatHoldersState, Readable: true}, + {Name: engineReadsState, Readable: true}, } // buckets is every bucket the assigner opened, by name. @@ -156,6 +166,12 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any logf("the bucket %s cannot be opened (%v): no want waits for a verb it invokes until it is bound once the bus says it was made", name, err) continue } + if name == engineReadsBucket && err != nil { + // New with ADR 0618 §3: without it the statements the verb answers are kept in memory only, and + // assigner_engine_reads lists nothing. + logf("the bucket %s cannot be opened (%v): what each node's engine reads is kept in memory only until it starts again", name, err) + continue + } if name == delivery.PartGrantsBucket && err != nil { // The grants part's grants (ADR 0571 clause 9): without them no module counts as granted until it is bound. logf("the bucket %s cannot be opened (%v): no grant is read until it is bound once the bus says it was made", name, err) @@ -174,7 +190,7 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any func startsWithout(name string) bool { switch name { case delivery.SeatHoldersBucket, provisions.ProvisionsBucket, assign.PoliciesBucket, assign.FindingsBucket, - delivery.AssignerPutsBucket, delivery.SourceBucket, delivery.MergesBucket, graph.Bucket: + delivery.AssignerPutsBucket, delivery.SourceBucket, delivery.MergesBucket, graph.Bucket, engineReadsBucket: return true } return slices.Contains(seatBuckets, name) || name == delivery.PartGrantsBucket @@ -260,6 +276,7 @@ var watched = []string{ delivery.SourceBucket, delivery.MergesBucket, graph.Bucket, + engineReadsBucket, } // follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after @@ -275,6 +292,16 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, } errs := make(chan error, len(watched)+2) settled := make(chan struct{}, len(watched)+2) + // What each node's engine reads (ADR 0618 §3, §4): asked through the seat's verb, kept in the state engine-reads, read + // from the node's record where the engine does not answer (the fallback, said in the journal). + reads := &enginereads.Keeper{Fallback: w.recordReads, Changed: wake, Logf: logger.Printf} + if bus != nil { + reads.Call = bus.CallOn + } + if kv := kvs[engineReadsBucket]; kv != nil { + reads.Put = func(node string, v []byte) error { _, err := kv.Put(ctx, node, v); return err } + reads.Delete = func(node string) error { return kv.Delete(ctx, node) } + } watch := func(name string, opts ...jetstream.WatchOpt) error { kw, err := kvs[name].WatchAll(ctx, opts...) if err != nil { @@ -308,6 +335,10 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, if e.Operation() == jetstream.KeyValuePut { value = e.Value() } + if name == engineReadsBucket { + reads.Heard(e.Key(), value, value == nil) // it wakes a pass when a statement changed + continue + } w.putEntry(name, e.Key(), value, e.Revision(), e.Created()) wake() } @@ -405,7 +436,14 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber, case <-settled: } } - p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, findings: findings, said: map[string]bool{}, answeredAt: map[string]uint64{}} + p := &passer{kvs: kvs, w: w, logf: logger.Printf, say: say, findings: findings, said: map[string]bool{}, answeredAt: map[string]uint64{}, + reads: reads} + // The node-engine seat's manifest-reads-changed, for one node (ADR 0618 §4): that node is asked again, with no timer. + // Heard on the assigner's durable consumer on EVENTS, which the grants part makes from the manifest's consumes; where it + // is not there yet, a node is asked again only when it comes back online. + if js != nil { + hearReadsChanged(ctx, js, reads, logger.Printf) + } if bus != nil && kvs[delivery.AssignerPutsBucket] != nil { p.acts = newPutActs(bus, wake, logger.Printf) } @@ -470,3 +508,37 @@ func sayVia(ctx context.Context, js jetstream.JetStream) func(asks.Condition) er return err } } + +// hearReadsChanged hears the node-engine seat's event manifest-reads-changed on the assigner's durable consumer on EVENTS, +// and asks the node it names again. Any other event the consumer hands is acknowledged and left. +func hearReadsChanged(ctx context.Context, js jetstream.JetStream, reads *enginereads.Keeper, logf func(string, ...any)) { + account := process.Account{Node: thisNode, Module: delivery.AssignerModule} + cons, err := account.EventConsumer(ctx, js, process.EventsStream) + if err != nil { + logf("the consumer %s on %s does not exist yet, so %s is not heard until the assigner starts again (a node is asked again when it comes back online): %v", + account.EventConsumerName(), process.EventsStream, enginereads.Event, err) + return + } + cc, err := process.HearEvents(cons, func(m jetstream.Msg) { + if err := m.Ack(); err != nil { + logf("acknowledging %s: %v", m.Subject(), err) + } + node, ours, err := enginereads.NodeOf(m.Subject(), m.Data()) + switch { + case !ours: + case err != nil: + logf("%s holds what is not %s, and %s is asked again all the same: %v", m.Subject(), enginereads.Event, node, err) + reads.AskNode(node, enginereads.Event+" that does not read") + default: + reads.AskNode(node, enginereads.Event) + } + }, logf) + if err != nil { + logf("%s is not heard: %v", enginereads.Event, err) + return + } + go func() { + <-ctx.Done() + cc.Stop() + }() +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go index cc520b40..02b3dc23 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -15,6 +15,8 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/provisions" + "git.novox.be/novox/mesh-delivery/sdk/enginereads" + "mesh-delivery-assigner/internal/assign" ) @@ -68,6 +70,40 @@ type passer struct { // saidWaits is the last said of the seat's wait for its states (sayWaits). saidWaits string intended map[string]map[string]map[string]bool + // reads keeps what each node's engine states it reads, asked through the node-engine seat's verb (ADR 0618 §3); nil reads + // each node's record, as the rules did before the verb. online is, by node, whether the last pass saw it online. + reads *enginereads.Keeper + online map[string]bool +} + +// engineReads sets what the rules read of each node's engine (ADR 0618 §3, §4): the statement the verb manifest-reads +// answered, or its record where the verb was not answered (the fallback) or not asked yet. A node is asked when the +// assigner first hears of it, and again when it comes back online, since an engine built before the event says none. +func (p *passer) engineReads(n assign.Nodes) { + if p.reads == nil { + return + } + known := p.w.knownNodes() + first := p.online == nil + for _, node := range known { + switch { + case first: + p.reads.AskNode(node, "the assigner started") + case !p.reads.Known(node): + p.reads.AskNode(node, "first heard of") + case !first && n.Online[node] && !p.online[node]: + p.reads.AskNode(node, "back online") + } + } + p.online = maps.Clone(n.Online) + if p.online == nil { + p.online = map[string]bool{} + } + for node, reads := range p.reads.Reads(known) { + if p.reads.Known(node) { + n.Reads[node] = reads + } + } } func (p *passer) clock() time.Time { @@ -120,6 +156,7 @@ func (p *passer) wants(ctx context.Context) { } } nodes := p.w.nodes(manifests, assigned) + p.engineReads(nodes) nodesOf := map[string][]string{} for node, modules := range assigned { for _, m := range modules { @@ -162,7 +199,7 @@ func (p *passer) wants(ctx context.Context) { for name := range names { fp := trunk[name].FP2 name := name - m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Needs: trunk[name].Needs, Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Offered: offered(manifests), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], + m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Needs: trunk[name].Needs, Chosen: chosenOf(chosen, name), Provisions: provisionsOf(name), Answering: answering(manifests, nodes), Offered: offered(manifests), Nodes: nodesOf[name], Now: time.Now(), Puts: puts[name], FormNeeds: p.w.buildNeeds(name, fp), Self: func(node string) bool { return placed[node] }, Composed: func(node, fp string) (delivery.ComposedDigest, bool) { return p.w.composed(node, name, fp) // a placed node's own: the digest its own have names (ADR 0571 §6) diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go index ec5f1bc0..560572a4 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass_test.go @@ -14,9 +14,12 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/sdk/enginereads" "git.novox.be/novox/mesh-delivery/sdk/graph" "mesh-delivery-assigner/internal/assign" @@ -166,6 +169,72 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) { if !slices.Equal(n.Reads["ace"], []string{"word:module", "word:tool-runner"}) || len(n.Reads["g14"]) != 0 || len(n.Reads["novox"]) != 0 { t.Errorf("the nodes' reads are %v", n.Reads) } + // ADR 0618 §3: with the verb asked, a node's reads are what its engine answered; where the engine answers it does not + // serve the verb, its record is read once in its place (the fallback, issue 774); a node asked again when it comes + // back online, since an engine built before the event says none. + asked := map[string]int{} + call := func(node, key string, _ any) (json.RawMessage, error) { + asked[node]++ + if node == "g14" || key != enginereads.Invoke { + return nil, &busretry.Answered{Words: "the seat node-engine serves no verb manifest-reads"} + } + return json.Marshal(nodeengine.ManifestReadsAnswer{V: 1, Node: node, ManifestReads: []string{"form:new"}, StatedAt: time.Now().Format(time.RFC3339Nano)}) + } + p := &passer{w: w, reads: &enginereads.Keeper{Call: call, Fallback: w.recordReads, Sync: true}} + on, _ := delivery.EncodePresence(delivery.Presence{Online: false}) + w.put(delivery.PresenceBucket, "ace", on) + n = w.nodes(nil, nil) + p.engineReads(n) + if !slices.Equal(n.Reads["ace"], []string{"form:new"}) || len(n.Reads["g14"]) != 0 || asked["ace"] != 1 { + t.Errorf("with the verb the nodes' reads are %v (asked %v)", n.Reads, asked) + } + if s := p.reads.Statements(); s["ace"].From != enginereads.FromVerb || s["g14"].From != enginereads.FromRecord { + t.Errorf("the statements are %+v", s) + } + p.engineReads(w.nodes(nil, nil)) + on, _ = delivery.EncodePresence(delivery.Presence{Online: true}) + w.put(delivery.PresenceBucket, "ace", on) + p.engineReads(w.nodes(nil, nil)) + if asked["ace"] != 2 { + t.Errorf("ace, back online, was asked %d times, want 2", asked["ace"]) + } + // The contract carries no checks, so its reader judges what the engine says (mesh-contracts seats/nodeengine): an + // answer or an event that does not hold is no statement, and an event's node is its subject's last token. + now := time.Now().Format(time.RFC3339Nano) + for _, row := range []struct { + name string + raw string + stays bool + }{ + {"a whole answer", `{"v":1,"node":"ace","manifest-reads":["form:new"],"stated-at":"` + now + `"}`, true}, + {"an engine stating nothing", `{"v":1,"node":"ace","stated-at":"` + now + `"}`, true}, + {"another layout", `{"v":2,"node":"ace","stated-at":"` + now + `"}`, false}, + {"no node", `{"v":1,"node":"","stated-at":"` + now + `"}`, false}, + {"a node that is no token", `{"v":1,"node":"a.ce","stated-at":"` + now + `"}`, false}, + {"another node's answer", `{"v":1,"node":"g14","stated-at":"` + now + `"}`, false}, + {"an empty capability", `{"v":1,"node":"ace","manifest-reads":[""],"stated-at":"` + now + `"}`, false}, + {"a time that is no time", `{"v":1,"node":"ace","stated-at":"yesterday"}`, false}, + } { + s := enginereads.Ask(func(string, string, any) (json.RawMessage, error) { return json.RawMessage(row.raw), nil }, "ace", time.Now()) + if (s.From == enginereads.FromVerb) != row.stays || !s.Answered { + t.Errorf("answer %s: %+v", row.name, s) + } + } + for _, row := range []struct { + name, subject, raw string + ours, fails bool + }{ + {"the event", enginereads.SubjectPrefix + "ace", `{"v":1,"node":"ace","at":"` + now + `","why":"returned"}`, true, false}, + {"another reason", enginereads.SubjectPrefix + "ace", `{"v":1,"node":"ace","at":"` + now + `","why":"bored"}`, true, true}, + {"another node in the body", enginereads.SubjectPrefix + "ace", `{"v":1,"node":"g14","at":"` + now + `","why":"started"}`, true, true}, + {"a time that is no time", enginereads.SubjectPrefix + "ace", `{"v":1,"node":"ace","at":"x","why":"started"}`, true, true}, + {"another event", "mesh.seat.node-engine.event.done.ace", `{}`, false, false}, + } { + _, ours, err := enginereads.NodeOf(row.subject, []byte(row.raw)) + if ours != row.ours || (err != nil) != row.fails { + t.Errorf("event %s: ours %v, %v", row.name, ours, err) + } + } }) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go index 2ed4d101..ddf992b2 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go @@ -14,6 +14,8 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/delivery" + + "git.novox.be/novox/mesh-delivery/sdk/enginereads" ) // The assigner's tools (novox/hq ADR 0546), served through the tool runner from its bundle: a process never serves (ADR @@ -21,6 +23,9 @@ import ( // every step as the event put. They are the module's own tools; the seat mesh-placement's verbs are served beside them // (seat_tools.go, ADR 0567 §10). +// engineReadsTool lists what each node's engine reads, as the assigner uses it, and asks one node's engine again. +const engineReadsTool = "assigner_engine_reads" + // putsState is the assigner's state of puts, as the runtime names a module's own state. const putsState = "puts" @@ -358,6 +363,8 @@ func serveTools() error { }) wants.watched = stdio.NewWatchedStates() // the large states read whole from one watched copy (ADR 0580 clause 3) tools = append(tools, wants.tools()...) + // What each node's engine reads, as the assigner uses it (ADR 0618 §3): its own state engine-reads, asked again through it. + tools = append(tools, enginereads.Tool(engineReadsTool, "the assigner", enginereads.KeptState(stdio.State(engineReadsState)), false, time.Now)) return stdio.Serve("", tools) } diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go index f37724e7..0cd66950 100644 --- a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -3,6 +3,7 @@ package main import ( "context" "errors" + "maps" "slices" "sort" "sync" @@ -420,6 +421,42 @@ func (w *world) built(module, fp string) bool { return err == nil && b.State == delivery.BuildDone } +// buildNeeds are the reference forms the builder's done build of module at fp is written in (the build key's needs, ADR +// 0618 §1): nil when it is not done or names none. +func (w *world) buildNeeds(module, fp string) []string { + key, err := delivery.BuilderBuildKey(module, fp) + if err != nil { + return nil + } + b, err := delivery.DecodeBuilderBuild(w.value(delivery.BuilderBuildsBucket, key)) + if err != nil || b.State != delivery.BuildDone { + return nil + } + return b.Needs +} + +// recordReads is what node's record `.node` states its engine reads (ADR 0614 §4), and whether the assigner holds +// a record of it: the fallback where the engine does not answer the verb manifest-reads (enginereads.FromRecord). +func (w *world) recordReads(node string) ([]string, bool) { + r, ok := w.records()[node] + return r.ManifestReads, ok +} + +// knownNodes is every node the assigner hears of: placed, with a record, or with a presence. +func (w *world) knownNodes() []string { + set := map[string]bool{} + for node := range w.placedNodes() { + set[node] = true + } + for node := range w.records() { + set[node] = true + } + for node := range w.online() { + set[node] = true + } + return slices.Sorted(maps.Keys(set)) +} + // composed is the composition of module for node: a placed node composes its own, and its composition is the digest its own // have names (ADR 0571 §6); an unplaced node has none, so it gets no want (and the pass says so, unplacedFindings). The // controller's copy (mesh-controller_composed) is read no more (issue 774). diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod index 777c0c97..f1ce8f99 100644 --- a/modules/mesh-delivery-assigner/go.mod +++ b/modules/mesh-delivery-assigner/go.mod @@ -24,3 +24,4 @@ require ( require git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk + diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum index 7679a5dc..98dc72c4 100644 --- a/modules/mesh-delivery-assigner/go.sum +++ b/modules/mesh-delivery-assigner/go.sum @@ -1,5 +1,3 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go index 5057439e..f029f074 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -12,6 +12,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/provisions" + "git.novox.be/novox/mesh-delivery/sdk/forms" "git.novox.be/novox/mesh-delivery/sdk/graph" ) @@ -80,6 +81,10 @@ type Module struct { // Needs are the reader capabilities the manifest at FP needs (the indexer's modules value, catalogue.ManifestNeeds, ADR // 0614 §4): a node's want moves only while that node's engine states each of them. Nil: none. Needs []string + // FormNeeds are the reference forms the module's build at FP is written in (`form:`, the builder's build key's + // needs, ADR 0618 §1): a node's want is held only while that node's engine reads none of them (ADR 0624 §2). Nil: the + // build names none, and holds nothing. + FormNeeds []string // Self says whether node composes the module itself: the catalogue's placement names the node (ADR 0547 §2). It is the one // source of a want's Self: every want the rules write or keep is derived from it (selfDerived), so a want written Self while // the node was placed does not outlive the placement (issue 1126). Nil: no node does. @@ -179,8 +184,14 @@ func holds(m Module, node string, n Nodes, ordering bool) (bool, int, string) { // manifest needs that the engine does not state, read as the baseline when it states none. It is held under rule 7 (the // composition), since no rule of its own is numbered in this release, with words naming the capability; it moves once the // engine states it. The node-engine's own new build is held only by a word of its own manifest no engine states. +// +// A build's reference forms hold the want only when the engine reads none of them (ADR 0624 §2): every engine reads a +// first form, so a build written in one is never held by its forms; the words name the forms it reads none of. func (m Module) readerLacking(node string, n Nodes) (string, bool) { missing := catalogue.MissingManifestReads(m.Needs, n.Reads[node]) + if len(missing) == 0 && forms.Current.Held(m.FormNeeds, n.Reads[node]) { + missing = slices.Clone(m.FormNeeds) + } if len(missing) == 0 { return "", false } diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go index 5e61414d..a3ce7e58 100644 --- a/modules/mesh-delivery-assigner/internal/assign/wants_test.go +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -122,21 +122,33 @@ func TestEachRuleHoldsTheWantBack(t *testing.T) { needs []string reads map[string][]string held bool + // forms are the build's reference forms (ADR 0624 §2), and names what the hold's words name. + forms []string + names string }{ - {"a word the baseline lacks, on engines stating nothing", "mail", []string{"word:module", "word:tool-runner"}, nil, true}, + {"a word the baseline lacks, on engines stating nothing", "mail", []string{"word:module", "word:tool-runner"}, nil, true, nil, "word:tool-runner"}, {"the same, once both engines state it", "mail", []string{"word:module", "word:tool-runner"}, - map[string][]string{"ace": catalogue.ManifestReads(), "g14": catalogue.ManifestReads()}, false}, - {"the baseline's words on engines stating nothing", "mail", []string{"word:claims.serves", "seat:mesh-catalog", "form:config"}, nil, false}, - {"the node-engine's new build, whose contract has more words than its manifest uses", "mesh-host", hostNeeds, nil, false}, + map[string][]string{"ace": catalogue.ManifestReads(), "g14": catalogue.ManifestReads()}, false, nil, ""}, + {"the baseline's words on engines stating nothing", "mail", []string{"word:claims.serves", "seat:mesh-catalog", "form:config"}, nil, false, nil, ""}, + {"the node-engine's new build, whose contract has more words than its manifest uses", "mesh-host", hostNeeds, nil, false, nil, ""}, + // ADR 0624 §2, with form:new a form no engine reads unless it states it and artifact-store the first form. + {"g14 back stating {old}, a build carrying {old,new}: moves", "mail", nil, map[string][]string{"g14": {"word:module"}}, false, + []string{"form:artifact-store", "form:new"}, ""}, + {"an engine stating nothing, a build carrying the baseline's form: moves", "mail", nil, nil, false, []string{"form:artifact-store"}, ""}, + {"a build carrying {new} only: held, naming the form", "mail", nil, map[string][]string{"g14": {"word:module"}}, true, + []string{"form:new"}, "form:new"}, + {"the same, once g14's engine states it", "mail", nil, map[string][]string{"g14": {"form:new"}}, false, []string{"form:new"}, ""}, + {"a manifest needing an unstated word, its build in the first form: held, naming the word", "mail", []string{"word:tool-runner"}, nil, true, + []string{"form:artifact-store"}, "word:tool-runner"}, } { m, n := ready() - m.Name, m.Needs, n.Reads = c.name, c.needs, c.reads + m.Name, m.Needs, n.Reads, m.FormNeeds = c.name, c.needs, c.reads, c.forms n.Haves["ace"][c.name], n.Haves["g14"][c.name] = n.Haves["ace"]["mail"], n.Haves["g14"]["mail"] out := Wants(m, n, wantedOld(), nil) if moved := out.Wants["g14"].FP == fNew; moved == c.held { t.Errorf("%s: g14's want %+v, held by %+v", c.why, out.Wants["g14"], out.Rules["g14"]) } - if r := out.Rules["g14"]; c.held && (r.Rule != delivery.RuleComposed || !strings.Contains(r.Words, "word:tool-runner") || + if r := out.Rules["g14"]; c.held && (r.Rule != delivery.RuleComposed || !strings.Contains(r.Words, c.names) || !strings.Contains(r.Words, "node-engine of g14")) { t.Errorf("%s: held by %+v, want rule 7 naming the capability", c.why, r) } diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go index 087b34cd..0fdfc72d 100644 --- a/modules/mesh-delivery-assigner/manifest_test.go +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -47,7 +47,8 @@ func TestManifest(t *testing.T) { states[s.Name] = s.History } if want := map[string]int{"asks": 5, "differences": 64, "puts": 20, "wants": 20, - "placement": 64, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5}; !reflect.DeepEqual(states, want) { + "placement": 64, "proposals": 5, "candidates": 5, "pulses": 0, "findings": 5, "policies": 20, "seat-holders": 5, + "engine-reads": 1}; !reflect.DeepEqual(states, want) { t.Errorf("state = %v, want %v", states, want) } if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 { @@ -78,8 +79,9 @@ func TestManifest(t *testing.T) { if seat, _ := catalogue.SeatNamed(catalogue.PlacementSeat); catalogue.CanHold(m, seat) != nil { t.Errorf("the assigner cannot hold the seat: %v", catalogue.CanHold(m, seat)) } - if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts"}) { - t.Errorf("the assigner serves tools %v, want its three put tools", m.Tools) + // ADR 0618 §3: and assigner_engine_reads, which lists and asks again what each node's engine reads. + if !slices.Equal(m.Tools, []string{"assigner_put", "assigner_release", "assigner_puts", "assigner_engine_reads"}) { + t.Errorf("the assigner serves tools %v, want its three put tools and assigner_engine_reads", m.Tools) } if len(m.Streams) != 1 || m.Streams[0].Name != delivery.PlacementStream || m.Streams[0].MaxAgeDays != delivery.PlacementStreamDays || !slices.Equal(slices.Sorted(slices.Values(m.Streams[0].Events)), slices.Sorted(slices.Values(delivery.PlacementEvents))) { @@ -87,9 +89,14 @@ func TestManifest(t *testing.T) { } if want := []string{ "seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", // ADR 0565 §9, §11: the seat alone - "seat:node-backup.backed-up", "seat:node-backup.now"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) { + "seat:node-backup.backed-up", "seat:node-backup.now", + "seat:node-engine.manifest-reads"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) { // ADR 0618 §3 t.Errorf("invokes = %v, want %v", m.Invokes, want) } + // ADR 0618 §4: it hears each node's engine say its reads changed. + if !slices.Equal(m.Consumes, []string{"node-engine.manifest-reads-changed"}) { + t.Errorf("consumes = %v", m.Consumes) + } if !slices.Equal(m.Emits, []string{"put"}) { t.Errorf("emits = %v, want put", m.Emits) } diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json index 7f94deaa..392718f2 100644 --- a/modules/mesh-delivery-assigner/module.json +++ b/modules/mesh-delivery-assigner/module.json @@ -45,7 +45,11 @@ "seat:mesh-hand-acts.hand-acts", "seat:mesh-hand-acts.warranted", "seat:node-backup.backed-up", - "seat:node-backup.now" + "seat:node-backup.now", + "seat:node-engine.manifest-reads" + ], + "consumes": [ + "node-engine.manifest-reads-changed" ], "emits": [ "put" @@ -53,7 +57,8 @@ "tools": [ "assigner_put", "assigner_release", - "assigner_puts" + "assigner_puts", + "assigner_engine_reads" ], "state": [ { "name": "placement", "history": 64 }, @@ -66,7 +71,8 @@ { "name": "wants", "history": 20 }, { "name": "findings", "history": 5 }, { "name": "policies", "history": 20 }, - { "name": "seat-holders", "history": 5 } + { "name": "seat-holders", "history": 5 }, + { "name": "engine-reads", "history": 1 } ], "streams": [ { diff --git a/modules/mesh-delivery-builder/buildrun/standing.go b/modules/mesh-delivery-builder/buildrun/standing.go index c0b92e96..fdaf1da6 100644 --- a/modules/mesh-delivery-builder/buildrun/standing.go +++ b/modules/mesh-delivery-builder/buildrun/standing.go @@ -6,6 +6,7 @@ import ( "fmt" "maps" "slices" + "sort" "sync" "time" @@ -42,6 +43,9 @@ type ToolchainGate struct { known bool // the placement is settled raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it left map[string]string // / -> why it was said left out, so each reason is said once + // OnPlaced is told, outside the gate's lock, once the placement's history is read and after each revision of it + // FollowPlacement hears from then on (the counted nodes, Nodes, may have changed). Nil tells nobody. + OnPlaced func() } // NewToolchainGate is a gate over the toolchains of modules, passing what stands on to forward and saying with logf what @@ -90,6 +94,19 @@ func (g *ToolchainGate) Placement(key string, value []byte, deleted bool, at tim return g.again(node, changed) } +// Nodes are the nodes the placement places any module on, sorted: the nodes the mesh counts (a node leaves the count +// when the placement no longer names it; ADR 0624 §1). None until the placement is read. +func (g *ToolchainGate) Nodes() []string { + var out []string + for node, modules := range g.placed.ByNode() { + if len(modules) > 0 { + out = append(out, node) + } + } + sort.Strings(out) + return out +} + // Settled says the placement's history is read: from now on only the reports of placed nodes pass, once a node's // placement is held. It answers whether what it passed on changed. func (g *ToolchainGate) Settled() bool { @@ -203,6 +220,9 @@ func FollowPlacement(ctx context.Context, js process.BucketOpener, gate *Toolcha if gate.Settled() { changed(ctx) } + if gate.OnPlaced != nil { + gate.OnPlaced() + } go func() { defer func() { _ = w.Stop() }() for { @@ -218,6 +238,9 @@ func FollowPlacement(ctx context.Context, js process.BucketOpener, gate *Toolcha if e != nil && put(e) { changed(ctx) } + if e != nil && gate.OnPlaced != nil { + gate.OnPlaced() + } } } }() diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go index 9f4e12e4..cba63ec1 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder-tools/main.go @@ -18,10 +18,15 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/sdk/enginereads" + "mesh-delivery-builder/internal/build" "mesh-delivery-builder/internal/versions" ) +// EngineReadsTool lists, asks again and drops what each counted node's engine reads, as the builder keeps it. +const EngineReadsTool = "builder_engine_reads" + // queueAsker writes a queues value and waits for the builder's answer to it. type queueAsker func(key string, value []byte) (json.RawMessage, error) @@ -373,7 +378,11 @@ func main() { } return e.Value, true, nil }} - if err := stdio.Serve("", Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(delivery.BuilderLog).Since, time.Now, vr, tags, tr, fr)); err != nil { + tools := Tools(r, put, askQueue(stdio.State(build.QueuesState), 25*time.Second), buildAgain, stdio.Log(delivery.BuilderLog).Since, time.Now, vr, tags, tr, fr) + // What each counted node's engine reads, as the builder keeps it (ADR 0618 §2, ADR 0624 §1): listed, one node asked + // again, or one node's key dropped. + tools = append(tools, enginereads.Tool(EngineReadsTool, "the builder", enginereads.KeptState(stdio.State("engine-reads")), true, time.Now)) + if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-builder-tools] %v\n", err) os.Exit(1) } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index 96c9d54d..9cbaf12e 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -439,6 +439,10 @@ func TestBuilderMakesADoneKeyWithTheDigests(t *testing.T) { // 1607/AC7 if v.Toolchains["go"].Version != "go1.27.1" || v.Toolchains["typescript"].Version != "tsc5" { t.Errorf("toolchains = %+v", v.Toolchains) } + // ADR 0618 §1: the done key names the reference forms it is written in. + if len(v.Needs) != 1 || v.Needs[0] != "form:artifact-store" { + t.Errorf("needs = %v, want the first form", v.Needs) + } // 1955/L7 (ADR 0571 §1): the build's start and end are in the module log build-log under the module's key, read as a // reader with the log's read reads them, the end naming the manifest and what was made. ended := b.logged(t, "netcheck") diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index c817114a..0c8b865d 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -159,7 +159,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // revision (the gate's source state). The source's revisions are watched with their history (a direct get of one revision // is not what a read grants): a revision heard after the candidate that names it builds that candidate then. Where it // cannot be read the trunk's keys are built and the candidates wait (issue 1945). - errs := make(chan error, 8) + errs := make(chan error, 9) trunks := &trunkSource{ctx: ctx, js: js, heads: &sourceHeads{}, moved: func(ctx context.Context) { // A source revision heard is the forge's merge: it wakes a floor whose bridge_tags was silent (design 51 *The floor*). (&floorWake{b: b, waits: floorWaits, logf: logger.Printf}).wake(ctx, versions.WakeForge, "", "") @@ -204,6 +204,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // placement is settled first, so the toolchains settle through the gate already knowing it. gate := buildrun.NewToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf) watch.put = gate.Toolchain + // What each counted node's engine reads (ADR 0618 §2, ADR 0624 §1): asked through the node-engine seat's verb as the + // builder's own account, for every node the placement names, and kept in the state engine-reads once it is bound. + reads := newEngineReads(calls.CallOn, gate, logger.Printf) + gate.OnPlaced = reads.placed + b.Forms = reads.forms placementErrs, err := buildrun.FollowPlacement(ctx, js, gate, b.NodesChanged, logger.Printf) if err != nil { return err @@ -241,7 +246,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // And the four registry seats' published, refused and removed (DLV-B4), which move a numbered version. seats := seatHearer{n: numberer, b: b, logf: logger.Printf} cc, err := process.HearEvents(cons, func(m jetstream.Msg) { - if floor.heard(ctx, m) || seats.heard(ctx, m) { + if reads.heard(m, logger.Printf) || floor.heard(ctx, m) || seats.heard(ctx, m) { // reads: the node-engine seat's manifest-reads-changed (ADR 0618 §4) return } heardGone(ctx, b, m, logger.Printf) @@ -341,6 +346,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } defer cc.Stop() } + // The state engine-reads, new with ADR 0618 (issue 1790's kind): until it is on the bus each statement is kept in memory + // only, and builder_engine_reads answers nothing. + process.AwaitBucket(ctx, js, conn, engineReadsBucket, "what each node's engine reads is kept in memory only", logger.Printf).Ready(func(kv jetstream.KeyValue) { + reads.bind(ctx, kv, errs, logger.Printf) + }) logger.Printf("started on %s", cred.Node) go func() { errs <- follow(ctx, w, each) }() select { diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go new file mode 100644 index 00000000..42e844f1 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/reads.go @@ -0,0 +1,105 @@ +package main + +import ( + "context" + "slices" + "sync" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/process" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-delivery/sdk/enginereads" +) + +// EngineReadsState is the builder's state of what each counted node's engine states it reads (novox/hq ADR 0618 §2, ADR +// 0624 §1): one key ``, written from the node-engine seat's verb manifest-reads at the builder's start, on the +// seat's event manifest-reads-changed for that node, and when a person asks again (builder_engine_reads); a node that +// is away keeps its last statement, and its key is removed when the placement no longer names the node. A counted node +// with no key reads as stating the baseline: its first forms. No node's record is read in its place. +const EngineReadsState = "engine-reads" + +// engineReadsBucket is that state's bucket. +const engineReadsBucket = delivery.BuilderModule + "_" + EngineReadsState + +// engineReads keeps the statements and the counted nodes for the builder's forms (build.Builder.Forms). +type engineReads struct { + keeper *enginereads.Keeper + gate *buildrun.ToolchainGate + mu sync.Mutex + counted []string +} + +// newEngineReads asks through call (process.Conn.CallOn) and keeps each answer in the state once it is bound. +func newEngineReads(call enginereads.Caller, gate *buildrun.ToolchainGate, logf func(string, ...any)) *engineReads { + return &engineReads{gate: gate, keeper: &enginereads.Keeper{Call: call, Logf: logf}} +} + +// bind keeps the statements in kv and hears it: every node's last statement at the start, a person's ask again and drop +// later. Bound once the state is on the bus (process.AwaitBucket). +func (r *engineReads) bind(ctx context.Context, kv jetstream.KeyValue, errs chan<- error, logf func(string, ...any)) { + r.keeper.Put = func(node string, v []byte) error { _, err := kv.Put(ctx, node, v); return err } + r.keeper.Delete = func(node string) error { return kv.Delete(ctx, node) } + each := func(e jetstream.KeyValueEntry) { + r.keeper.Heard(e.Key(), e.Value(), e.Operation() != jetstream.KeyValuePut) + } + w, err := settle(ctx, kv, each) + if err != nil { + process.NotWatched(logf, engineReadsBucket, err, "builder_engine_reads is not answered until the builder starts again") + return + } + go func() { errs <- follow(ctx, w, each) }() + // Every counted node asked again, so what was asked before the state was bound is kept in it too. + r.mu.Lock() + nodes := slices.Clone(r.counted) + r.mu.Unlock() + for _, node := range nodes { + r.keeper.AskNode(node, "the state engine-reads is bound") + } +} + +// placed takes the placement in: a node it names newly is asked, one it no longer names is forgotten, key and all. The +// first call asks every counted node (the builder's start). +func (r *engineReads) placed() { + nodes := r.gate.Nodes() + r.mu.Lock() + before, first := r.counted, r.counted == nil + r.counted = nodes + r.mu.Unlock() + for _, node := range nodes { + if first || !slices.Contains(before, node) { + r.keeper.AskNode(node, "counted: the placement names it") + } + } + for _, node := range before { + if !slices.Contains(nodes, node) { + r.keeper.Forget(node) + } + } +} + +// heard takes an event of the builder's consumer on EVENTS: manifest-reads-changed asks its node again, and answers true. +func (r *engineReads) heard(m jetstream.Msg, logf func(string, ...any)) bool { + node, ours, err := enginereads.NodeOf(m.Subject(), m.Data()) + if !ours { + return false + } + if ackErr := m.Ack(); ackErr != nil { + logf("acknowledging %s: %v", m.Subject(), ackErr) + } + if err != nil { + logf("%s holds what is not %s, and %s is asked again all the same: %v", m.Subject(), enginereads.Event, node, err) + } + r.keeper.AskNode(node, enginereads.Event) + return true +} + +// forms is build.Builder.Forms: the counted nodes' statements, and those of the nodes whose engine answered. +func (r *engineReads) forms() (present, counted map[string][]string) { + r.mu.Lock() + nodes := slices.Clone(r.counted) + r.mu.Unlock() + return r.keeper.Present(nodes), r.keeper.Reads(nodes) +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go index 21f6ac58..b5c1dce9 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/refused_test.go @@ -3,8 +3,11 @@ package main import ( "bytes" "context" + "encoding/json" "errors" "log" + "maps" + "slices" "strings" "testing" "time" @@ -16,6 +19,8 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/sdk/enginereads" "git.novox.be/novox/mesh-sdk/go/delivery" "mesh-delivery-builder/internal/build" @@ -54,6 +59,100 @@ func TestARefusedBucketIsSaidAndThePartGoesOn(t *testing.T) { if src.opened { t.Error("a refused source counts as opened, so it is never opened again") } + t.Run("ADR 0624: the counted nodes' engine reads", theCountedNodesEngineReads) +} + +// theCountedNodesEngineReads: the builder asks the node-engine seat's verb of every node the placement names; a node away +// and never asked has no statement and reads as the baseline's first forms, with no node record read in its place; an +// away node keeps its last statement; a node the placement no longer names leaves the count, its key removed; and +// builder_engine_reads drops one node's key. +func theCountedNodesEngineReads(t *testing.T) { + gate := buildrun.NewToolchainGate(nil, func(string, string, []byte, bool) bool { return false }, t.Logf) + place := func(nodes ...string) { + for _, node := range []string{"n1", "n2"} { + pl := delivery.Placement{Node: node} + if slices.Contains(nodes, node) { + pl.Modules = []delivery.PlacedModule{{Module: "web"}} + } + values, err := delivery.PlacementValues(pl) + if err != nil { + t.Fatal(err) + } + for key, raw := range values { + gate.Placement(key, raw, false, time.Now()) + } + if !slices.Contains(nodes, node) { + key, _ := delivery.PlacementModuleKey(node, "web") + gate.Placement(key, nil, true, time.Now()) + } + } + } + away := map[string]bool{"n2": true} + asked := map[string]int{} + r := newEngineReads(func(node, key string, _ any) (json.RawMessage, error) { + asked[node]++ + if away[node] { + return nil, nats.ErrNoResponders + } + return json.Marshal(nodeengine.ManifestReadsAnswer{V: 1, Node: node, ManifestReads: []string{"form:new"}, StatedAt: time.Now().Format(time.RFC3339Nano)}) + }, gate, t.Logf) + r.keeper.Sync = true + kept := map[string][]byte{} + r.keeper.Put = func(node string, v []byte) error { kept[node] = v; return nil } + r.keeper.Delete = func(node string) error { delete(kept, node); return nil } + place("n1", "n2") + r.placed() + present, counted := r.forms() + if !slices.Equal(present["n1"], []string{"form:new"}) || len(present) != 1 || len(counted) != 2 || counted["n2"] != nil { + t.Errorf("away and never asked: present %v, counted %v", present, counted) + } + if s := r.keeper.Statements()["n2"]; s.From != enginereads.FromNone || s.Answered || asked["n2"] != 1 { + t.Errorf("n2, away and never asked, is %+v (asked %d)", s, asked["n2"]) + } + // n2 back, stating {word:module}, then away again: it keeps that statement. + away["n2"] = false + r.keeper.Heard("n2", mustStatement(t, enginereads.Statement{Node: "n2", From: enginereads.FromVerb, Answered: true, + AskAgain: &enginereads.Again{By: "n1.ada", Why: "back"}}), false) + away["n2"] = true + r.keeper.AskNode("n2", "its event") + if _, counted = r.forms(); !slices.Equal(counted["n2"], []string{"form:new"}) || r.keeper.Statements()["n2"].Answered { + t.Errorf("away again, n2 reads %v (%+v)", counted["n2"], r.keeper.Statements()["n2"]) + } + // n2 retired: it leaves the placement, and its key goes. + place("n1") + r.placed() + if _, counted = r.forms(); len(counted) != 1 || kept["n2"] != nil { + t.Errorf("retired n2: counted %v, its key %s", counted, kept["n2"]) + } + // builder_engine_reads drops one key. + st := &memReads{v: map[string][]byte{"n1": kept["n1"]}} + tool := enginereads.Tool("builder_engine_reads", "the builder", st, true, time.Now) + if _, err := tool.Run(map[string]any{"node": "n1", "drop": "true", "why": "a test"}); err != nil || st.v["n1"] != nil { + t.Errorf("drop: %v, the key %s", err, st.v["n1"]) + } +} + +func mustStatement(t *testing.T, s enginereads.Statement) []byte { + t.Helper() + raw, err := enginereads.Encode(s) + if err != nil { + t.Fatal(err) + } + return raw +} + +// memReads is the builder's state of engine reads in memory, as the tool reads it. +type memReads struct{ v map[string][]byte } + +func (m *memReads) Keys() ([]string, error) { return slices.Collect(maps.Keys(m.v)), nil } +func (m *memReads) Get(key string) ([]byte, uint64, bool, error) { + v, ok := m.v[key] + return v, 1, ok, nil +} +func (m *memReads) Put(key string, value []byte) (uint64, error) { m.v[key] = value; return 2, nil } +func (m *memReads) Delete(key string) error { delete(m.v, key); return nil } +func (m *memReads) Await(string, uint64, func([]byte) bool, time.Duration) ([]byte, error) { + return nil, errors.New("no part answers here") } // refusingBus is a bus that refuses: every open (refuseOpen), or every watch of a bucket it opens, with err. diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index ab72384a..3204c90a 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -26,3 +26,8 @@ require ( ) replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun => ./buildrun + + +require git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 + +replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index 7ff42872..4fbe62eb 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -1,7 +1,5 @@ git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a h1:oxgksCiq272/afXbUION/dxtAS5MPGnmyq6+AZQxmXg= git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep v0.0.0-20261102193506-d221c633799a/go.mod h1:kpsX4Pgss5EeuBvel1zWa2ogj7MkR3jZso6oa/J0YUQ= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 49c4a744..0b5aad8d 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -14,6 +14,7 @@ import ( "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-delivery/sdk/forms" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -84,6 +85,11 @@ type Builder struct { // Numbers numbers each trunk build at its claim and records what it was built with (design 51 *The versions*); nil // numbers nothing. Numbers Numbers + // Forms answers what each node's engine states it reads (ADR 0618 §2, ADR 0624 §1): present, the nodes whose engine + // answered when last asked, and counted, every node the placement names, present or away, a node with no statement + // stating nothing (the baseline's first forms). A done key names each artifact in the forms these keep, the oldest in + // its single reference, and its needs. Nil: no node states anything, and the first forms are kept. + Forms func() (present, counted map[string][]string) once sync.Once sem chan struct{} @@ -623,6 +629,10 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p return failed("%s", keyError(errs)) } var artifacts []delivery.BuiltArtifact + var present, counted map[string][]string + if b.Forms != nil { + present, counted = b.Forms() + } keptModule := map[string]bool{} for _, bm := range kept { keptModule[bm] = true @@ -651,6 +661,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p return failed("%s", err) } built := delivery.BuiltArtifact{Name: t.Artifact, Kind: t.Kind, Digest: d} + var answered []string for _, l := range lanes { if l.module != t.Module { continue @@ -662,10 +673,11 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p return failed("%s%s on %s: the build answered no reference, so its push to the artifact store was not answered", delivery.NotStored, t.Artifact, l.node) } if r != "" { - built.Reference = r + answered = append(answered, r) } } - artifacts = append(artifacts, built) + // The forms a counted node's engine reads are kept, the oldest in the single reference (ADR 0624 §1, §3). + artifacts = append(artifacts, forms.Current.Choose(built, answered, present, counted)) } var steps []string for _, l := range lanes { @@ -687,7 +699,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p used = nil } done := delivery.BuilderBuild{State: delivery.BuildDone, By: b.Claimer.By, Manifest: m.Manifest, Step: step, Artifacts: artifacts, Toolchains: used, - Imports: importsOf(lanes)} + Imports: importsOf(lanes), Needs: forms.Current.Needs(artifacts)} // the forms it is written in (ADR 0618 §1) if _, err := delivery.EncodeBuilderBuild(done); err != nil { // A reference the build module answered that does not name the artifact and its digest is no reference. return failed("%s%v", delivery.NotStored, err) @@ -782,7 +794,7 @@ func (n *askNotes) note(ctx context.Context, language, id, step string) error { n.mu.Lock() defer n.mu.Unlock() v := n.cl.Build - v.State, v.By, v.Step, v.Error, v.Artifacts = delivery.BuildBuilding, n.b.Claimer.By, step, "", nil + v.State, v.By, v.Step, v.Error, v.Artifacts, v.Needs = delivery.BuildBuilding, n.b.Claimer.By, step, "", nil, nil v.Asks = maps.Clone(v.Asks) if v.Asks == nil { v.Asks = map[string]string{} diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 54a54568..509c093f 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "reflect" + "slices" "strconv" "strings" "sync" @@ -14,6 +15,7 @@ import ( "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-delivery/sdk/forms" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -224,6 +226,7 @@ var ( func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7, ADR 0478 clauses 1, 4 t.Run("the imports a build answers reach the done key and the log", importsABuildAnswersReachTheDoneKeyAndTheLog) + t.Run("ADR 0624: a done key keeps each form a counted node reads", aDoneKeyKeepsEachFormACountedNodeReads) r := newRig(t, goBundle, archiveA, tsBundle) // ADR 0584: a third node agrees with novox and ace's typescript is the odd one; the build is done and its step says ace sits out. r.tools.Put(buildrun.NodeBuildModule, "g14.typescript", tcValue(t, "typescript", "tsc5"), false) @@ -280,6 +283,11 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7 if v.Artifacts[0].Kind != "archive" || v.Artifacts[1].Kind != "bundle" { t.Errorf("kinds = %+v", v.Artifacts) } + // ADR 0618 §1: the done key names the forms its references are written in, here the first form alone, and keeps no list + // beside the single reference while no counted node needs another form. + if !slices.Equal(v.Needs, []string{"form:artifact-store"}) || v.Artifacts[0].References != nil { + t.Errorf("needs = %v, references = %v", v.Needs, v.Artifacts[0].References) + } if got := v.Toolchains["go"]; got.Version != "go1.27.1" || got.Recipe != "1" { t.Errorf("toolchains = %+v, want go and typescript named", v.Toolchains) } @@ -1270,3 +1278,71 @@ type silentWatch struct{} func (silentWatch) WatchAsk(context.Context, string, string) (<-chan delivery.Step, error) { return make(chan delivery.Step), nil } + +// aDoneKeyKeepsEachFormACountedNodeReads holds the builder's rows of ADR 0624's check: every build keeps each form a +// counted node's engine reads, present or away, a node stating nothing reading the baseline's first form; the oldest kept +// form is the single reference; a retired node leaves the count. old is the first form, new a fixture second form read +// only by an engine stating form:new. +func aDoneKeyKeepsEachFormACountedNodeReads(t *testing.T) { + fs := forms.ArtifactForms{ + {Name: "old", First: true, Kinds: []string{"bundle"}, Is: func(r string) bool { return strings.HasPrefix(r, "artifact-store://") }}, + {Name: "new", Kinds: []string{"bundle"}, Is: func(r string) bool { return strings.HasPrefix(r, "10.0.0.1:5000/") }}, + } + both, nothing, onlyNew := []string{"form:new", "word:module"}, []string(nil), []string{"form:new"} + oldRef, newRef := "artifact-store://m/code/blobs/"+digest, "10.0.0.1:5000/m/code@"+digest + for _, row := range []struct { + name string + present, counted map[string][]string + kept []string + reference string + references []string + }{ + {"A present {old,new}, B away {old}: both forms, the old in the single reference", + map[string][]string{"a": both}, map[string][]string{"a": both, "b": {"word:module"}}, []string{"old", "new"}, oldRef, []string{oldRef, newRef}}, + {"A present {new}, B away stating nothing: the baseline's form stays", + map[string][]string{"a": onlyNew}, map[string][]string{"a": onlyNew, "b": nothing}, []string{"old", "new"}, oldRef, []string{oldRef, newRef}}, + {"A present {old,new}, B away and never asked (no key): the baseline's form stays", + map[string][]string{"a": both}, map[string][]string{"a": both, "b": nil}, []string{"old", "new"}, oldRef, []string{oldRef, newRef}}, + {"B retired: the new form only", map[string][]string{"a": both}, map[string][]string{"a": both}, []string{"new"}, newRef, nil}, + {"B present stating nothing: the new form is not written", + map[string][]string{"a": both, "b": nothing}, map[string][]string{"a": both, "b": nothing}, []string{"old"}, oldRef, nil}, + {"no node present: the first form", nil, map[string][]string{"a": both}, []string{"old"}, oldRef, nil}, + } { + if got := fs.Kept("bundle", row.present, row.counted); !slices.Equal(got, row.kept) { + t.Errorf("%s: kept %v, want %v", row.name, got, row.kept) + } + a := fs.Choose(delivery.BuiltArtifact{Name: "code", Kind: "bundle", Digest: digest}, []string{newRef, oldRef}, row.present, row.counted) + if a.Reference != row.reference || !slices.Equal(a.References, row.references) { + t.Errorf("%s: %q %v, want %q %v", row.name, a.Reference, a.References, row.reference, row.references) + } + // An engine that predates the list reads the single reference, in a form every counted node reads (ADR 0624 §3). + if form, _ := fs.FormOf("bundle", a.Reference); form != row.kept[0] { + t.Errorf("%s: the single reference is in %q, want the oldest kept %q", row.name, form, row.kept[0]) + } + } + // An answered reference no kept form names still names where the artifact is (ADR 0498 clause 1). + if a := fs.Choose(delivery.BuiltArtifact{Name: "code", Kind: "bundle", Digest: digest}, []string{oldRef}, map[string][]string{"a": both}, + map[string][]string{"a": both}); a.Reference != oldRef { + t.Errorf("a lone answered reference was dropped: %+v", a) + } + withBoth := []delivery.BuiltArtifact{{Name: "code", Kind: "bundle", Digest: digest, Reference: oldRef, References: []string{oldRef, newRef}}} + if got := fs.Needs(withBoth); !slices.Equal(got, []string{"form:new", "form:old"}) { + t.Errorf("needs %v", got) + } + // What the assigner holds a want by (ADR 0624 §2), from the same forms. + for _, row := range []struct { + name string + needs []string + stated []string + held bool + }{ + {"a node stating {old}, a build carrying {old,new}: moves", []string{"form:new", "form:old"}, []string{"word:module"}, false}, + {"a node stating nothing, a build carrying the baseline's form: moves", []string{"form:old"}, nil, false}, + {"a build carrying {new} only, the node stating {old}: held", []string{"form:new"}, []string{"word:module"}, true}, + {"a build naming no form: moves", nil, nil, false}, + } { + if got := fs.Held(row.needs, row.stated); got != row.held { + t.Errorf("%s: held %v", row.name, got) + } + } +} diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 5e5b03e6..9dc2b71a 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -93,7 +93,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if m.Module != "mesh-delivery-builder" { t.Errorf("module = %q", m.Module) } - if want := []state{{Name: "builds", History: 5}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}, {Name: "tag-asks", History: 1}, {Name: "versions", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory + if want := []state{{Name: "builds", History: 5}, {Name: "engine-reads", History: 1}, {Name: "findings", History: 5}, {Name: "floor-waits", History: 1}, {Name: "hearing", History: 1}, {Name: "instances", History: 1}, {Name: "queues", History: 1}, {Name: "reasks", History: 1}, {Name: "tag-asks", History: 1}, {Name: "versions", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory t.Errorf("state = %+v, want %+v", m.State, want) } // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and @@ -110,7 +110,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 // 1955/L7 (ADR 0571 §2): the seat mesh-artifact-sweep's held, the controller's `held` is no longer asked (issue 1955). // Design 51 *The floor*: the floor's three sources, the registry seats' versions, bridge_tags and indexer_requirements. if want := []string{"mesh-delivery-forge-bridge.bridge_tags", "mesh-delivery-indexer.indexer_requirements", "seat:container-image-registry.remove", "seat:container-image-registry.versions", - "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:npm-package-registry.remove", + "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", "seat:node-engine.manifest-reads", "seat:npm-package-registry.remove", "seat:npm-package-registry.versions", "seat:oci-artifact-registry.remove", "seat:oci-artifact-registry.versions"}; !reflect.DeepEqual(sorted(m.Invokes), want) || len(m.Config) != 0 { t.Errorf("invokes = %v, config = %v, want %v and no config", m.Invokes, m.Config, want) } @@ -122,7 +122,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if want := []string{"container-image-registry.published", "container-image-registry.refused", "container-image-registry.removed", "go-package-registry.published", "go-package-registry.refused", "go-package-registry.removed", "mesh-artifact-sweep.artifacts-gone", "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", - "mesh-conditions.condition-cleared", "mesh-delivery-checker.tag-wanted", "mesh-delivery-forge-bridge.forge.**", "mesh-delivery-indexer.manifest-accepted", + "mesh-conditions.condition-cleared", "mesh-delivery-checker.tag-wanted", "mesh-delivery-forge-bridge.forge.**", "mesh-delivery-indexer.manifest-accepted", "node-engine.manifest-reads-changed", "npm-package-registry.published", "npm-package-registry.refused", "npm-package-registry.removed", "oci-artifact-registry.published", "oci-artifact-registry.refused", "oci-artifact-registry.removed"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) @@ -134,7 +134,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 t.Errorf("secrets-owner = %q", m.SecretsOwner) } // 1945: the builder's tools (builder_seat_queue: note 62), served by its tools bundle, and the event it says when it asks an ask again. - if want := []string{"builder_announce_again", "builder_ask_again", "builder_build_again", "builder_builds_since", "builder_floor_again", "builder_floor_waits", + if want := []string{"builder_announce_again", "builder_ask_again", "builder_build_again", "builder_builds_since", "builder_engine_reads", "builder_floor_again", "builder_floor_waits", "builder_publish_again", "builder_seat_queue", "builder_stuck", "builder_tag", "builder_tag_asks", "builder_untag", "builder_versions"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index f7392a4e..b4f51539 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -7,6 +7,7 @@ "builder_ask_again", "builder_build_again", "builder_builds_since", + "builder_engine_reads", "builder_floor_again", "builder_floor_waits", "builder_publish_again", @@ -36,6 +37,7 @@ "mesh-delivery-checker.tag-wanted", "mesh-delivery-forge-bridge.forge.**", "mesh-delivery-indexer.manifest-accepted", + "node-engine.manifest-reads-changed", "npm-package-registry.published", "npm-package-registry.refused", "npm-package-registry.removed", @@ -51,6 +53,7 @@ "seat:go-package-registry.remove", "seat:go-package-registry.versions", "seat:mesh-artifact-sweep.held", + "seat:node-engine.manifest-reads", "seat:npm-package-registry.remove", "seat:npm-package-registry.versions", "seat:oci-artifact-registry.remove", @@ -58,6 +61,7 @@ ], "state": [ { "name": "builds", "history": 5 }, + { "name": "engine-reads", "history": 1 }, { "name": "findings", "history": 5 }, { "name": "floor-waits", "history": 1 }, { "name": "hearing", "history": 1 }, diff --git a/modules/mesh-delivery-checker/internal/check/engines.go b/modules/mesh-delivery-checker/internal/check/engines.go index 2e077789..7b7f1909 100644 --- a/modules/mesh-delivery-checker/internal/check/engines.go +++ b/modules/mesh-delivery-checker/internal/check/engines.go @@ -214,7 +214,13 @@ func (e EngineCompositions) inSandbox(ctx context.Context, c *Case, head, node s delete(verdicts, name) // a manifest the change brings is the checker's own judge's (ADR 0578 §2) } } - req := composer.Request{V: composer.Version, Input: a.Input, Depth: composer.DepthCompose, Machines: []string{node}, + // The answer carries its input as the object the seat's contract names, decoded with its numbers as written + // (json.Number), so encoding it again gives the composer the same values the engine sent. + input, err := json.Marshal(a.Input) + if err != nil { + return a, fmt.Errorf("the engine of %s answered a composition input that cannot be encoded again: %w", node, err) + } + req := composer.Request{V: composer.Version, Input: input, Depth: composer.DepthCompose, Machines: []string{node}, Cases: []composer.Case{{Name: "change", Manifests: manifests, Verdicts: verdicts}}} raw, err := json.Marshal(req) if err != nil { diff --git a/modules/mesh-delivery-checker/internal/check/engines_test.go b/modules/mesh-delivery-checker/internal/check/engines_test.go index fed2774f..18f5ac10 100644 --- a/modules/mesh-delivery-checker/internal/check/engines_test.go +++ b/modules/mesh-delivery-checker/internal/check/engines_test.go @@ -143,7 +143,7 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) { } host := CaseOf(c, map[string]string{repoA: trunkA, nodeEngineRepo: trunkA}) withInput := composed("anchor") - withInput.Input = json.RawMessage(`{"nodes":[{"name":"anchor"}]}`) + withInput.Input = map[string]any{"nodes": []any{map[string]any{"name": "anchor"}}} e := &fakeEngines{answers: map[string]delivery.ComposeCandidateAnswer{"anchor": withInput}} sb := &fakeSandbox{v: func(node string) composer.Verdict { return composer.Verdict{V: composer.Version, Cases: []composer.CaseVerdict{{Name: "change", @@ -162,7 +162,7 @@ func TestTheCompositionIsMadeFromTheEnginesAnswers(t *testing.T) { var req composer.Request _ = json.Unmarshal(job.Request, &req) if job.Fetch["repository"] != nodeEngineRepo || job.Fetch["commit"] != head2 || job.Package != composer.Package || - string(req.Input) != string(withInput.Input) || req.Cases[0].Manifests["web"].Digest == "" { + string(req.Input) != `{"nodes":[{"name":"anchor"}]}` || req.Cases[0].Manifests["web"].Digest == "" { t.Errorf("job = %+v, request %+v", job, req) } if v.State != delivery.CompositionFail || v.SDK != head2 || !strings.Contains(v.Words, "the new composer refuses it") { diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 4c66d058..1de264a7 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -27,3 +27,4 @@ replace git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun require git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk + diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index 7679a5dc..98dc72c4 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,3 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/judge/rules_test.go b/modules/mesh-delivery-indexer/judge/rules_test.go index 5e3c4325..57409c6d 100644 --- a/modules/mesh-delivery-indexer/judge/rules_test.go +++ b/modules/mesh-delivery-indexer/judge/rules_test.go @@ -4873,7 +4873,8 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) { } } // The node seat node-engine is the node's engine's (novox/hq ADR 0563 §1): at node scope, delivering nothing, its verbs - // (compose-candidate since ADR 0593 §7) and three events, held by no assignment, and a module's claim of it refused. + // (compose-candidate since ADR 0593 §7, manifest-reads since ADR 0618 §3) and four events (manifest-reads-changed since + // ADR 0618 §4), held by no assignment, and a module's claim of it refused. if engine, ok := catalogue.SeatNamed(catalogue.EngineSeat); !ok || engine.Scope != catalogue.ScopeNode || engine.Delivers != "" || !engine.HeldByTheEngine { t.Errorf("node-engine is %+v, not a node seat delivering nothing that the engine holds", engine) } else { @@ -4885,8 +4886,8 @@ func TestTheSeatsAreAClosedSetAndEachNamesItsDecision(t *testing.T) { if props, _ := engine.Serves[5].Input["properties"].(map[string]any); props["id"] == nil { t.Errorf("acts takes no id: %v", engine.Serves[5].Input) } - if strings.Join(verbs, ",") != "hand-over,access-act,judge-directories,setuid-search,root-free,acts,packages,composition,compose,compose-candidate" || - strings.Join(engine.Emits, ",") != "asked,done,refused" { + if strings.Join(verbs, ",") != "hand-over,access-act,judge-directories,setuid-search,root-free,acts,packages,composition,compose,compose-candidate,pull-waits,manifest-reads" || + strings.Join(engine.Emits, ",") != "asked,done,refused,manifest-reads-changed" { t.Errorf("node-engine serves %v and emits %v", verbs, engine.Emits) } } @@ -5129,6 +5130,9 @@ func TestUsingASeatNobodyDeclaresIsRefused(t *testing.T) { catalogue.Manifest{Module: "shop", Invokes: []string{"seat:node-engine.root-free"}}, telegram(), false}, {"a verb the engine's seat does not promise", `the seat node-engine serves no verb "root-freed"`, catalogue.Manifest{Module: "shop", Invokes: []string{"seat:node-engine.root-freed"}}, telegram(), false}, + {"what the engine reads of a manifest, and its event (ADR 0618 §3, §4)", "", + catalogue.Manifest{Module: "shop", Invokes: []string{"seat:node-engine.manifest-reads"}, + Consumes: []string{"node-engine.manifest-reads-changed"}}, telegram(), false}, } { got := CatalogueProblems(catalogue.Shelf{"telegram": row.seat, "shop": row.shop}) joined := strings.Join(got, "; ") diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/decided.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/decided.go index ee7780b5..18c2b3d8 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/decided.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/decided.go @@ -11,6 +11,7 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/process" ) // The teller's three bus adapters for mesh-sdk go/asker, which in a bundle reaches the bus through the runtime: here the @@ -65,15 +66,32 @@ func recordVia(ctx context.Context, conn *nats.Conn, module string) func(id stri } // decidedVia hears the warrants the router says to this module alone: its own `decided` event, a subject only the router -// publishes (the grant to subscribe it names the module). The handler is the asker's: pattern is the key it expects. -func decidedVia(conn *nats.Conn, module string, logf func(string, ...any)) func(pattern string, handle func(stdio.Envelope) error) error { +// publishes (the grant to subscribe it names the module). They are pulled through the module's durable consumer on EVENTS +// (made with its grant for the seat it uses), so an answer the router published while the teller was down is delivered +// when it is back, never lost (issue 1126). hear starts the pull, handing each message to its handler; the message is +// acknowledged once handled. The handler is the asker's: pattern is the key it expects. +func decidedVia(hear func(func(jetstream.Msg)) error, logf func(string, ...any)) func(pattern string, handle func(stdio.Envelope) error) error { return func(pattern string, handle func(stdio.Envelope) error) error { - _, err := conn.Subscribe(asks.DecidedSubject(module), func(m *nats.Msg) { - if err := handle(stdio.Envelope{Key: pattern, Body: m.Data}); err != nil { + return hear(func(m jetstream.Msg) { + if err := handle(stdio.Envelope{Key: pattern, Body: m.Data()}); err != nil { logf("a warrant could not be handled and is read again from the router's record at the next start: %v", err) } + if err := m.Ack(); err != nil { + logf("a warrant was handled but could not be acknowledged: %v", err) + } }) - return err + } +} + +// pullDecided starts the pull of the module's durable consumer (process.HearEvents) and stops it when ctx ends. +func pullDecided(ctx context.Context, cons jetstream.Consumer, logf func(string, ...any)) func(func(jetstream.Msg)) error { + return func(handle func(jetstream.Msg)) error { + cc, err := process.HearEvents(cons, handle, logf) + if err != nil { + return err + } + context.AfterFunc(ctx, cc.Stop) + return nil } } diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/decided_test.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/decided_test.go new file mode 100644 index 00000000..ee1c86db --- /dev/null +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/decided_test.go @@ -0,0 +1,54 @@ +package main + +import ( + "errors" + "testing" + + "github.com/nats-io/nats.go/jetstream" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// pulled is a message of the durable consumer: only what decidedVia reads and does. +type pulled struct { + jetstream.Msg + data []byte + acked *int +} + +func (m pulled) Data() []byte { return m.data } +func (m pulled) Ack() error { *m.acked++; return nil } + +// An answer the router published while the teller was down is pulled from its durable consumer when it starts again: +// the handler hears the warrant under the asker's key, and the message is acknowledged, handled or not (issue 1126). +func TestDecidedViaHearsThePulledWarrantAndAcknowledgesIt(t *testing.T) { + var deliver func(jetstream.Msg) + var logs []string + sub := decidedVia(func(h func(jetstream.Msg)) error { deliver = h; return nil }, + func(f string, _ ...any) { logs = append(logs, f) }) + var got []stdio.Envelope + fail := false + if err := sub("k", func(e stdio.Envelope) error { + got = append(got, e) + if fail { + return errors.New("no") + } + return nil + }); err != nil || deliver == nil { + t.Fatalf("subscribe: %v, delivering %v", err, deliver != nil) + } + acked := 0 + deliver(pulled{data: []byte(`{"ask":"a1"}`), acked: &acked}) + fail = true + deliver(pulled{data: []byte(`{"ask":"a2"}`), acked: &acked}) + if len(got) != 2 || got[0].Key != "k" || string(got[0].Body) != `{"ask":"a1"}` || string(got[1].Body) != `{"ask":"a2"}` { + t.Errorf("heard %+v", got) + } + if acked != 2 || len(logs) != 1 { + t.Errorf("acknowledged %d of 2, logged %d lines, want 1 (the failed handling)", acked, len(logs)) + } + failing := decidedVia(func(func(jetstream.Msg)) error { return errors.New("no consumer") }, nil) + if err := failing("k", nil); err == nil { + t.Error("a pull that could not start is no error") + } +} diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go index 5754987a..f7e015f0 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go @@ -165,15 +165,16 @@ func user(name string, pub, sub []string) string { } const ( - tellerAnswers = "mesh-delivery-teller_answers" - tellerAsked = "mesh-delivery-teller_asked" - gateAsksKV = "mesh-delivery-gate_asks" - assignerAsks = "mesh-delivery-assigner_asks" - viewAnswers = "mesh-delivery-view_answers" - seatAsk = "mesh.seat.operator-channel.accept.ask.mesh-delivery-teller" - seatCancel = "mesh.seat.operator-channel.accept.cancel.mesh-delivery-teller" - seatCondition = "mesh.seat.operator-channel.accept.condition.mesh-delivery-teller" - seatDecided = "mesh.seat.operator-channel.event.decided.mesh-delivery-teller" + tellerAnswers = "mesh-delivery-teller_answers" + tellerAsked = "mesh-delivery-teller_asked" + gateAsksKV = "mesh-delivery-gate_asks" + assignerAsks = "mesh-delivery-assigner_asks" + viewAnswers = "mesh-delivery-view_answers" + tellerConsumer = "n1_mesh-delivery-teller" + seatAsk = "mesh.seat.operator-channel.accept.ask.mesh-delivery-teller" + seatCancel = "mesh.seat.operator-channel.accept.cancel.mesh-delivery-teller" + seatCondition = "mesh.seat.operator-channel.accept.condition.mesh-delivery-teller" + seatDecided = "mesh.seat.operator-channel.event.decided.mesh-delivery-teller" ) func fullConf(dir string) string { @@ -181,6 +182,8 @@ func fullConf(dir string) string { // without it held the start ten seconds on a read the bus never answered, and the tests raced that stall. teller := append(append(append(append(keepsOf(tellerAnswers), keepsOf(tellerAsked)...), keepsOf(ends.Bucket)...), readOf(gateAsksKV)...), seatAsk, seatCancel, seatCondition, "$JS.API.DIRECT.GET.KV_"+messengerAsks+".$KV."+messengerAsks+".mesh-delivery-teller.>") + // The durable consumer on EVENTS the controller makes for the seat it uses, pulled and acknowledged (issue 1126). + teller = append(teller, "$JS.API.CONSUMER.INFO.EVENTS."+tellerConsumer, "$JS.API.CONSUMER.MSG.NEXT.EVENTS."+tellerConsumer, "$JS.ACK.EVENTS."+tellerConsumer+".>") teller = append(append(teller, readOf(assignerAsks)...), readOf(viewAnswers)...) // issue 1297 B10: the manifest's reads view := append(append(append(readOf(tellerAnswers), readOf(gateAsksKV)...), readOf(assignerAsks)...), keepsOf(viewAnswers)...) assigner := keepsOf(assignerAsks) @@ -197,7 +200,7 @@ accounts { } } `, filepath.Join(dir, "js"), - user("n1.mesh-delivery-teller", teller, []string{seatDecided}), + user("n1.mesh-delivery-teller", teller, nil), user("n1.mesh-delivery-view", view, nil), user("n1.mesh-delivery-assigner", assigner, nil), user("n1.node-tools", tools, nil)) @@ -257,20 +260,30 @@ func startFull(t *testing.T) *fullBed { Retention: jetstream.WorkQueuePolicy}); err != nil { t.Fatal(err) } + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.seat.operator-channel.event.>"}}); err != nil { + t.Fatal(err) + } + if _, err := js.CreateOrUpdateConsumer(ctx, "EVENTS", jetstream.ConsumerConfig{Durable: tellerConsumer, FilterSubject: seatDecided, + AckPolicy: jetstream.AckExplicitPolicy}); err != nil { + t.Fatal(err) + } return &fullBed{srv: srv, ctl: ctl, js: js, ctx: ctx} } -// listening waits until the program hears the router's warrants (its subscription on the decided subject is on the bus), +// listening waits until the program hears the router's warrants (its pull of the durable consumer is waiting on the bus), // which it does before it watches any bucket: an event, not a pause a loaded machine can outlast. func (b *fullBed) listening(t *testing.T) { t.Helper() - acc, err := b.srv.LookupAccount("MESH") + cons, err := b.js.Consumer(b.ctx, "EVENTS", tellerConsumer) if err != nil { t.Fatal(err) } - for deadline := time.Now().Add(20 * time.Second); !acc.SubscriptionInterest(seatDecided); time.Sleep(10 * time.Millisecond) { + for deadline := time.Now().Add(20 * time.Second); ; time.Sleep(10 * time.Millisecond) { + if info, err := cons.Info(b.ctx); err == nil && info.NumWaiting > 0 { + return + } if time.Now().After(deadline) { - t.Fatal("the program did not subscribe to the router's warrants within 20 s") + t.Fatal("the program did not pull the router's warrants within 20 s") } } } diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go index ad4a99fe..8557c036 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go @@ -67,8 +67,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } state := kvStore{ctx: ctx, kv: kvs[1]} publish := publishVia(ctx, js) + cons, err := cred.EventConsumer(ctx, js, process.EventsStream) + if err != nil { + // Essential: the router's warrants are how an answer reaches the teller. + return fmt.Errorf("the consumer %s on %s cannot be read (%v): the teller hears no answer without it, so it stops; where the bus refused it or did not answer, this module's consumer (made with its grant for the seat it uses) has not reached the bus yet", cred.EventConsumerName(), process.EventsStream, err) + } client := &asker.Client{Module: delivery.TellerModule, State: state, Publish: publish, - Record: recordVia(ctx, conn, delivery.TellerModule), Subscribe: decidedVia(conn, delivery.TellerModule, logger.Printf)} + Record: recordVia(ctx, conn, delivery.TellerModule), Subscribe: decidedVia(pullDecided(ctx, cons, logger.Printf), logger.Printf)} t := &Teller{Asker: client, Store: state, Answers: kvAnswers{ctx: ctx, kv: kvs[0]}, Gate: kvGate{ctx: ctx, kv: gate}, Say: sayVia(ctx, js, delivery.TellerModule), Logf: logger.Printf, Publish: publish} // Hear the warrants first, so none that comes while the asks are read is missed. diff --git a/modules/mesh-delivery-teller/manifest_test.go b/modules/mesh-delivery-teller/manifest_test.go index 2111e273..74420f55 100644 --- a/modules/mesh-delivery-teller/manifest_test.go +++ b/modules/mesh-delivery-teller/manifest_test.go @@ -149,9 +149,7 @@ func TestManifestProcess(t *testing.T) { // 1240/AC1 func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1240/AC3 (ADR 0437 §8) forbidden := []string{"Subscribe(", "QueueSubscribe", "ChanSubscribe", "Respond(", "RespondMsg(", "time.NewTicker", "time.Tick", "time.NewTimer", "time.AfterFunc", "time.After(", "time.Sleep"} - // The one subscription the teller makes is to its own `decided` event, the router's word on its asks (ADR 0259, ADR 0437 - // §6): an event it hears, not a request it answers. It is allowed in decided.go and there only on that subject. - const decidedFile = "decided.go" + // The teller hears its own `decided` event through its durable consumer, pulled (decided.go): it holds no subscription. var files int for _, root := range []string{"cmd", "internal"} { err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { @@ -167,13 +165,6 @@ func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1240/AC3 (ADR return err } for _, f := range forbidden { - if f == "Subscribe(" && filepath.Base(path) == decidedFile { - if n := strings.Count(string(src), "conn.Subscribe("); n != 1 || !strings.Contains(string(src), "conn.Subscribe(asks.DecidedSubject(module),") || - strings.Count(string(src), f) != n { - t.Errorf("%s holds %d subscriptions, and not just one to asks.DecidedSubject(module)", path, strings.Count(string(src), f)) - } - continue - } if strings.Contains(string(src), f) { t.Errorf("%s holds %s: the teller serves no request subject and holds no timer", path, f) } diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go index 746dc902..4f681d8d 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main.go @@ -18,11 +18,88 @@ import ( "git.novox.be/novox/mesh-sdk/go/provisions" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/busretry" + + "git.novox.be/novox/mesh-delivery/sdk/enginereads" ) +// ManifestReadsTool lists, per node, what its engine states it reads, by asking the node-engine seat's verb of every node +// at once (ADR 0618 §3). +const ManifestReadsTool = "node_manifest_reads" + +// ManifestReadsBound bounds each node's ask of the survey: the whole call stays within a tool call's 30 s. +var ManifestReadsBound = 10 * time.Second + +// askEngine is the survey's caller: the node-engine seat's verb of one machine, through the runtime (stdio.Ask). An error +// that is no lack of an answer is the engine's own words: it answered and does not serve the verb. +func askEngine(node, key string, body any) (json.RawMessage, error) { + raw, err := stdio.Ask(key+"@"+node, body) + if err != nil { + words := strings.ToLower(err.Error()) + if !strings.Contains(words, "no responders") && !strings.Contains(words, "timeout") && !strings.Contains(words, "deadline") { + return nil, &busretry.Answered{Words: err.Error()} + } + } + return raw, err +} + +// engineCaller is how node_manifest_reads asks; a test's fake in its place. +var engineCaller enginereads.Caller = askEngine + +// manifestReads is node_manifest_reads over r's haves and presence: every node with a record or a presence, asked through +// call. +func manifestReads(r view.Reader, call enginereads.Caller) (any, error) { + set := map[string]bool{} + records := map[string][]string{} + if r.HavesState.Keys != nil { + keys, err := r.HavesState.Keys() + if err != nil { + return nil, fmt.Errorf("the haves cannot be read: %w", err) + } + for _, key := range keys { + node, _, isNode, err := delivery.ParseHaveKey(key) + if err != nil || !isNode { + continue + } + set[node] = true + if raw, err := r.HavesState.Get(key); err == nil && raw != nil { + if h, err := delivery.DecodeHaveNode(raw); err == nil && h.Record != nil { + records[node] = h.Record.ManifestReads + } + } + } + } + if r.PresenceState.Keys != nil { + if keys, err := r.PresenceState.Keys(); err == nil { + for _, key := range keys { + if node, part, _, err := delivery.ParsePresenceKey(key); err == nil && part == "" && node != "" { + set[node] = true + } + } + } + } + nodes := make([]string, 0, len(set)) + for node := range set { + nodes = append(nodes, node) + } + rows := enginereads.Survey(call, nodes, func(node string) ([]string, bool) { + reads, ok := records[node] + return reads, ok + }, ManifestReadsBound, time.Now()) + return map[string]any{"nodes": rows}, nil +} + // Tools are the read tools over a modules state, and build_usage over the build modules' states. func Tools(r view.Reader, u view.UsageReader) []stdio.Tool { return []stdio.Tool{ + {Name: ManifestReadsTool, + Description: "What each node's engine states it reads of a manifest (manifest words, forms and seat names, novox/hq ADR 0618 " + + "§3), for the whole mesh in one call: every node with a record or a presence is asked the node-engine seat's verb " + + "manifest-reads at once, each within " + ManifestReadsBound.String() + ". Per node: stated (with the capabilities and " + + "when the engine stated them), not served (the engine is built before the verb) or away (nobody answered), and for the " + + "last two the reads its node record states, which the assigner reads in their place. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { return manifestReads(r, engineCaller) }}, {Name: view.StatusTool, Description: "The mesh's status, derived by the view from the buckets it reads (novox/hq ADR 0564 §10), nothing written as a " + "summary: every open condition of the condition keeper (mesh-conditions), urgent first then oldest; what is quiet (each " + diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go index 144d9d30..33f9e8b7 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-view-tools/main_test.go @@ -11,8 +11,12 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/busretry" "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + "git.novox.be/novox/mesh-delivery/sdk/enginereads" + "mesh-delivery-view/internal/view" ) @@ -56,6 +60,7 @@ const ( func TestModuleToolAnswersTheStoredValue(t *testing.T) { t.Run("indexer_packages answers the published modules", packagesToolAnswersThePublishedModules) t.Run("view_tiers names each module's wait", tiersToolSaysEachModulesWait) + t.Run("node_manifest_reads asks every node's engine in one call", manifestReadsAsksEveryNode) r := fake{"a": withFP2, "b": withNone}.reader() for key, want := range map[string]string{"a": withFP2, "b": withNone} { out, err := run(t, r, view.ModuleReadTool, map[string]any{"module": key}) @@ -570,3 +575,48 @@ func packagesToolAnswersThePublishedModules(t *testing.T) { t.Error("a path that is no mesh Go module's was read") } } + +// manifestReadsAsksEveryNode: node_manifest_reads lists each node's capabilities by asking the node-engine seat's verb +// (ADR 0618 §3) of every node with a record or a presence, at once, each within its bound: stated, not served (with its +// record's reads, which the assigner reads in their place) or away (a node that never answers holds no other). +func manifestReadsAsksEveryNode(t *testing.T) { + was, bound := engineCaller, ManifestReadsBound + defer func() { engineCaller, ManifestReadsBound = was, bound }() + ManifestReadsBound = 200 * time.Millisecond + hang := make(chan struct{}) + defer close(hang) + engineCaller = func(node, key string, _ any) (json.RawMessage, error) { + switch { + case key != "seat:node-engine.manifest-reads": + return nil, fmt.Errorf("asked %s", key) + case node == "g14": + return nil, &busretry.Answered{Words: "the seat node-engine serves no verb manifest-reads"} + case node == "novox": + <-hang // never answers within the bound + return nil, nil + } + return json.Marshal(nodeengine.ManifestReadsAnswer{V: 1, Node: node, ManifestReads: []string{"form:new"}, StatedAt: time.Now().Format(time.RFC3339Nano)}) + } + record := func(node string, reads ...string) string { + raw, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: node, ManifestReads: reads}}) + return string(raw) + } + on, _ := delivery.EncodePresence(delivery.Presence{Online: true}) + r := view.Reader{HavesState: fake{"ace.node": record("ace"), "g14.node": record("g14", "word:module"), "ace.web": "{}"}.reader().State, + PresenceState: fake{"novox": string(on)}.reader().State} + started := time.Now() + out, err := run(t, r, ManifestReadsTool, nil) + if err != nil || time.Since(started) > 2*time.Second { + t.Fatalf("node_manifest_reads: %v after %s", err, time.Since(started)) + } + rows, _ := out.(map[string]any)["nodes"].([]enginereads.SurveyRow) + byNode := map[string]enginereads.SurveyRow{} + for _, row := range rows { + byNode[row.Node] = row + } + if len(rows) != 3 || byNode["ace"].Answer != enginereads.Stated || byNode["ace"].ManifestReads[0] != "form:new" || + byNode["g14"].Answer != enginereads.NotServed || !byNode["g14"].RecordRead || byNode["g14"].Record[0] != "word:module" || + byNode["novox"].Answer != enginereads.Away || byNode["novox"].RecordRead { + t.Errorf("node_manifest_reads = %+v", rows) + } +} diff --git a/modules/mesh-delivery-view/go.mod b/modules/mesh-delivery-view/go.mod index 035f39f7..46742fd5 100644 --- a/modules/mesh-delivery-view/go.mod +++ b/modules/mesh-delivery-view/go.mod @@ -24,3 +24,4 @@ require ( require git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 replace git.novox.be/novox/mesh-delivery/sdk => ../../sdk + diff --git a/modules/mesh-delivery-view/go.sum b/modules/mesh-delivery-view/go.sum index 7679a5dc..98dc72c4 100644 --- a/modules/mesh-delivery-view/go.sum +++ b/modules/mesh-delivery-view/go.sum @@ -1,5 +1,3 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-view/manifest_test.go b/modules/mesh-delivery-view/manifest_test.go index 4e65b88f..ee78cba2 100644 --- a/modules/mesh-delivery-view/manifest_test.go +++ b/modules/mesh-delivery-view/manifest_test.go @@ -53,6 +53,7 @@ type manifest struct { Reads []string `json:"reads"` Uses []string `json:"uses"` Tools []string `json:"tools"` + Invokes []string `json:"invokes"` Emits []string `json:"emits"` OwnSecrets map[string]string `json:"own-secrets"` SecretsOwner string `json:"secrets-owner"` @@ -119,9 +120,13 @@ func TestManifestIsTakenAndReadsOnly(t *testing.T) { // 1134/AC7 } // The two read tools of the indexer's modules (issue 1562), carried by a bundle the runtime launches; the indexer // itself serves no tool (ADR 0437 §4). Carrying them gives the tool runner read access to every bucket the view reads. - if want := []string{"view_status", "indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "indexer_predefined", "indexer_config_candidates", "indexer_packages", "builder_build", "builder_builds", "builder_history", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "node_trusts", "build_usage", "view_reach", "indexer_module_history", "indexer_heard", "view_tiers"}; !reflect.DeepEqual(m.Tools, want) { + if want := []string{"view_status", "indexer_module", "indexer_modules", "indexer_candidate", "indexer_candidates", "indexer_predefined", "indexer_config_candidates", "indexer_packages", "builder_build", "builder_builds", "builder_history", "assigner_wants", "assigner_want", "assigner_differences", "node_haves", "node_trusts", "build_usage", "view_reach", "indexer_module_history", "indexer_heard", "view_tiers", "node_manifest_reads"}; !reflect.DeepEqual(m.Tools, want) { t.Errorf("tools = %v, want %v", m.Tools, want) } + // ADR 0618 §3: node_manifest_reads asks the node-engine seat's verb of every node, and the view invokes nothing else. + if want := []string{"seat:node-engine.manifest-reads"}; !reflect.DeepEqual(m.Invokes, want) { + t.Errorf("invokes = %v, want %v", m.Invokes, want) + } for _, a := range m.Build.Artifacts { // A bundle compiled to a binary is one executable: what the runtime loads is that binary's name, or the controller's // module check refuses the manifest (mesh-sdk catalogue build.go, ADR 0193). diff --git a/modules/mesh-delivery-view/module.json b/modules/mesh-delivery-view/module.json index 55bbbfb6..14523359 100644 --- a/modules/mesh-delivery-view/module.json +++ b/modules/mesh-delivery-view/module.json @@ -24,7 +24,11 @@ "view_reach", "indexer_module_history", "indexer_heard", - "view_tiers" + "view_tiers", + "node_manifest_reads" + ], + "invokes": [ + "seat:node-engine.manifest-reads" ], "emits": [ "delivered", diff --git a/modules/mesh-grants/go.mod b/modules/mesh-grants/go.mod index ee23823d..3cf0be53 100644 --- a/modules/mesh-grants/go.mod +++ b/modules/mesh-grants/go.mod @@ -20,3 +20,4 @@ require ( golang.org/x/sys v0.48.0 // indirect golang.org/x/time v0.15.0 // indirect ) + diff --git a/modules/mesh-grants/go.sum b/modules/mesh-grants/go.sum index 7679a5dc..98dc72c4 100644 --- a/modules/mesh-grants/go.sum +++ b/modules/mesh-grants/go.sum @@ -1,5 +1,3 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-grants/internal/buscompose/buscompose_test.go b/modules/mesh-grants/internal/buscompose/buscompose_test.go index 16a053d0..599ac580 100644 --- a/modules/mesh-grants/internal/buscompose/buscompose_test.go +++ b/modules/mesh-grants/internal/buscompose/buscompose_test.go @@ -712,7 +712,16 @@ func healthChecks(t *testing.T) { // controller's built-before, and each node's engine's of restic's backed-up (issue 1126). "$JS.API.CONSUMER.INFO.EVENTS.novox_mesh-catalog", "$JS.API.CONSUMER.DELETE.EVENTS.novox_mesh-catalog", "$JS.API.CONSUMER.INFO.EVENTS.a_mesh-host", "$JS.API.CONSUMER.DELETE.EVENTS.a_mesh-host", - "$JS.API.CONSUMER.INFO.CONTROL.controller"}, + "$JS.API.CONSUMER.INFO.CONTROL.controller", + // The take-in of the forge's events the controller made for the forge module, which no longer consumes it. + "$JS.API.CONSUMER.INFO.EVENTS.novox_gitea", "$JS.API.CONSUMER.DELETE.EVENTS.novox_gitea", + // And the controller's take-in and config-follow consumers on three modules' buckets, each by its name (issue 1126). + "$JS.API.CONSUMER.INFO.KV_mesh-delivery-builder_builds.mesh-controller-take-in", + "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-builder_builds.mesh-controller-take-in", + "$JS.API.CONSUMER.INFO.KV_mesh-delivery-indexer_modules.mesh-controller-take-in", + "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-indexer_modules.mesh-controller-take-in", + "$JS.API.CONSUMER.INFO.KV_mesh-config_resolved.mesh-controller-config-follow", + "$JS.API.CONSUMER.DELETE.KV_mesh-config_resolved.mesh-controller-config-follow"}, } mayNot := map[string][]string{ // The keeper writes no findings and no attempts; no observer writes the keeper's buckets (clause 2). @@ -760,7 +769,13 @@ func healthChecks(t *testing.T) { "$JS.API.STREAM.UPDATE.KV_mesh-host_acts", "$JS.API.STREAM.UPDATE.KV_mesh-controller_calls", "$JS.API.STREAM.UPDATE.EVENTS", "$JS.API.STREAM.UPDATE.KV_mesh-controller_lease", "$JS.API.STREAM.PURGE.KV_mesh-controller_lease", "$JS.API.STREAM.DELETE.KV_mesh-controller_other", "$JS.API.STREAM.DELETE.OBJ_declaration-files-zz", - "$JS.API.CONSUMER.DELETE.EVENTS.a_mesh-delivery", "$KV.mesh-controller_lease.holder"}, + "$JS.API.CONSUMER.DELETE.EVENTS.a_mesh-delivery", "$KV.mesh-controller_lease.holder", + // Only the three named consumers of those buckets, never another of them nor a wider form (issue 1126). + "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-builder_builds.mesh-delivery-take-in", + "$JS.API.CONSUMER.DELETE.EVENTS.novox_gitea-other", "$JS.API.CONSUMER.DELETE.EVENTS.a_gitea", + "$JS.API.CONSUMER.DELETE.KV_mesh-delivery-indexer_modules.mesh-config-follow", + "$JS.API.CONSUMER.DELETE.KV_mesh-config_resolved.mesh-controller-take-in", + "$JS.API.CONSUMER.DELETE.KV_mesh-config_resolved.other", "$JS.API.STREAM.DELETE.KV_mesh-config_resolved"}, "a.nats.other": {"$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", "$KV.mesh-host_acts.g14.a1", "$JS.API.STREAM.UPDATE.KV_mesh-controller_lease"}, "a.nats": {"$JS.API.CONSUMER.DELETE.DEAD_LETTER_NOTICES.controller", "$KV.mesh-host_acts.g14.a1"}, @@ -843,6 +858,8 @@ func healthChecks(t *testing.T) { {"the holder says no other seat's", selfCheck, "mesh.seat.mesh-conditions.event.condition-raised", false}, {"a machine's event on a node seat is not composed here", Declared{Module: "pause", Holds: []Seat{{Name: "node-power", Scope: "node", Emits: []string{"paused.*"}}}}, "mesh.seat.node-power.event.paused.b", false}, + {"a flat event of a node seat held by a module is said flat", Declared{Module: "pause", Holds: []Seat{{Name: "node-power", + Scope: "node", Emits: []string{"slept"}}}}, "mesh.seat.node-power.event.slept", true}, } { m := MembershipFor("a", c.d, Placements{}) listed := m.SeatTraffic != nil && slices.ContainsFunc(m.SeatTraffic.Publish, func(p string) bool { return p == c.subject }) @@ -855,6 +872,28 @@ func healthChecks(t *testing.T) { MayPublish(perms, c.subject), c.may) } } + // ADR 0618 §4: the node-engine says each of its seat's events with its own machine as the last token, and a watcher hears + // that event from every machine, its consumer filtered the same way; a flat event of a node seat a module holds is + // heard flat. One rule (machineEvent) places both sides. + engine, err := PermissionsFor(Principal{Kind: KindNode, Node: "ace"}) + if err != nil { + t.Fatal(err) + } + watcher := Principal{Kind: KindModule, Node: "novox", Module: "mesh-delivery-assigner", Watches: []Seat{ + {Name: EngineSeat, Scope: "node", Emits: []string{"manifest-reads-changed"}, HeldByTheEngine: true}, + {Name: "node-power", Scope: "node", Emits: []string{"slept"}}}} + heard, err := PermissionsFor(watcher) + if err != nil { + t.Fatal(err) + } + consumer, _ := ConsumerFor(watcher) + said := "mesh.seat.node-engine.event.manifest-reads-changed.ace" + if !MayPublish(engine, said) || MayPublish(engine, "mesh.seat.node-engine.event.manifest-reads-changed.novox") || + !MaySubscribe(heard, said) || !MaySubscribe(heard, "mesh.seat.node-power.event.slept") || + !slices.Contains(consumer.Filters, "mesh.seat.node-engine.event.manifest-reads-changed.*") { + t.Errorf("the engine says %s: %v; the watcher hears it %v, the flat event %v; its consumer filters %v", said, + MayPublish(engine, said), MaySubscribe(heard, said), MaySubscribe(heard, "mesh.seat.node-power.event.slept"), consumer.Filters) + } seen := map[string]bool{} for _, u := range users { name := u.Username() diff --git a/modules/mesh-grants/internal/buscompose/engine.go b/modules/mesh-grants/internal/buscompose/engine.go index 6426db7c..70853571 100644 --- a/modules/mesh-grants/internal/buscompose/engine.go +++ b/modules/mesh-grants/internal/buscompose/engine.go @@ -51,7 +51,9 @@ func EnginePackagesGrants(node string) []string { // principal may subscribe this seat's subjects of this node, and a caller is still granted each verb by name. func engineGrants(p Principal) (pub, sub []string) { sub = append(sub, "mesh.seat."+EngineSeat+".tool.*."+p.Node) - pub = append(pub, "mesh.seat."+EngineSeat+".event.*."+p.Node) + // Any event of the seat about its own node, by the one rule a watcher of it is composed by (machineEvent). + every, _ := machineEvent(Seat{Name: EngineSeat, Scope: "node", HeldByTheEngine: true}, "*") + pub = append(pub, seatSubject(Seat{Name: EngineSeat}, "event", onMachine(every, p.Node))) sub = append(sub, announcing(EngineSeat)...) pub = append(pub, discovering()...) tp, ts := seatTrafficGrants(SeatTrafficOf(asks.EngineAsker(p.Node), nil, p.Uses, nil)) diff --git a/modules/mesh-grants/internal/buscompose/fixtures_test.go b/modules/mesh-grants/internal/buscompose/fixtures_test.go index bfeda2f4..aac9e87c 100644 --- a/modules/mesh-grants/internal/buscompose/fixtures_test.go +++ b/modules/mesh-grants/internal/buscompose/fixtures_test.go @@ -73,6 +73,17 @@ func UserListFixtures() []UserListFixture { }, RootFree: map[string]bool{"ace": true}, }}, + // ADR 0618 §4: an event of the node-engine's seat names the engine's machine, so its watcher hears it from every + // machine; a node seat held by a module says its other events flat, and is heard flat. + {Name: "a node seat's events, by machine and flat", Records: Records{ + Nodes: []string{"ace", "novox"}, + Assigned: map[string][]Declared{ + "novox": {{Module: "mesh-delivery-assigner", Watches: []Seat{ + {Name: EngineSeat, Scope: "node", Emits: []string{"manifest-reads-changed"}, HeldByTheEngine: true}, + {Name: "node-power", Scope: "node", Emits: []string{"slept"}}}}}, + "ace": {{Module: "pause", Holds: []Seat{{Name: "node-power", Scope: "node", Emits: []string{"slept", "paused.*"}}}}}, + }, + }}, } } diff --git a/modules/mesh-grants/internal/buscompose/health_grants.go b/modules/mesh-grants/internal/buscompose/health_grants.go index 7b06c49a..4b9bb5db 100644 --- a/modules/mesh-grants/internal/buscompose/health_grants.go +++ b/modules/mesh-grants/internal/buscompose/health_grants.go @@ -28,8 +28,12 @@ var HealersWould = health.HealersModule + "." + health.WouldState // mesh-delivery placed nowhere (issue 1126, milestone section 3), and mesh-catalog's of the controller's built-before, // which its manifest no longer consumes and no part composes; each `.`. var ControllerLeftoverConsumers = []string{ - "EVENTS." + ControllerName, "EVENTS." + MergesConsumerName, "EVENTS.novox_mesh-delivery", "EVENTS.novox_mesh-catalog", + "EVENTS." + ControllerName, "EVENTS." + MergesConsumerName, "EVENTS.novox_mesh-delivery", "EVENTS.novox_mesh-catalog", "EVENTS.novox_gitea", "CONTROL." + ControllerName, DeadLetterNoticesStream + "." + ControllerName, + // The controller's consumers on three modules' buckets (issue 1126): the take-in on the builder's builds and on the + // indexer's modules, and the config follow on the resolved configuration. + "KV_mesh-delivery-builder_builds.mesh-controller-take-in", "KV_mesh-delivery-indexer_modules.mesh-controller-take-in", + "KV_mesh-config_resolved.mesh-controller-config-follow", } // ControllerLeftoverConsumersOn are ControllerLeftoverConsumers and, for each node given, the consumer the controller diff --git a/modules/mesh-grants/internal/buscompose/nats.go b/modules/mesh-grants/internal/buscompose/nats.go index 7f85009a..8ad0c7b3 100644 --- a/modules/mesh-grants/internal/buscompose/nats.go +++ b/modules/mesh-grants/internal/buscompose/nats.go @@ -98,6 +98,9 @@ type Seat struct { // PulledByKind says each accepted kind is a worker of its own on the seat's queue, filtered to that kind, which // every holder pulls in the order Accepts lists them: the build seats' (novox/hq ADR 0525 §2, ADR 0536 §4). PulledByKind bool + // HeldByTheEngine says every node's engine holds the seat on its own account (catalogue.Seat.HeldByTheEngine, novox/hq + // ADR 0563 §1): each of its events names the engine's machine as its last token (machineEvent). + HeldByTheEngine bool } // A Principal is one user of the bus. Its permissions are derived from what it declares and @@ -406,6 +409,25 @@ func NodeLoginsReads(node string) []string { // holder's machine (novox/hq ADR 0219): `paused.`, the build agent saying whether it takes work. var perMachineEvents = map[string]bool{"paused.*": true} +// machineEvent is the one rule for where a node-scoped seat's event names its holder's machine as its last token (novox/hq +// ADR 0219, ADR 0618 §4), as a pattern whose last token `*` is that machine, and whether e is such an event: every event +// of a seat the node-engine holds (engineGrants), and each of perMachineEvents of any other node seat. The holder is +// granted the pattern with its own machine in place of `*`, a watcher the pattern, so the two cannot differ. +func machineEvent(s Seat, e string) (string, bool) { + switch { + case s.Scope != "node": + return "", false + case perMachineEvents[e]: + return e, true + case s.HeldByTheEngine: + return e + ".*", true + } + return "", false +} + +// onMachine is a machineEvent pattern with node in place of its last `*`. +func onMachine(pattern, node string) string { return strings.TrimSuffix(pattern, "*") + node } + // enrolmentPrefix is the space every enrolling node's user and inbox live under, so the one place the // controller may answer an enrolment is derived from the same constant the user is named from. const enrolmentPrefix = "enrol" @@ -879,6 +901,11 @@ func PermissionsFor(p Principal) (Permissions, error) { continue // composed by SeatTrafficOf below } for _, e := range w.Emits { + // An event that names its holder's machine is heard from every machine (machineEvent): a flat subject would + // match no event the holder says. + if pattern, ok := machineEvent(w, e); ok { + e = pattern + } sub = append(sub, seatSubject(w, "event", e)) } } @@ -921,8 +948,8 @@ func PermissionsFor(p Principal) (Permissions, error) { // **A machine says its own state and no other's** (novox/hq ADR 0219): on a node-scoped // seat, an event about the holder itself carries the machine as its last token, and // each holder is granted its own machine's alone. - if s.Scope == "node" && p.Node != "" && perMachineEvents[e] { - pub = append(pub, seatSubject(s, "event", strings.TrimSuffix(e, "*")+p.Node)) + if pattern, ok := machineEvent(s, e); ok && p.Node != "" { + pub = append(pub, seatSubject(s, "event", onMachine(pattern, p.Node))) continue } pub = append(pub, seatSubject(s, "event", e)) diff --git a/modules/mesh-grants/internal/buscompose/seattraffic.go b/modules/mesh-grants/internal/buscompose/seattraffic.go index 243f7ace..70356c86 100644 --- a/modules/mesh-grants/internal/buscompose/seattraffic.go +++ b/modules/mesh-grants/internal/buscompose/seattraffic.go @@ -64,7 +64,7 @@ func SeatTrafficOf(module string, holds, uses, watches []Seat) SeatTraffic { // runtime and the runtime is granted them. A machine's event on a node seat carries the machine, which // only the holder's own account is composed with, so it is left to that rule. for _, e := range s.Emits { - if s.Scope == "node" && perMachineEvents[e] { + if _, ok := machineEvent(s, e); ok { continue } t.Publish = append(t.Publish, seatSubject(s, "event", e)) diff --git a/modules/mesh-grants/internal/buscompose/testdata/user-lists.json b/modules/mesh-grants/internal/buscompose/testdata/user-lists.json index 842a9753..48c11799 100644 --- a/modules/mesh-grants/internal/buscompose/testdata/user-lists.json +++ b/modules/mesh-grants/internal/buscompose/testdata/user-lists.json @@ -285,5 +285,48 @@ ] } } + }, + { + "Name": "a node seat's events, by machine and flat", + "List": "# The mesh's users, composed by the grants part. Do not edit: the next\n# composition overwrites it. Permissions are derived from what each module\n# declares and nothing else (novox/hq ADR 0043, design 29 §2).\n\naccounts {\n MESH {\n jetstream: enabled\n imports = [\n { stream: { account: \"$SYS\", subject: \"$SYS.ACCOUNT.MESH.CONNECT\" }, to: \"mesh.bus.connect\" }\n { stream: { account: \"$SYS\", subject: \"$SYS.ACCOUNT.MESH.DISCONNECT\" }, to: \"mesh.bus.disconnect\" }\n { service: { account: \"$SYS\", subject: \"$SYS.REQ.ACCOUNT.MESH.CONNZ\" }, to: \"mesh.bus.connz\" }\n ]\n users = [\n { user: \"ace.pause\", password: \"$2a$11$ace.pauseace.pauseace.fixturefixturefixturefixture00\", permissions: {\n publish: { allow: [\"$JS.ACK.EVENTS.ace_pause.\u003e\", \"$JS.ACK.SEAT_NODE_POWER.SEAT_NODE_POWER_worker.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.ace.pause\", \"$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.\u003e\", \"$JS.API.CONSUMER.INFO.EVENTS.ace_pause\", \"$JS.API.CONSUMER.INFO.SEAT_NODE_POWER.SEAT_NODE_POWER_worker\", \"$JS.API.CONSUMER.MSG.NEXT.EVENTS.ace_pause\", \"$JS.API.CONSUMER.MSG.NEXT.SEAT_NODE_POWER.SEAT_NODE_POWER_worker\", \"$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.ace.pause\", \"$JS.API.STREAM.INFO.KV_mesh-grants_memberships\", \"$JS.FC.KV_mesh-grants_memberships.\u003e\", \"mesh.seat.node-power.event.paused.ace\", \"mesh.seat.node-power.event.slept\"] }\n subscribe: { allow: [\"$SRV.INFO\", \"$SRV.INFO.pause\", \"$SRV.INFO.pause.\u003e\", \"$SRV.PING\", \"$SRV.PING.pause\", \"$SRV.PING.pause.\u003e\", \"$SRV.STATS\", \"$SRV.STATS.pause\", \"$SRV.STATS.pause.\u003e\", \"_INBOX.ace.pause.\u003e\", \"mesh.mod.pause.call.\u003e\", \"mesh.mod.pause.tool.\u003e\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n { user: \"node.ace\", password: \"$2a$11$node.acenode.acenode.afixturefixturefixturefixture00\", permissions: {\n publish: { allow: [\"$JS.ACK.NODES.ace.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_composed-ace.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-config_resolved.*.$KV.mesh-config_resolved.ace.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_candidates.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_placement.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_wants.*.$KV.mesh-delivery-assigner_wants.want.ace.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-builder_builds.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-indexer_candidates.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-indexer_manifests.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-indexer_modules.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-indexer_verdicts.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-host_acts.*.$KV.mesh-host_acts.ace.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-host_data.*.$KV.mesh-host_data.ace.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-host_health.*.$KV.mesh-host_health.ace\", \"$JS.API.CONSUMER.CREATE.KV_mesh-host_packages.*.$KV.mesh-host_packages.ace\", \"$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_provisions.*.$KV.mesh-provisioner_provisions.ace.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.*.node\", \"$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.ace.\u003e\", \"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-ace.*.$O.declaration-files-ace.C.*\", \"$JS.API.CONSUMER.DELETE.KV_composed-ace.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-config_resolved.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-assigner_candidates.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-assigner_placement.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-assigner_wants.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-builder_builds.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-indexer_candidates.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-indexer_manifests.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-indexer_modules.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-indexer_verdicts.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-host_acts.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-host_data.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-host_health.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-host_packages.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_provisions.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_node-engine_have.\u003e\", \"$JS.API.CONSUMER.DELETE.OBJ_declaration-files-ace.*\", \"$JS.API.CONSUMER.INFO.NODES.ace\", \"$JS.API.DIRECT.GET.KV_composed-ace.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-config_resolved.$KV.mesh-config_resolved.ace.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-delivery-assigner_candidates.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-delivery-assigner_placement.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-delivery-assigner_wants.$KV.mesh-delivery-assigner_wants.want.ace.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-delivery-builder_builds.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-delivery-indexer_candidates.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-delivery-indexer_manifests.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-delivery-indexer_modules.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-delivery-indexer_verdicts.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-host_acts.$KV.mesh-host_acts.ace.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-host_data.$KV.mesh-host_data.ace.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-host_health.$KV.mesh-host_health.ace\", \"$JS.API.DIRECT.GET.KV_mesh-host_packages.$KV.mesh-host_packages.ace\", \"$JS.API.DIRECT.GET.KV_mesh-provisioner_provisions.$KV.mesh-provisioner_provisions.ace.\u003e\", \"$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.*.node\", \"$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.ace.\u003e\", \"$JS.API.DIRECT.GET.OBJ_declaration-files-ace.$O.declaration-files-ace.M.*\", \"$JS.API.STREAM.INFO.KV_composed-ace\", \"$JS.API.STREAM.INFO.KV_mesh-config_resolved\", \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_candidates\", \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_placement\", \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_wants\", \"$JS.API.STREAM.INFO.KV_mesh-delivery-builder_builds\", \"$JS.API.STREAM.INFO.KV_mesh-delivery-indexer_candidates\", \"$JS.API.STREAM.INFO.KV_mesh-delivery-indexer_manifests\", \"$JS.API.STREAM.INFO.KV_mesh-delivery-indexer_modules\", \"$JS.API.STREAM.INFO.KV_mesh-delivery-indexer_verdicts\", \"$JS.API.STREAM.INFO.KV_mesh-host_acts\", \"$JS.API.STREAM.INFO.KV_mesh-host_data\", \"$JS.API.STREAM.INFO.KV_mesh-host_health\", \"$JS.API.STREAM.INFO.KV_mesh-host_packages\", \"$JS.API.STREAM.INFO.KV_mesh-provisioner_provisions\", \"$JS.API.STREAM.INFO.KV_node-engine_have\", \"$JS.FC.KV_composed-ace.\u003e\", \"$JS.FC.KV_mesh-config_resolved.\u003e\", \"$JS.FC.KV_mesh-delivery-assigner_candidates.\u003e\", \"$JS.FC.KV_mesh-delivery-assigner_placement.\u003e\", \"$JS.FC.KV_mesh-delivery-assigner_wants.\u003e\", \"$JS.FC.KV_mesh-delivery-builder_builds.\u003e\", \"$JS.FC.KV_mesh-delivery-indexer_candidates.\u003e\", \"$JS.FC.KV_mesh-delivery-indexer_manifests.\u003e\", \"$JS.FC.KV_mesh-delivery-indexer_modules.\u003e\", \"$JS.FC.KV_mesh-delivery-indexer_verdicts.\u003e\", \"$JS.FC.KV_mesh-host_acts.\u003e\", \"$JS.FC.KV_mesh-host_data.\u003e\", \"$JS.FC.KV_mesh-host_health.\u003e\", \"$JS.FC.KV_mesh-host_packages.\u003e\", \"$JS.FC.KV_mesh-provisioner_provisions.\u003e\", \"$JS.FC.KV_node-engine_have.\u003e\", \"$JS.FC.OBJ_declaration-files-ace.\u003e\", \"$KV.composed-ace.\u003e\", \"$KV.mesh-host_acts.ace.\u003e\", \"$KV.mesh-host_data.ace.\u003e\", \"$KV.mesh-host_health.ace\", \"$KV.mesh-host_packages.ace\", \"$KV.node-engine_have.ace.\u003e\", \"$SRV.INFO\", \"$SRV.INFO.\u003e\", \"$SRV.PING\", \"$SRV.PING.*.ace\", \"$SRV.PING.\u003e\", \"mesh.control.ace.\u003e\", \"mesh.seat.node-backup.call.backed-up.ace.node~ace\", \"mesh.seat.node-backup.call.now.ace.node~ace\", \"mesh.seat.node-backup.tool.backed-up.ace\", \"mesh.seat.node-backup.tool.now.ace\", \"mesh.seat.node-engine.event.*.ace\"] }\n subscribe: { allow: [\"$JS.EVENT.ADVISORY.STREAM.CREATED.KV_mesh-delivery-assigner_candidates\", \"$JS.EVENT.ADVISORY.STREAM.CREATED.KV_mesh-delivery-assigner_placement\", \"$SRV.INFO\", \"$SRV.INFO.node-engine\", \"$SRV.INFO.node-engine.\u003e\", \"$SRV.PING\", \"$SRV.PING.node-engine\", \"$SRV.PING.node-engine.\u003e\", \"$SRV.STATS\", \"$SRV.STATS.node-engine\", \"$SRV.STATS.node-engine.\u003e\", \"_DELIVER.ace\", \"_DELIVER.ace.\u003e\", \"_INBOX.node.ace.\u003e\", \"mesh.mod.restic.event.backed-up\", \"mesh.node.ace.ask.access-act\", \"mesh.node.ace.ask.hand-over\", \"mesh.node.ace.ask.judge-directories\", \"mesh.node.ace.ask.move-secret\", \"mesh.node.ace.ask.report\", \"mesh.node.ace.ask.setuid-search\", \"mesh.node.ace.declare\", \"mesh.seat.node-engine.call.*.ace.*\", \"mesh.seat.node-engine.tool.*.ace\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n { user: \"node.novox\", password: \"$2a$11$node.novoxnode.novoxnofixturefixturefixturefixture00\", permissions: {\n publish: { allow: [\"$JS.ACK.NODES.novox.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_composed-novox.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-config_resolved.*.$KV.mesh-config_resolved.novox.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_candidates.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_placement.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-assigner_wants.*.$KV.mesh-delivery-assigner_wants.want.novox.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-builder_builds.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-indexer_candidates.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-indexer_manifests.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-indexer_modules.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-indexer_verdicts.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-host_acts.*.$KV.mesh-host_acts.novox.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-host_data.*.$KV.mesh-host_data.novox.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-host_health.*.$KV.mesh-host_health.novox\", \"$JS.API.CONSUMER.CREATE.KV_mesh-host_packages.*.$KV.mesh-host_packages.novox\", \"$JS.API.CONSUMER.CREATE.KV_mesh-provisioner_provisions.*.$KV.mesh-provisioner_provisions.novox.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.*.node\", \"$JS.API.CONSUMER.CREATE.KV_node-engine_have.*.$KV.node-engine_have.novox.\u003e\", \"$JS.API.CONSUMER.CREATE.OBJ_declaration-files-novox.*.$O.declaration-files-novox.C.*\", \"$JS.API.CONSUMER.DELETE.KV_composed-novox.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-config_resolved.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-assigner_candidates.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-assigner_placement.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-assigner_wants.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-builder_builds.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-indexer_candidates.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-indexer_manifests.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-indexer_modules.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-indexer_verdicts.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-host_acts.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-host_data.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-host_health.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-host_packages.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_mesh-provisioner_provisions.\u003e\", \"$JS.API.CONSUMER.DELETE.KV_node-engine_have.\u003e\", \"$JS.API.CONSUMER.DELETE.OBJ_declaration-files-novox.*\", \"$JS.API.CONSUMER.INFO.NODES.novox\", \"$JS.API.DIRECT.GET.KV_composed-novox.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-config_resolved.$KV.mesh-config_resolved.novox.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-delivery-assigner_candidates.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-delivery-assigner_placement.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-delivery-assigner_wants.$KV.mesh-delivery-assigner_wants.want.novox.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-delivery-builder_builds.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-delivery-indexer_candidates.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-delivery-indexer_manifests.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-delivery-indexer_modules.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-delivery-indexer_verdicts.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-host_acts.$KV.mesh-host_acts.novox.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-host_data.$KV.mesh-host_data.novox.\u003e\", \"$JS.API.DIRECT.GET.KV_mesh-host_health.$KV.mesh-host_health.novox\", \"$JS.API.DIRECT.GET.KV_mesh-host_packages.$KV.mesh-host_packages.novox\", \"$JS.API.DIRECT.GET.KV_mesh-provisioner_provisions.$KV.mesh-provisioner_provisions.novox.\u003e\", \"$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.*.node\", \"$JS.API.DIRECT.GET.KV_node-engine_have.$KV.node-engine_have.novox.\u003e\", \"$JS.API.DIRECT.GET.OBJ_declaration-files-novox.$O.declaration-files-novox.M.*\", \"$JS.API.STREAM.INFO.KV_composed-novox\", \"$JS.API.STREAM.INFO.KV_mesh-config_resolved\", \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_candidates\", \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_placement\", \"$JS.API.STREAM.INFO.KV_mesh-delivery-assigner_wants\", \"$JS.API.STREAM.INFO.KV_mesh-delivery-builder_builds\", \"$JS.API.STREAM.INFO.KV_mesh-delivery-indexer_candidates\", \"$JS.API.STREAM.INFO.KV_mesh-delivery-indexer_manifests\", \"$JS.API.STREAM.INFO.KV_mesh-delivery-indexer_modules\", \"$JS.API.STREAM.INFO.KV_mesh-delivery-indexer_verdicts\", \"$JS.API.STREAM.INFO.KV_mesh-host_acts\", \"$JS.API.STREAM.INFO.KV_mesh-host_data\", \"$JS.API.STREAM.INFO.KV_mesh-host_health\", \"$JS.API.STREAM.INFO.KV_mesh-host_packages\", \"$JS.API.STREAM.INFO.KV_mesh-provisioner_provisions\", \"$JS.API.STREAM.INFO.KV_node-engine_have\", \"$JS.FC.KV_composed-novox.\u003e\", \"$JS.FC.KV_mesh-config_resolved.\u003e\", \"$JS.FC.KV_mesh-delivery-assigner_candidates.\u003e\", \"$JS.FC.KV_mesh-delivery-assigner_placement.\u003e\", \"$JS.FC.KV_mesh-delivery-assigner_wants.\u003e\", \"$JS.FC.KV_mesh-delivery-builder_builds.\u003e\", \"$JS.FC.KV_mesh-delivery-indexer_candidates.\u003e\", \"$JS.FC.KV_mesh-delivery-indexer_manifests.\u003e\", \"$JS.FC.KV_mesh-delivery-indexer_modules.\u003e\", \"$JS.FC.KV_mesh-delivery-indexer_verdicts.\u003e\", \"$JS.FC.KV_mesh-host_acts.\u003e\", \"$JS.FC.KV_mesh-host_data.\u003e\", \"$JS.FC.KV_mesh-host_health.\u003e\", \"$JS.FC.KV_mesh-host_packages.\u003e\", \"$JS.FC.KV_mesh-provisioner_provisions.\u003e\", \"$JS.FC.KV_node-engine_have.\u003e\", \"$JS.FC.OBJ_declaration-files-novox.\u003e\", \"$KV.composed-novox.\u003e\", \"$KV.mesh-host_acts.novox.\u003e\", \"$KV.mesh-host_data.novox.\u003e\", \"$KV.mesh-host_health.novox\", \"$KV.mesh-host_packages.novox\", \"$KV.node-engine_have.novox.\u003e\", \"$SRV.INFO\", \"$SRV.INFO.\u003e\", \"$SRV.PING\", \"$SRV.PING.*.novox\", \"$SRV.PING.\u003e\", \"mesh.control.novox.\u003e\", \"mesh.seat.node-backup.call.backed-up.novox.node~novox\", \"mesh.seat.node-backup.call.now.novox.node~novox\", \"mesh.seat.node-backup.tool.backed-up.novox\", \"mesh.seat.node-backup.tool.now.novox\", \"mesh.seat.node-engine.event.*.novox\"] }\n subscribe: { allow: [\"$JS.EVENT.ADVISORY.STREAM.CREATED.KV_mesh-delivery-assigner_candidates\", \"$JS.EVENT.ADVISORY.STREAM.CREATED.KV_mesh-delivery-assigner_placement\", \"$SRV.INFO\", \"$SRV.INFO.node-engine\", \"$SRV.INFO.node-engine.\u003e\", \"$SRV.PING\", \"$SRV.PING.node-engine\", \"$SRV.PING.node-engine.\u003e\", \"$SRV.STATS\", \"$SRV.STATS.node-engine\", \"$SRV.STATS.node-engine.\u003e\", \"_DELIVER.novox\", \"_DELIVER.novox.\u003e\", \"_INBOX.node.novox.\u003e\", \"mesh.mod.restic.event.backed-up\", \"mesh.node.novox.ask.access-act\", \"mesh.node.novox.ask.hand-over\", \"mesh.node.novox.ask.judge-directories\", \"mesh.node.novox.ask.move-secret\", \"mesh.node.novox.ask.report\", \"mesh.node.novox.ask.setuid-search\", \"mesh.node.novox.declare\", \"mesh.seat.node-engine.call.*.novox.*\", \"mesh.seat.node-engine.tool.*.novox\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n { user: \"novox.mesh-delivery-assigner\", password: \"$2a$11$novox.mesh-delivery-asfixturefixturefixturefixture00\", permissions: {\n publish: { allow: [\"$JS.ACK.EVENTS.novox_mesh-delivery-assigner.\u003e\", \"$JS.API.CONSUMER.CREATE.KV_mesh-grants_memberships.*.$KV.mesh-grants_memberships.novox.mesh-delivery-assigner\", \"$JS.API.CONSUMER.DELETE.KV_mesh-grants_memberships.\u003e\", \"$JS.API.CONSUMER.INFO.EVENTS.novox_mesh-delivery-assigner\", \"$JS.API.CONSUMER.MSG.NEXT.EVENTS.novox_mesh-delivery-assigner\", \"$JS.API.DIRECT.GET.KV_mesh-grants_memberships.$KV.mesh-grants_memberships.novox.mesh-delivery-assigner\", \"$JS.API.STREAM.INFO.KV_mesh-grants_memberships\", \"$JS.FC.KV_mesh-grants_memberships.\u003e\"] }\n subscribe: { allow: [\"$SRV.INFO\", \"$SRV.INFO.mesh-delivery-assigner\", \"$SRV.INFO.mesh-delivery-assigner.\u003e\", \"$SRV.PING\", \"$SRV.PING.mesh-delivery-assigner\", \"$SRV.PING.mesh-delivery-assigner.\u003e\", \"$SRV.STATS\", \"$SRV.STATS.mesh-delivery-assigner\", \"$SRV.STATS.mesh-delivery-assigner.\u003e\", \"_INBOX.novox.mesh-delivery-assigner.\u003e\", \"mesh.mod.mesh-delivery-assigner.call.\u003e\", \"mesh.mod.mesh-delivery-assigner.tool.\u003e\", \"mesh.seat.node-engine.event.manifest-reads-changed.*\", \"mesh.seat.node-power.event.slept\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n \"$SYS\" {\n exports = [\n { stream: \"$SYS.ACCOUNT.MESH.CONNECT\", accounts: [\"MESH\"] }\n { stream: \"$SYS.ACCOUNT.MESH.DISCONNECT\", accounts: [\"MESH\"] }\n { service: \"$SYS.REQ.ACCOUNT.MESH.CONNZ\", accounts: [\"MESH\"] }\n ]\n }\n}\n", + "Missing": null, + "Err": "", + "Memberships": { + "ace.pause": { + "node": "ace", + "module": "pause", + "serves": [ + { + "subject": "mesh.mod.pause.tool.{tool}.ace" + }, + { + "subject": "mesh.mod.pause.tool.{tool}", + "queue": "serve.pause" + } + ], + "emits": "mesh.mod.pause.event.{event}", + "tools": "mesh.mod.pause.tool.tools", + "seat-traffic": { + "publish": [ + "mesh.seat.node-power.event.slept" + ] + } + }, + "novox.mesh-delivery-assigner": { + "node": "novox", + "module": "mesh-delivery-assigner", + "serves": [ + { + "subject": "mesh.mod.mesh-delivery-assigner.tool.{tool}.novox" + }, + { + "subject": "mesh.mod.mesh-delivery-assigner.tool.{tool}", + "queue": "serve.mesh-delivery-assigner" + } + ], + "emits": "mesh.mod.mesh-delivery-assigner.event.{event}", + "tools": "mesh.mod.mesh-delivery-assigner.tool.tools" + } + } } ] diff --git a/modules/mesh-grants/internal/buscompose/writers.go b/modules/mesh-grants/internal/buscompose/writers.go index efb0712b..0fab7628 100644 --- a/modules/mesh-grants/internal/buscompose/writers.go +++ b/modules/mesh-grants/internal/buscompose/writers.go @@ -154,7 +154,7 @@ var WritersTable = []WriterRow{ Shared: "the grants part creates a stream a manifest declares, by its name, and never updates or deletes one (novox/hq ADR 0547 §5; the operator, 2026-10-26 08:54); the bus module's tools account deletes the ended controller's leftover streams, each by its name, and no other (novox/hq ADR 0590 clause 6)"}, {State: "builds and their outcomes", Writer: "the build seat's holder", KeptIn: "its own state", Others: "the builder asks", - Subjects: []string{"mesh.seat.node-build-agent.event.built", "mesh.seat.mesh-build-machine.event.built"}, + Subjects: []string{"mesh.seat.mesh-build-machine.event.built"}, Writes: holdsSeatOf, Shared: "every machine holding the build seat answers the asks it took; each outcome names its ask"}, // What a module keeps and who may read it (novox/hq ADR 0495 clauses 2 and 3, issue 1790): said by the module. diff --git a/modules/mesh-grants/internal/grants/records.go b/modules/mesh-grants/internal/grants/records.go index c93bf116..cc6dfaf7 100644 --- a/modules/mesh-grants/internal/grants/records.go +++ b/modules/mesh-grants/internal/grants/records.go @@ -125,7 +125,10 @@ func declaredFor(m catalogue.Manifest, seats map[string]catalogue.SeatDeclaratio } } if s, isASeat := seats[emitter]; isASeat && catalogue.SeatSays(s.Emits, event) { - watches = append(watches, buscompose.Seat{Name: s.Name, Emits: []string{event}}) + // Its scope, and whether the node-engine holds it, place an event that names its holder's machine (ADR 0618 §4). + compiled, _ := catalogue.SeatNamed(s.Name) + watches = append(watches, buscompose.Seat{Name: s.Name, Scope: s.Scope, Emits: []string{event}, + HeldByTheEngine: compiled.HeldByTheEngine}) continue } } diff --git a/sdk/enginereads/enginereads.go b/sdk/enginereads/enginereads.go new file mode 100644 index 00000000..a50ae917 --- /dev/null +++ b/sdk/enginereads/enginereads.go @@ -0,0 +1,641 @@ +// Package enginereads is how a mesh-delivery part learns what each node's engine reads of a manifest (novox/hq ADR 0618 +// §3, §4): it asks the node seat node-engine's verb manifest-reads, on the part's start, on the seat's event +// manifest-reads-changed for that node and when a person asks again, and keeps each node's last answer in a state of its +// own, one key per node, so a node that is away keeps the statement it last made (ADR 0624 §1). The assigner, the +// builder and the delivery view share it; it is mesh-delivery's, not mesh-sdk's (ADR 0593 §1, ADR 0608 §5). +package enginereads + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + "sync" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/busretry" + + "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" +) + +// The verb and the event as the seat's contract names them (mesh-contracts seats/nodeengine), as a manifest names them, +// and the subject the event is said on with the engine's machine last. +const ( + Verb = nodeengine.VerbManifestReads + Event = nodeengine.EventManifestReadsChanged + Invoke = "seat:" + nodeengine.Seat + "." + Verb + Consumes = nodeengine.Seat + "." + Event +) + +// SubjectPrefix is the event's subject up to the machine, its last token. +var SubjectPrefix = strings.TrimSuffix(nodeengine.SubjectManifestReadsChanged("*"), "*") + +// The reasons an engine says its reads, in the event's why (ADR 0618 §4). +const ( + // ReadsStarted: the engine started, or what it states changed. + ReadsStarted = "started" + // ReadsReturned: the engine's node returned after being away. + ReadsReturned = "returned" +) + +// ContractVersion is the layout version of the verb's answer and the event's body this reader reads. +const ContractVersion = 1 + +// checkNode refuses a node that is no subject token. +func checkNode(node string) error { + if node == "" || strings.ContainsAny(node, ".*> \t\n") { + return fmt.Errorf("%q is no node", node) + } + return nil +} + +// checkReads refuses an empty reader capability. +func checkReads(node string, reads []string) error { + for _, c := range reads { + if strings.TrimSpace(c) == "" { + return fmt.Errorf("the reads of %s name an empty reader capability", node) + } + } + return nil +} + +// DecodeAnswer reads the verb's answer as the contract lays it out, and judges it: the layout's version, the node a +// name, no empty capability, and its stated-at an RFC 3339 time (the contract carries no checks; its reader does). +func DecodeAnswer(raw []byte) (nodeengine.ManifestReadsAnswer, time.Time, error) { + var a nodeengine.ManifestReadsAnswer + if err := json.Unmarshal(raw, &a); err != nil { + return a, time.Time{}, err + } + if a.V != ContractVersion { + return a, time.Time{}, fmt.Errorf("the answer's layout is version %d, not %d", a.V, ContractVersion) + } + if err := checkNode(a.Node); err != nil { + return a, time.Time{}, err + } + if err := checkReads(a.Node, a.ManifestReads); err != nil { + return a, time.Time{}, err + } + at, err := time.Parse(time.RFC3339Nano, a.StatedAt) + if err != nil { + return a, time.Time{}, fmt.Errorf("stated-at %q is no RFC 3339 time", a.StatedAt) + } + return a, at, nil +} + +// DecodeChanged reads the event's body as the contract lays it out, and judges it as DecodeAnswer does, its why one of +// ReadsStarted and ReadsReturned. +func DecodeChanged(raw []byte) (nodeengine.ManifestReadsChangedEvent, error) { + var e nodeengine.ManifestReadsChangedEvent + if err := json.Unmarshal(raw, &e); err != nil { + return e, err + } + if e.V != ContractVersion { + return e, fmt.Errorf("the event's layout is version %d, not %d", e.V, ContractVersion) + } + if err := checkNode(e.Node); err != nil { + return e, err + } + if e.Why != ReadsStarted && e.Why != ReadsReturned { + return e, fmt.Errorf("the reads of %s changed for %q, not %s or %s", e.Node, e.Why, ReadsStarted, ReadsReturned) + } + if err := checkReads(e.Node, e.ManifestReads); err != nil { + return e, err + } + if _, err := time.Parse(time.RFC3339Nano, e.At); err != nil { + return e, fmt.Errorf("at %q is no RFC 3339 time", e.At) + } + return e, nil +} + +// Where a node's statement came from. +const ( + // FromVerb: the node's engine answered the verb. + FromVerb = "verb" + // FromRecord: the engine did not answer it (an engine built before the verb, or nobody answered) and the node's record + // `.node` was read once in its place (the fallback, until every counted engine serves the verb; issue 774). + FromRecord = "record" + // FromNone: neither said anything, so the node is read as stating the baseline (ADR 0614 §4). + FromNone = "none" +) + +// Version is the version of a statement this package writes and the only one it reads. +const Version = 1 + +// Statement is one node's key: what its engine states it reads, when it stated it, when it was asked, where it came +// from, whether the engine answered that ask (it is present), and, while a person's ask again waits, that ask. +type Statement struct { + V int `json:"v"` + Node string `json:"node"` + ManifestReads []string `json:"manifest-reads,omitempty"` + StatedAt time.Time `json:"stated-at,omitzero"` + AskedAt time.Time `json:"asked-at,omitzero"` + From string `json:"from"` + Answered bool `json:"answered"` + Words string `json:"words,omitempty"` + AskAgain *Again `json:"ask-again,omitempty"` +} + +// Again is a person's ask again of one node, written by the tool and answered by the part that keeps the state. +type Again struct { + By string `json:"by"` + Why string `json:"why"` + At time.Time `json:"at"` +} + +// Encode is s as JSON at Version. +func Encode(s Statement) ([]byte, error) { + if s.Node == "" || strings.ContainsAny(s.Node, ".*> ") { + return nil, fmt.Errorf("enginereads: %q is no node", s.Node) + } + s.V = Version + return json.Marshal(s) +} + +// Decode reads a statement; it refuses another version and a value naming no node. +func Decode(raw []byte) (Statement, error) { + var s Statement + if err := json.Unmarshal(raw, &s); err != nil { + return Statement{}, err + } + if s.V != Version { + return Statement{}, fmt.Errorf("enginereads: version %d, not %d", s.V, Version) + } + if s.Node == "" { + return Statement{}, errors.New("enginereads: a statement names no node") + } + return s, nil +} + +// Caller calls a seat's verb on one node: process.Conn.CallOn. +type Caller func(node, key string, body any) (json.RawMessage, error) + +// Ask asks node's engine the verb once and says what it answered. An engine that answered the call with an error is +// present and states nothing through it; one that did not answer is not present. +func Ask(call Caller, node string, now time.Time) Statement { + s := Statement{Node: node, AskedAt: now.UTC(), From: FromNone} + raw, err := call(node, Invoke, map[string]any{}) + var answered *busretry.Answered + switch { + case err == nil: + a, at, derr := DecodeAnswer(raw) + if derr == nil && a.Node != node { + derr = fmt.Errorf("it names the node %s", a.Node) + } + if derr != nil { + s.Answered, s.Words = true, "the engine answered what is no statement: "+derr.Error() + return s + } + s.ManifestReads, s.StatedAt, s.From, s.Answered = slices.Clone(a.ManifestReads), at.UTC(), FromVerb, true + case errors.As(err, &answered): + s.Answered, s.Words = true, "the engine does not serve "+Verb+": "+answered.Words + default: + s.Words = "nobody answered " + Verb + ": " + err.Error() + } + return s +} + +// NodeOf is the node an event of manifest-reads-changed is about, read from its subject's last token and held equal to +// its body's; ok false for a subject of another event. +func NodeOf(subject string, data []byte) (string, bool, error) { + node, ok := strings.CutPrefix(subject, SubjectPrefix) + if !ok || node == "" || strings.Contains(node, ".") { + return "", false, nil + } + e, err := DecodeChanged(data) + if err != nil { + return node, true, err + } + if e.Node != node { + return node, true, fmt.Errorf("the event on %s names the node %s", subject, e.Node) + } + return node, true, nil +} + +// Keeper keeps every node's statement for one part: in memory, and in the part's state when Put is set, so a statement +// outlives the part's restart. Nothing in it waits on a timer: a node is asked when the part says so. +type Keeper struct { + // Call asks the verb; nil asks nothing (every node then reads as the fallback or the baseline). + Call Caller + // Fallback reads a node's record `.node` once where its engine did not answer the verb; nil reads none. + Fallback func(node string) ([]string, bool) + // Put writes a node's key of the part's state, Delete removes it; nil keeps the statements in memory only. + Put func(node string, value []byte) error + Delete func(node string) error + // Changed is told after a node's statement changed. + Changed func() + Logf func(string, ...any) + Now func() time.Time + // Sync asks in the caller's goroutine (tests); otherwise each ask runs on its own. + Sync bool + + mu sync.Mutex + by map[string]Statement + asking map[string]bool +} + +func (k *Keeper) logf(f string, a ...any) { + if k.Logf != nil { + k.Logf(f, a...) + } +} + +func (k *Keeper) now() time.Time { + if k.Now != nil { + return k.Now() + } + return time.Now() +} + +// AskNode asks node's engine, unless an ask of it runs already; why is said in the log. +func (k *Keeper) AskNode(node, why string) { + k.mu.Lock() + if k.asking == nil { + k.asking = map[string]bool{} + } + if k.asking[node] { + k.mu.Unlock() + return + } + k.asking[node] = true + k.mu.Unlock() + if k.Sync { + k.ask(node, why) + return + } + go k.ask(node, why) +} + +func (k *Keeper) ask(node, why string) { + defer func() { + k.mu.Lock() + delete(k.asking, node) + k.mu.Unlock() + }() + s := Statement{Node: node, AskedAt: k.now().UTC(), From: FromNone, Words: "nothing asks the verb"} + if k.Call != nil { + s = Ask(k.Call, node, k.now()) + } + if s.From != FromVerb && k.Fallback != nil { + if reads, ok := k.Fallback(node); ok { + s.ManifestReads, s.From = slices.Clone(reads), FromRecord + k.logf("%s on %s (%s): %s; its record %s.node is read once in its place (until every counted engine serves the verb)", + Verb, node, why, s.Words, node) + } + } + if s.From == FromNone { + k.logf("%s on %s (%s): %s; the node reads as its last statement, or the baseline", Verb, node, why, s.Words) + } + k.take(s) +} + +// take keeps s, unless the node did not answer and an earlier statement stands: an away node keeps its last one (ADR 0624 +// §1), marked as not answering. +func (k *Keeper) take(s Statement) { + k.mu.Lock() + if k.by == nil { + k.by = map[string]Statement{} + } + if old, ok := k.by[s.Node]; ok && !s.Answered && old.From != FromNone { + old.Answered, old.AskedAt, old.Words, old.AskAgain = false, s.AskedAt, s.Words, nil + s = old + } + k.by[s.Node] = s + k.mu.Unlock() + if k.Put != nil { + raw, err := Encode(s) + if err == nil { + err = k.Put(s.Node, raw) + } + if err != nil { + k.logf("the statement of %s could not be kept: %v", s.Node, err) + } + } + if k.Changed != nil { + k.Changed() + } +} + +// Heard takes a change of the part's own state in: at the start, every node's last statement; later, a person's ask again +// (asked now) or a key a person dropped (forgotten). The part's own writes come back here and change nothing. +func (k *Keeper) Heard(node string, raw []byte, deleted bool) { + if deleted { + k.mu.Lock() + _, had := k.by[node] + delete(k.by, node) + k.mu.Unlock() + if had && k.Changed != nil { + k.Changed() + } + return + } + s, err := Decode(raw) + if err != nil || s.Node != node { + k.logf("the key %s of the engine reads is no statement of it, left: %v", node, err) + return + } + if s.AskAgain != nil { + k.AskNode(node, fmt.Sprintf("asked again by %s: %s", s.AskAgain.By, s.AskAgain.Why)) + return + } + k.mu.Lock() + if k.by == nil { + k.by = map[string]Statement{} + } + _, had := k.by[node] + if !had { + k.by[node] = s + } + k.mu.Unlock() + if !had && k.Changed != nil { + k.Changed() + } +} + +// Forget drops a node the part no longer counts (it left the placement): its key too. +func (k *Keeper) Forget(node string) { + k.mu.Lock() + _, had := k.by[node] + delete(k.by, node) + k.mu.Unlock() + if had && k.Delete != nil { + if err := k.Delete(node); err != nil { + k.logf("the statement of %s could not be removed: %v", node, err) + } + } +} + +// Known says whether the part holds a statement of node, from any source. +func (k *Keeper) Known(node string) bool { + k.mu.Lock() + defer k.mu.Unlock() + _, ok := k.by[node] + return ok +} + +// Statements is every node's statement, by node. +func (k *Keeper) Statements() map[string]Statement { + k.mu.Lock() + defer k.mu.Unlock() + out := make(map[string]Statement, len(k.by)) + for n, s := range k.by { + out[n] = s + } + return out +} + +// Reads is, of the nodes given, what each states it reads: its last statement, nil (the baseline) for a node with none. +func (k *Keeper) Reads(nodes []string) map[string][]string { + k.mu.Lock() + defer k.mu.Unlock() + out := make(map[string][]string, len(nodes)) + for _, n := range nodes { + out[n] = slices.Clone(k.by[n].ManifestReads) + } + return out +} + +// Present is, of the nodes given, those whose engine answered when last asked, with what each states. +func (k *Keeper) Present(nodes []string) map[string][]string { + k.mu.Lock() + defer k.mu.Unlock() + out := map[string][]string{} + for _, n := range nodes { + if s, ok := k.by[n]; ok && s.Answered { + out[n] = slices.Clone(s.ManifestReads) + } + } + return out +} + +// State is what the tool needs of the part's state: the runtime's (stdio.KeptState through KeptState), or a test's. +type State interface { + Keys() ([]string, error) + Get(key string) ([]byte, uint64, bool, error) + Put(key string, value []byte) (uint64, error) + Delete(key string) error + // Await waits at most bound for a revision of key after after whose value done says is the answer. + Await(key string, after uint64, done func([]byte) bool, bound time.Duration) ([]byte, error) +} + +// Read lists every node's statement in st, by node name, or one node's; a key that does not read is named with why. +func Read(st State, node string) (map[string]any, error) { + keys := []string{node} + if node == "" { + var err error + if keys, err = st.Keys(); err != nil { + return nil, err + } + } + sort.Strings(keys) + nodes := map[string]any{} + for _, key := range keys { + raw, _, found, err := st.Get(key) + switch { + case err != nil: + return nil, err + case !found: + nodes[key] = map[string]any{"from": FromNone, "words": "no statement is kept: read as the baseline"} + continue + } + s, derr := Decode(raw) + if derr != nil { + nodes[key] = map[string]any{"unread": derr.Error()} + continue + } + nodes[key] = s + } + return map[string]any{"nodes": nodes}, nil +} + +// AskAgainOf writes node's key with a person's ask again and waits at most bound for the part's answer: the next +// statement it writes there. +func AskAgainOf(st State, node, by, why string, now time.Time, bound time.Duration) (Statement, error) { + if node == "" || strings.ContainsAny(node, ".*> ") { + return Statement{}, fmt.Errorf("%q is no node", node) + } + if strings.TrimSpace(why) == "" { + return Statement{}, errors.New("say why it is asked again") + } + s := Statement{Node: node, From: FromNone} + if raw, _, found, err := st.Get(node); err != nil { + return Statement{}, err + } else if found { + if old, err := Decode(raw); err == nil { + s = old + } + } + s.AskAgain = &Again{By: by, Why: why, At: now.UTC()} + raw, err := Encode(s) + if err != nil { + return Statement{}, err + } + rev, err := st.Put(node, raw) + if err != nil { + return Statement{}, err + } + answer, err := st.Await(node, rev, func(v []byte) bool { + got, err := Decode(v) + return err == nil && got.AskAgain == nil + }, bound) + if err != nil { + return Statement{}, fmt.Errorf("the part did not answer the ask again of %s within %s: %w", node, bound, err) + } + return Decode(answer) +} + +// KeptState is a part's state as the runtime keeps it, for the tool. +func KeptState(s stdio.KeptState) State { return keptState{s} } + +type keptState struct{ s stdio.KeptState } + +func (k keptState) Keys() ([]string, error) { return k.s.Keys() } + +func (k keptState) Get(key string) ([]byte, uint64, bool, error) { + e, err := k.s.Get(key) + if err != nil || e == nil { + return nil, 0, false, err + } + return e.Value, e.Revision, true, nil +} + +func (k keptState) Put(key string, value []byte) (uint64, error) { return k.s.Put(key, json.RawMessage(value)) } + +func (k keptState) Delete(key string) error { return k.s.Delete(key) } + +// Await watches key for the answer, and reads it once too in case the watch heard it before it began; a context's bound, +// no timer (ADR 0437 §8). +func (k keptState) Await(key string, after uint64, done func([]byte) bool, bound time.Duration) ([]byte, error) { + heard := make(chan []byte, 1) + if err := k.s.Watch(key, func(c stdio.StateChange) error { + if c.Op == "put" && c.Revision > after && done(c.Value) { + select { + case heard <- c.Value: + default: + } + } + return nil + }); err != nil { + return nil, err + } + if e, err := k.s.Get(key); err == nil && e != nil && e.Revision > after && done(e.Value) { + return e.Value, nil + } + wait, cancel := context.WithTimeout(context.Background(), bound) + defer cancel() + select { + case v := <-heard: + return v, nil + case <-wait.Done(): + return nil, errors.New("it is not running, or cannot read the verb's answer") + } +} + +// AnswerWithin is how long the tool waits for the part to answer an ask again: the verb's own call bound and a margin. +var AnswerWithin = 25 * time.Second + +// Tool is a part's tool over its state of engine reads (tools-first, ADR 0618 §3): read every node's statement or one, +// ask one node's engine again now, and, where drops, drop one node's key (a node the part keeps by mistake). owner names +// the part in the words. +func Tool(name, owner string, st State, drops bool, now func() time.Time) stdio.Tool { + props := map[string]any{ + "node": map[string]any{"type": "string", "description": "one node; every node when left out"}, + "ask_again": map[string]any{"type": "string", "description": "\"true\": ask the node's engine " + Verb + " again now, and answer what it states"}, + "why": map[string]any{"type": "string", "description": "why it is asked again or dropped, in plain words"}, + } + description := "What each node's engine states it reads of a manifest (manifest words, forms and seat names, novox/hq ADR 0618 §3), " + + "as " + owner + " last asked the node-engine seat's verb " + Verb + " and uses it: per node its capabilities, when the engine " + + "stated them and when it was asked, whether it answered (present) and where the statement came from: verb, record (the " + + "engine did not answer and its node record was read once in its place) or none (read as the baseline). A node that is " + + "away keeps its last statement. With node and ask_again, " + owner + " asks that node's engine again and answers its new " + + "statement." + if drops { + props["drop"] = map[string]any{"type": "string", "description": "\"true\": drop the node's key, so it reads as stating the baseline until its engine is asked again"} + description += " With node and drop, the node's key is dropped." + } + return stdio.Tool{Name: name, Description: description, Input: map[string]any{"type": "object", "properties": props}, + Run: func(args map[string]any) (any, error) { + str := func(k string) string { s, _ := args[k].(string); return strings.TrimSpace(s) } + node := str("node") + switch { + case str("ask_again") == "true": + return AskAgainOf(st, node, stdio.CallerOf(args), str("why"), now(), AnswerWithin) + case drops && str("drop") == "true": + if node == "" || str("why") == "" { + return nil, errors.New("drop takes node and why") + } + if err := st.Delete(node); err != nil { + return nil, err + } + return map[string]any{"node": node, "dropped": true, + "next": "the node reads as stating the baseline until its engine is asked again (ask_again, its event, or " + owner + "'s start)"}, nil + } + return Read(st, node) + }} +} + +// How a node answered a survey (Survey). +const ( + // Stated: the engine answered the verb. + Stated = "stated" + // NotServed: the engine answered, but does not serve the verb (an engine built before it). + NotServed = "not served" + // Away: nobody answered within the bound: the node is away, or its engine does not serve the verb and nothing else + // answers for it. + Away = "away" +) + +// SurveyRow is one node's row of a survey: how it answered, its statement when it stated one, and its record's reads +// where it did not (the fallback the assigner reads in its place). +type SurveyRow struct { + Node string `json:"node"` + Answer string `json:"answer"` + ManifestReads []string `json:"manifest-reads,omitempty"` + StatedAt time.Time `json:"stated-at,omitzero"` + Record []string `json:"record,omitempty"` + RecordRead bool `json:"record-read,omitempty"` + Words string `json:"words,omitempty"` +} + +// Survey asks every node's engine the verb at once, each ask bounded by bound, so the whole survey takes at most about +// bound however many nodes there are; a node that did not answer is named with its record's reads where record reads +// one. Sorted by node. +func Survey(call Caller, nodes []string, record func(node string) ([]string, bool), bound time.Duration, now time.Time) []SurveyRow { + rows := make([]SurveyRow, len(nodes)) + var wg sync.WaitGroup + for i, node := range nodes { + wg.Add(1) + go func() { + defer wg.Done() + got := make(chan Statement, 1) + go func() { got <- Ask(call, node, now) }() + var s Statement + wait, cancel := context.WithTimeout(context.Background(), bound) + defer cancel() + select { + case s = <-got: + case <-wait.Done(): + s = Statement{Node: node, Words: fmt.Sprintf("no answer within %s", bound)} + } + row := SurveyRow{Node: node, Words: s.Words} + switch { + case s.From == FromVerb: + row.Answer, row.ManifestReads, row.StatedAt = Stated, s.ManifestReads, s.StatedAt + case s.Answered: + row.Answer = NotServed + default: + row.Answer = Away + } + if row.Answer != Stated && record != nil { + row.Record, row.RecordRead = record(node) + } + rows[i] = row + }() + } + wg.Wait() + sort.Slice(rows, func(i, j int) bool { return rows[i].Node < rows[j].Node }) + return rows +} diff --git a/sdk/forms/forms.go b/sdk/forms/forms.go new file mode 100644 index 00000000..298b27fd --- /dev/null +++ b/sdk/forms/forms.go @@ -0,0 +1,226 @@ +// Package forms holds the artifact reference forms the builder writes a done build key in and the assigner holds a want +// by (novox/hq ADR 0618 §1-§2, ADR 0624): logic only those two modules of mesh-delivery share, so it is mesh-delivery's +// and not mesh-sdk's (ADR 0593 §1, ADR 0608 §5). The build key's fields (delivery.BuilderBuild.Needs, +// delivery.BuiltArtifact.References) are the contract, in mesh-sdk. +package forms + +import ( + "slices" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// artifactStore opens the artifact store's references (delivery.BuiltArtifact). +const artifactStore = "artifact-store://" + +// The artifact reference forms (novox/hq ADR 0605 §9, ADR 0618 §1-§2, ADR 0624): each form a done key's reference may be +// written in, named as the reader capability `form:` a node-engine states when it reads it. The forms written +// before any engine stated what it reads are first forms: every engine reads them, so none needs a statement, and a +// node whose engine states nothing (the baseline, ADR 0614 §4) reads exactly those. A new form is added to +// Current, newest last, in the change that teaches the node-engine to read it. + +// The first forms, by name. +const ( + // FormArtifactStore: artifact-store:////blobs/, or an image's kept artifact-store://…@. + FormArtifactStore = "artifact-store" + // FormRegistry: an image's or an upstream copy's //@. + FormRegistry = "registry" + // FormPackageVersion: a package's @. + FormPackageVersion = "package-version" +) + +// ArtifactForm is one reference form: its name, whether it is a first form, the artifact kinds it names, and how a +// reference is told to be in it. +type ArtifactForm struct { + Name string + First bool + Kinds []string + Is func(reference string) bool +} + +// ArtifactForms are the forms a builder knows, oldest first. +type ArtifactForms []ArtifactForm + +// Current are the forms the builder writes and the assigner knows: the three first forms (delivery.BuiltArtifact's +// reference by kind). +var Current = ArtifactForms{ + {Name: FormArtifactStore, First: true, Is: func(r string) bool { return strings.HasPrefix(r, artifactStore) }, + Kinds: []string{catalogue.ArtifactBundle, catalogue.ArtifactArchive, catalogue.ArtifactImage, catalogue.ArtifactUpstream}}, + {Name: FormRegistry, First: true, Is: func(r string) bool { return !strings.HasPrefix(r, artifactStore) && strings.Contains(r, "@sha256:") }, + Kinds: []string{catalogue.ArtifactImage, catalogue.ArtifactUpstream}}, + {Name: FormPackageVersion, First: true, Is: func(r string) bool { return strings.LastIndex(r, "@") > 0 && !strings.Contains(r, "@sha256:") }, + Kinds: []string{catalogue.ArtifactPackage}}, +} + +// FormNeed is the reader capability a form needs of a node-engine: `form:`. +func FormNeed(form string) string { return catalogue.ReadForm + form } + +// FormOf is the name of the form reference is written in for an artifact of kind, and whether one of fs names it. +func (fs ArtifactForms) FormOf(kind, reference string) (string, bool) { + for i := len(fs) - 1; i >= 0; i-- { // the newest first: a newer form may be a narrower shape of an older one + f := fs[i] + if (kind == "" || slices.Contains(f.Kinds, kind)) && f.Is(reference) { + return f.Name, true + } + } + return "", false +} + +// Reads says whether an engine whose statement is stated reads form: a first form always (the baseline included), any +// other only when stated names `form:`. A form fs does not know is read only when stated. +func (fs ArtifactForms) Reads(stated []string, form string) bool { + for _, f := range fs { + if f.Name == form && f.First { + return true + } + } + return slices.Contains(stated, FormNeed(form)) +} + +// Needs are the reader capabilities a done build's artifacts carry (ADR 0618 §1): `form:` of every form any of +// its references, single or listed, is written in, sorted. Nil when it names none (no artifact, or no reference). +func (fs ArtifactForms) Needs(artifacts []delivery.BuiltArtifact) []string { + set := map[string]bool{} + for _, a := range artifacts { + for _, r := range append([]string{a.Reference}, a.References...) { + if r == "" { + continue + } + if form, ok := fs.FormOf(a.Kind, r); ok { + set[FormNeed(form)] = true + } + } + } + if len(set) == 0 { + return nil + } + out := make([]string, 0, len(set)) + for n := range set { + out = append(out, n) + } + sort.Strings(out) + return out +} + +// Held says whether a build's reference forms hold a node's want (ADR 0624 §2): only when the build names forms and the +// node's engine, its statement stated, reads none of them. A build that names none holds no want. +func (fs ArtifactForms) Held(needs, stated []string) bool { + if len(needs) == 0 { + return false + } + for _, n := range needs { + if form, ok := strings.CutPrefix(n, catalogue.ReadForm); ok && fs.Reads(stated, form) { + return false + } + } + return true +} + +// Writable are the forms of kind every present node's engine reads (ADR 0618 §2: a new form is written only once every +// present engine states it), oldest first, by node its statement. With no node present, the first forms only. +func (fs ArtifactForms) Writable(kind string, present map[string][]string) []string { + var out []string + for _, f := range fs { + if !slices.Contains(f.Kinds, kind) { + continue + } + every := f.First || len(present) > 0 + for _, stated := range present { + every = every && fs.Reads(stated, f.Name) + } + if every { + out = append(out, f.Name) + } + } + return out +} + +// Kept are the forms a build key names an artifact of kind in (ADR 0624 §1, §3), oldest first, so the first is the one +// the single reference holds: the newest form every present engine reads (Writable), then, for each counted node, present +// or away, whose engine reads none of those kept so far, the newest form of kind it reads. A node stating nothing reads +// the first forms. A form leaves only once no counted node reads it alone; a node leaves the count when it is retired. +func (fs ArtifactForms) Kept(kind string, present, counted map[string][]string) []string { + writable := fs.Writable(kind, present) + kept := map[string]bool{} + if len(writable) > 0 { + kept[writable[len(writable)-1]] = true + } + nodes := make([]string, 0, len(counted)) + for node := range counted { + nodes = append(nodes, node) + } + sort.Strings(nodes) + for _, node := range nodes { + stated := counted[node] + var newest string + readsKept := false + for _, f := range fs { + if !slices.Contains(f.Kinds, kind) || !fs.Reads(stated, f.Name) { + continue + } + newest = f.Name + readsKept = readsKept || kept[f.Name] + } + if !readsKept && newest != "" { + kept[newest] = true + } + } + var out []string + for _, f := range fs { + if kept[f.Name] { + out = append(out, f.Name) + } + } + return out +} + +// Choose names a built artifact by the references the build answered in the forms Kept keeps (ADR 0624 §1, §3): the +// oldest kept form in the single Reference, every kept one in References when there are more than one. A reference in a +// form Kept does not keep is left out, unless none of the answered is kept: then the artifact keeps what was answered, +// its first answered in Reference, since a key names where its artifact is (ADR 0498 clause 1). +func (fs ArtifactForms) Choose(a delivery.BuiltArtifact, answered []string, present, counted map[string][]string) delivery.BuiltArtifact { + kept := fs.Kept(a.Kind, present, counted) + type ref struct { + at int + reference string + } + var in []ref + for _, r := range answered { + if r == "" { + continue + } + form, ok := fs.FormOf(a.Kind, r) + if at := slices.Index(kept, form); ok && at >= 0 { + in = append(in, ref{at, r}) + } + } + sort.SliceStable(in, func(i, j int) bool { return in[i].at < in[j].at }) + a.References = nil + switch { + case len(in) > 0: + a.Reference = in[0].reference + if len(in) > 1 { + for _, r := range in { + a.References = append(a.References, r.reference) + } + } + default: + a.Reference = "" + for _, r := range answered { + if r == "" { + continue + } + if a.Reference == "" { + a.Reference = r + } + a.References = append(a.References, r) + } + if len(a.References) < 2 { + a.References = nil + } + } + return a +} diff --git a/sdk/go.mod b/sdk/go.mod index 7e692d40..aa399329 100644 --- a/sdk/go.mod +++ b/sdk/go.mod @@ -3,3 +3,13 @@ module git.novox.be/novox/mesh-delivery/sdk go 1.26.0 require git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 + +require ( + github.com/klauspost/compress v1.20.0 // indirect + github.com/nats-io/nats.go v1.54.0 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect +) + diff --git a/sdk/go.sum b/sdk/go.sum index bede8250..65100b8a 100644 --- a/sdk/go.sum +++ b/sdk/go.sum @@ -1,5 +1,3 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8 h1:XEbIVImp6yGjA91rp0iZBoDdL5KfamjOuDLwWvqh0hc= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261103221822-85ba0a9a70d8/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=