H-A-104: the assigner keeps each node's engine words in its own state engine-reads
The words of each node-engine's manifest-reads-changed are written into the
assigner's state engine-reads, key the node, value {words, at}, before the
event is acknowledged, and read back from the state's watch at the start, so
a restart loses no node's words. The verb manifest-reads is no longer asked:
the keeper's asks, the unanswered-call finding and the node-record fallback go,
with the invoke in module.json. assigner_engine_reads lists the state.
Builds: design 47 H §772; H-A-104 (1), H-A-45
Claude-Session: https://claude.ai/code/session_01JG73KHUnBze48rdua8u2sy
This commit is contained in:
@@ -0,0 +1,153 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
)
|
||||
|
||||
// engineWords is one node's key of the assigner's state engine-reads (design 47 H-A-104): the words its node-engine said it
|
||||
// reads in the last manifest-reads-changed the assigner heard of it, and when the engine said them.
|
||||
type engineWords struct {
|
||||
Words []string `json:"words"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// engineReadsKept is what the assigner holds of its state engine-reads (design 47 H-A-104, §772): written when it hears the
|
||||
// node-engine seat's manifest-reads-changed, and read back from the state's watch at the start, so a restart loses no node's
|
||||
// words; no verb is asked and no node's record is read in its place.
|
||||
type engineReadsKept struct {
|
||||
mu sync.Mutex
|
||||
by map[string]engineWords
|
||||
// put writes one node's key; nil keeps the words in memory only (the state could not be opened).
|
||||
put func(node string, value []byte) error
|
||||
// changed wakes a pass when a node's words changed; nil wakes nothing.
|
||||
changed func()
|
||||
}
|
||||
|
||||
// heard keeps what one manifest-reads-changed says of its node: written into the state first, then held.
|
||||
func (k *engineReadsKept) heard(e nodeengine.ManifestReadsChangedEvent) error {
|
||||
at, err := time.Parse(time.RFC3339Nano, e.At)
|
||||
if err != nil {
|
||||
return fmt.Errorf("at %q is no RFC 3339 time", e.At)
|
||||
}
|
||||
v := engineWords{Words: slices.Clone(e.ManifestReads), At: at}
|
||||
if k.put != nil {
|
||||
raw, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := k.put(e.Node, raw); err != nil {
|
||||
return fmt.Errorf("the words of %s are not written into the state engine-reads: %w", e.Node, err)
|
||||
}
|
||||
}
|
||||
k.hold(e.Node, v, false)
|
||||
return nil
|
||||
}
|
||||
|
||||
// take takes in one key of the state as its watch hands it: at the start every node's words, later the assigner's own writes
|
||||
// (which change nothing) and a key dropped.
|
||||
func (k *engineReadsKept) take(node string, raw []byte, deleted bool) error {
|
||||
if deleted {
|
||||
k.hold(node, engineWords{}, true)
|
||||
return nil
|
||||
}
|
||||
var v engineWords
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return fmt.Errorf("the key %s of the state engine-reads does not read: %w", node, err)
|
||||
}
|
||||
k.hold(node, v, false)
|
||||
return nil
|
||||
}
|
||||
|
||||
// hold sets one node's words, waking a pass when they changed.
|
||||
func (k *engineReadsKept) hold(node string, v engineWords, deleted bool) {
|
||||
k.mu.Lock()
|
||||
old, had := k.by[node]
|
||||
switch {
|
||||
case deleted:
|
||||
delete(k.by, node)
|
||||
default:
|
||||
if k.by == nil {
|
||||
k.by = map[string]engineWords{}
|
||||
}
|
||||
k.by[node] = v
|
||||
}
|
||||
changed := (had && deleted) || (!deleted && (!had || !old.At.Equal(v.At) || !slices.Equal(old.Words, v.Words)))
|
||||
k.mu.Unlock()
|
||||
if changed && k.changed != nil {
|
||||
k.changed()
|
||||
}
|
||||
}
|
||||
|
||||
// words is what node's engine said it reads, and whether the assigner holds its key; nil k holds none.
|
||||
func (k *engineReadsKept) words(node string) ([]string, bool) {
|
||||
if k == nil {
|
||||
return nil, false
|
||||
}
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
v, ok := k.by[node]
|
||||
return slices.Clone(v.Words), ok
|
||||
}
|
||||
|
||||
// nodes is every node whose key the assigner holds, sorted.
|
||||
func (k *engineReadsKept) nodes() []string {
|
||||
if k == nil {
|
||||
return nil
|
||||
}
|
||||
k.mu.Lock()
|
||||
defer k.mu.Unlock()
|
||||
return slices.Sorted(maps.Keys(k.by))
|
||||
}
|
||||
|
||||
// engineReadsTool lists the assigner's state engine-reads: what each node's node-engine said it reads, and when.
|
||||
const engineReadsTool = "assigner_engine_reads"
|
||||
|
||||
// readEngineReads is the state engine-reads, every node's key or node's alone.
|
||||
func readEngineReads(st store, node string) (any, error) {
|
||||
keys := []string{node}
|
||||
if node == "" {
|
||||
var err error
|
||||
if keys, err = st.keys(engineReadsState); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slices.Sort(keys)
|
||||
}
|
||||
out := map[string]engineWords{}
|
||||
for _, key := range keys {
|
||||
raw, _, found, err := st.get(engineReadsState, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !found {
|
||||
if node != "" {
|
||||
return nil, fmt.Errorf("the assigner holds no words of %s: its node-engine said no %s it heard", node, nodeengine.EventManifestReadsChanged)
|
||||
}
|
||||
continue
|
||||
}
|
||||
var v engineWords
|
||||
if err := json.Unmarshal(raw, &v); err != nil {
|
||||
return nil, errors.Join(fmt.Errorf("the key %s does not read", key), err)
|
||||
}
|
||||
out[key] = v
|
||||
}
|
||||
return map[string]any{"nodes": out}, nil
|
||||
}
|
||||
|
||||
// engineReadsTools is the tool over the runtime's state.
|
||||
func engineReadsTools(st store) []stdio.Tool {
|
||||
return []stdio.Tool{{Name: engineReadsTool, Description: "What each node's node-engine says it reads of a manifest, as the " +
|
||||
"assigner keeps it in its state engine-reads (design 47 H-A-104): per node the words of the last manifest-reads-changed " +
|
||||
"it heard and when the engine said them. A node with no key states nothing the assigner heard, so its wants are held " +
|
||||
"awaiting its hand placement (§772). With node, that node's key alone.",
|
||||
Input: schema(map[string]string{"node": "one node; every node when left out"}, nil),
|
||||
Run: func(a map[string]any) (any, error) { return readEngineReads(st, arg(a, "node")) }}}
|
||||
}
|
||||
@@ -119,16 +119,12 @@ func (p *passer) capabilityDigests() []meshwants.AssignmentRevisionCapabilitiesI
|
||||
slices.Sort(known)
|
||||
var out []meshwants.AssignmentRevisionCapabilitiesItem
|
||||
for _, node := range known {
|
||||
read := p.reads != nil && p.reads.Known(node)
|
||||
reads, read := p.reads.words(node)
|
||||
rec, recorded := records[node]
|
||||
if !read && !recorded {
|
||||
continue
|
||||
}
|
||||
var reads []string
|
||||
if read {
|
||||
reads = slices.Clone(p.reads.Reads([]string{node})[node])
|
||||
slices.Sort(reads)
|
||||
}
|
||||
slices.Sort(reads)
|
||||
item := meshwants.AssignmentRevisionCapabilitiesItem{Node: node, Readings: slices.Clone(rec.RunnerCapabilities),
|
||||
Fingerprint: rec.RunnerFingerprint}
|
||||
slices.Sort(item.Readings)
|
||||
|
||||
@@ -759,123 +759,29 @@ func TestWantsAreWrittenAndNoDifferenceIs(t *testing.T) {
|
||||
t.Errorf("app's want = %+v", w)
|
||||
}
|
||||
|
||||
// ADR 0618 §1, §3, §4, ADR 0624 §2: ace's engine answers the node-engine seat's manifest-reads; a build written only in a
|
||||
// form it does not state is held on ace, naming the form, and moves once the engine says manifest-reads-changed on its
|
||||
// subject and states the form; no timer moves it. g14's engine answers nothing and has no record: read as the baseline.
|
||||
stated := catalogue.ManifestReads()
|
||||
var statedMu sync.Mutex
|
||||
// hold, when set, holds the next ask of ace's engine, answered with what it stated when asked, until it is closed;
|
||||
// held is closed once that ask is held (issue 1966 note 170).
|
||||
var hold, held chan struct{}
|
||||
nc, err := nats.Connect(b.srv.ClientURL())
|
||||
// Design 47 H-A-104: ace's engine says manifest-reads-changed; the assigner writes its words and when it said them into
|
||||
// its state engine-reads, asking no verb; g14's engine said nothing, so the state holds no key of it.
|
||||
watch, err := b.buckets[engineReadsBucket()].Watch(b.ctx, "ace", jetstream.UpdatesOnly())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer nc.Close()
|
||||
if _, err := nc.Subscribe("mesh.seat.node-engine.tool.manifest-reads.ace", func(m *nats.Msg) {
|
||||
statedMu.Lock()
|
||||
answer, _ := json.Marshal(nodeengine.ManifestReadsAnswer{V: 1, Node: "ace", ManifestReads: stated, StatedAt: time.Now().Format(time.RFC3339Nano)})
|
||||
release, isHeld := hold, held
|
||||
hold = nil
|
||||
statedMu.Unlock()
|
||||
if release != nil {
|
||||
close(isHeld)
|
||||
<-release
|
||||
}
|
||||
reply, _ := json.Marshal(map[string]any{"result": json.RawMessage(answer), "node": "ace"})
|
||||
_ = m.Respond(reply)
|
||||
}); err != nil {
|
||||
defer watch.Stop()
|
||||
at := time.Now().UTC().Truncate(time.Millisecond)
|
||||
body, _ := json.Marshal(nodeengine.ManifestReadsChangedEvent{V: 1, Node: "ace", ManifestReads: []string{nodeengine.ManifestReadWordWantVersion}, At: at.Format(time.RFC3339Nano), Why: enginereads.ReadsStarted})
|
||||
if _, err := b.js.Publish(b.ctx, enginereads.SubjectPrefix+"ace", body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// The responder's interest is on the bus before the event that asks it is said: the assigner asks a node once per
|
||||
// manifest-reads-changed and never on a timer, so an ask that lands before this subscription reached the server is
|
||||
// answered by nobody and is never asked again (the group check of 616c42434ac2 on a loaded build node).
|
||||
subscribed(t, nc)
|
||||
say := func(why string) {
|
||||
body, _ := json.Marshal(nodeengine.ManifestReadsChangedEvent{V: 1, Node: "ace", ManifestReads: stated, At: time.Now().Format(time.RFC3339Nano), Why: why})
|
||||
if _, err := b.js.Publish(b.ctx, enginereads.SubjectPrefix+"ace", body); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
say(enginereads.ReadsStarted)
|
||||
formsOld, formsNew := fpOf('5'), fpOf('6')
|
||||
formsDigest := b.manifest(t, "forms", `{"module":"forms","version":"1"}`)
|
||||
formsAt := func(fp string) {
|
||||
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: fp, Manifest: formsDigest, Repository: "novox/forms", Commit: strings.Repeat("3", 40), Path: meshcatalogclient.RootPath})
|
||||
b.putModule(t, "forms", mod)
|
||||
}
|
||||
for _, fp := range []string{formsOld, formsNew} {
|
||||
build := builds.BuilderBuild{State: builds.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: formsDigest}
|
||||
if fp == formsNew {
|
||||
build.Needs = []string{"form:new"}
|
||||
}
|
||||
built, err := builds.EncodeBuilderBuild(build)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bkey, _ := builds.BuilderBuildKey("forms", fp)
|
||||
b.putBuild(t, bkey, built)
|
||||
gkey, _ := grantsstate.PartGrantKey("forms", fp)
|
||||
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
|
||||
b.putJSON(t, buckets.Grants, gkey, grant)
|
||||
}
|
||||
runsFormsOld, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: formsOld, Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
|
||||
b.putJSON(t, buckets.Have, "ace.forms", runsFormsOld)
|
||||
b.place(t, "ace", "forms")
|
||||
formsAt(formsOld)
|
||||
b.wantNow(t, "ace", "forms", "the want at what ace runs, a build in the first form", func(w mwc.Want) bool { return w.Fp == formsOld })
|
||||
formsAt(formsNew)
|
||||
statement := func(node string, ok func(enginereads.Statement) bool) enginereads.Statement {
|
||||
var last enginereads.Statement
|
||||
for deadline := time.Now().Add(15 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := b.buckets[engineReadsBucket()].Get(b.ctx, node); err == nil {
|
||||
if last, err = enginereads.Decode(e.Value()); err == nil && ok(last) {
|
||||
return last
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Fatalf("%s's engine reads are %+v", node, last)
|
||||
return last
|
||||
}
|
||||
statement("ace", func(s enginereads.Statement) bool { return s.From == enginereads.FromVerb && s.Answered })
|
||||
if s := statement("g14", func(s enginereads.Statement) bool { return !s.Answered }); s.From != enginereads.FromNone {
|
||||
t.Errorf("g14, whose engine answers nothing and which has no record, reads %+v", s)
|
||||
}
|
||||
if w := b.wantNow(t, "ace", "forms", "the want held on ace, naming the form", func(w mwc.Want) bool {
|
||||
return strings.Contains(w.Words, "form:new")
|
||||
}); w.Fp != formsOld {
|
||||
t.Errorf("forms' want = %+v, want it held at what ace runs under rule 7", w)
|
||||
}
|
||||
// Issue 1966 note 170, 1966/AC170-4: ace's engine says its statement, and while the assigner's ask of it is held,
|
||||
// restates it with the form and says that too; the ask held answers the first statement. The restatement's event,
|
||||
// heard while that ask ran, is asked once more after it, said in the journal, so the state ends at the restatement
|
||||
// and the held want moves.
|
||||
release := make(chan struct{})
|
||||
statedMu.Lock()
|
||||
hold, held = release, make(chan struct{})
|
||||
isHeld := held
|
||||
statedMu.Unlock()
|
||||
say(enginereads.ReadsStarted)
|
||||
select {
|
||||
case <-isHeld:
|
||||
case e := <-watch.Updates():
|
||||
var v engineWords
|
||||
if err := json.Unmarshal(e.Value(), &v); err != nil || !slices.Equal(v.Words, []string{nodeengine.ManifestReadWordWantVersion}) || !v.At.Equal(at) {
|
||||
t.Errorf("the state engine-reads holds %s for ace (%v)", e.Value(), err)
|
||||
}
|
||||
case <-time.After(15 * time.Second):
|
||||
t.Fatal("the assigner did not ask ace's engine on its manifest-reads-changed")
|
||||
t.Fatal("the assigner did not write ace's words into its state engine-reads")
|
||||
}
|
||||
from := b.journal.mark()
|
||||
statedMu.Lock()
|
||||
stated = append(slices.Clone(stated), "form:new")
|
||||
statedMu.Unlock()
|
||||
say(enginereads.ReadsStarted)
|
||||
b.journal.await(t, from, "manifest-reads on ace: an ask runs, so it is asked once more after it", "the restatement heard while ace was asked", 15*time.Second)
|
||||
close(release)
|
||||
if s := statement("ace", func(s enginereads.Statement) bool { return slices.Contains(s.ManifestReads, "form:new") }); s.EventAt.IsZero() {
|
||||
t.Errorf("ace's statement does not show when its event was heard: %+v", s)
|
||||
}
|
||||
b.wantNow(t, "ace", "forms", "the want moved once ace's engine said it reads the form", func(w mwc.Want) bool { return w.Fp == formsNew })
|
||||
// assigner_engine_reads asks ace's engine again through the state: the assigner answers there with what it states now.
|
||||
again, err := enginereads.AskAgainOf(bedState{b, engineReadsBucket()}, "ace", "n1.ada", "a person asks", time.Now(), 15*time.Second)
|
||||
if err != nil || again.From != enginereads.FromVerb || !slices.Contains(again.ManifestReads, "form:new") || again.AskAgain != nil {
|
||||
t.Errorf("the ask again of ace answered %+v, %v", again, err)
|
||||
if _, err := b.buckets[engineReadsBucket()].Get(b.ctx, "g14"); err == nil {
|
||||
t.Error("the state engine-reads holds a key of g14, whose engine said nothing")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -889,33 +795,6 @@ func subscribed(t *testing.T, nc *nats.Conn) {
|
||||
}
|
||||
}
|
||||
|
||||
// bedState is one of the bed's buckets as the engine reads' tool reads it.
|
||||
type bedState struct {
|
||||
b *bed
|
||||
bucket string
|
||||
}
|
||||
|
||||
func (s bedState) Keys() ([]string, error) { return s.b.buckets[s.bucket].Keys(s.b.ctx) }
|
||||
func (s bedState) Get(key string) ([]byte, uint64, bool, error) {
|
||||
e, err := s.b.buckets[s.bucket].Get(s.b.ctx, key)
|
||||
if err != nil {
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
return e.Value(), e.Revision(), true, nil
|
||||
}
|
||||
func (s bedState) Put(key string, value []byte) (uint64, error) {
|
||||
return s.b.buckets[s.bucket].Put(s.b.ctx, key, value)
|
||||
}
|
||||
func (s bedState) Delete(key string) error { return s.b.buckets[s.bucket].Delete(s.b.ctx, key) }
|
||||
func (s bedState) Await(key string, after uint64, done func([]byte) bool, bound time.Duration) ([]byte, error) {
|
||||
for deadline := time.Now().Add(bound); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||
if v, rev, found, _ := s.Get(key); found && rev > after && done(v) {
|
||||
return v, nil
|
||||
}
|
||||
}
|
||||
return nil, fmt.Errorf("no answer at %s within %s", key, bound)
|
||||
}
|
||||
|
||||
// conditionsSaid reads what the assigner said of its conditions, by finding key: each a finding in its own bucket, which
|
||||
// the condition keeper carries (novox/hq ADR 0590 §3, issue 1126), raised while it stands and cleared once deleted, each
|
||||
// with the words its last finding said and the subject the finding names (`<module>[.<fingerprint>]`).
|
||||
|
||||
@@ -16,7 +16,6 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/buscalls"
|
||||
"io"
|
||||
"log"
|
||||
"maps"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
@@ -41,6 +40,7 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
@@ -92,8 +92,8 @@ func ownBuckets() []string {
|
||||
engineReadsBucket(), retiredBucket(), copiesBucket(), writtenBucket(), buckets.Waits(), buckets.PutWaits(), buckets.Inputs(), buckets.CompositionRuns()}
|
||||
}
|
||||
|
||||
// engineReadsState is the assigner's state of what each node's engine states it reads (ADR 0618 §3), one key `<node>`, as
|
||||
// the verb manifest-reads last answered it or the fallback read it: what assigner_engine_reads lists and asks again.
|
||||
// engineReadsState is the assigner's state of what each node's engine says it reads (design 47 H-A-104), one key `<node>`
|
||||
// whose value is the words and the time of the last manifest-reads-changed heard of it: what assigner_engine_reads lists.
|
||||
const engineReadsState = "engine-reads"
|
||||
|
||||
// engineReadsBucket is that state's bucket.
|
||||
@@ -177,8 +177,8 @@ func open(ctx context.Context, js process.BucketOpener, logf func(string, ...any
|
||||
continue
|
||||
}
|
||||
if name == engineReadsBucket() && err != nil {
|
||||
// New with ADR 0618 §3: without it the statements the verb answers are kept in memory only, and
|
||||
// assigner_engine_reads lists nothing.
|
||||
// Without it the words each manifest-reads-changed says are kept in memory only, and assigner_engine_reads
|
||||
// lists nothing.
|
||||
logf("the bucket %s cannot be opened (%v): what each node's engine reads is kept in memory only until it starts again", name, err)
|
||||
continue
|
||||
}
|
||||
@@ -383,16 +383,11 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
}
|
||||
errs := make(chan error, len(watched())+2)
|
||||
settled := make(chan struct{}, len(watched())+2)
|
||||
// What each node's engine reads (ADR 0618 §3, §4): asked through the seat's verb, kept in the state engine-reads, read
|
||||
// from the node's record where the engine does not answer (the fallback, said in the journal).
|
||||
reads := &enginereads.Keeper{Fallback: w.recordReads, Changed: wake, Logf: logger.Printf}
|
||||
if bus != nil {
|
||||
reads.Call = bus.CallKeptOn // a kept call, called again through a transport fault (ADR 0668 §1)
|
||||
reads.Once = bus.CallOn // its fault keeps the fallback at once while the kept call asks on (ADR 0668 §3)
|
||||
}
|
||||
// What each node's engine reads (design 47 H-A-104): the words of its manifest-reads-changed, kept in the state
|
||||
// engine-reads and read back from its watch, so a restart loses none; no verb is asked.
|
||||
reads := &engineReadsKept{changed: wake}
|
||||
if kv := kvs[engineReadsBucket()]; kv != nil {
|
||||
reads.Put = func(node string, v []byte) error { _, err := kv.Put(ctx, node, v); return err }
|
||||
reads.Delete = func(node string) error { return kv.Delete(ctx, node) }
|
||||
reads.put = func(node string, v []byte) error { _, err := kv.Put(ctx, node, v); return err }
|
||||
}
|
||||
watch := func(name string, opts ...jetstream.WatchOpt) error {
|
||||
kw, err := kvs[name].WatchAll(ctx, opts...)
|
||||
@@ -428,7 +423,9 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
value = e.Value()
|
||||
}
|
||||
if name == engineReadsBucket() {
|
||||
reads.Heard(e.Key(), value, value == nil) // it wakes a pass when a statement changed
|
||||
if err := reads.take(e.Key(), value, value == nil); err != nil { // it wakes a pass when the words changed
|
||||
logger.Printf("%v", err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
w.putEntry(name, e.Key(), value, e.Revision(), e.Created())
|
||||
@@ -630,9 +627,8 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
p.graph.revisions.judge()
|
||||
}
|
||||
}
|
||||
// The node-engine seat's manifest-reads-changed, for one node (ADR 0618 §4): that node is asked again, with no timer.
|
||||
// Heard on the assigner's durable consumer on EVENTS, which the grants part makes from the manifest's consumes; where it
|
||||
// is not there yet, a node is asked again only when it comes back online.
|
||||
// The node-engine seat's manifest-reads-changed, for one node (design 47 H-A-104): its words are written into the state
|
||||
// engine-reads. Heard on the assigner's durable consumer on EVENTS, which the grants part makes from the manifest's consumes.
|
||||
if js != nil {
|
||||
// The teller's ask-ended (issue 1966): an ask ended at the channel without an answer is written again; registered
|
||||
// before hearReadsChanged starts the hub.
|
||||
@@ -764,45 +760,45 @@ func sayVia(ctx context.Context, js jetstream.JetStream) func(oc.Condition) erro
|
||||
}
|
||||
}
|
||||
|
||||
// readsChangedHandler is what the assigner does with each event manifest-reads-changed: acknowledges it, keeps what it says
|
||||
// of the node (a statement older than it does not stand) and asks the node again.
|
||||
func readsChangedHandler(reads *enginereads.Keeper, logf func(string, ...any)) func(jetstream.Msg) {
|
||||
// readsChangedHandler is what the assigner does with each event manifest-reads-changed: writes the words it says of its node
|
||||
// into the state engine-reads, then acknowledges it; an event not written is not acknowledged, so the bus delivers it again.
|
||||
func readsChangedHandler(reads *engineReadsKept, logf func(string, ...any)) func(jetstream.Msg) {
|
||||
return func(m jetstream.Msg) {
|
||||
node, ours, err := enginereads.NodeOf(m.Subject(), m.Data())
|
||||
if ours && err == nil {
|
||||
var e nodeengine.ManifestReadsChangedEvent
|
||||
if e, err = enginereads.DecodeChanged(m.Data()); err == nil {
|
||||
if err = reads.heard(e); err != nil {
|
||||
logf("%s of %s: %v", enginereads.Event, node, err)
|
||||
if nerr := m.Nak(); nerr != nil {
|
||||
logf("not acknowledging %s: %v", m.Subject(), nerr)
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if ours && err != nil {
|
||||
logf("%s holds what is not %s, and is not kept: %v", m.Subject(), enginereads.Event, err)
|
||||
}
|
||||
if err := m.Ack(); err != nil {
|
||||
logf("acknowledging %s: %v", m.Subject(), err)
|
||||
}
|
||||
node, ours, err := enginereads.NodeOf(m.Subject(), m.Data())
|
||||
switch {
|
||||
case !ours:
|
||||
case err != nil:
|
||||
logf("%s holds what is not %s, and %s is asked again all the same: %v", m.Subject(), enginereads.Event, node, err)
|
||||
reads.AskNode(node, enginereads.Event+" that does not read")
|
||||
default:
|
||||
// What the event says is kept first: a statement older than it does not stand while the node is asked again.
|
||||
if e, derr := enginereads.DecodeChanged(m.Data()); derr == nil {
|
||||
reads.HeardChanged(e)
|
||||
}
|
||||
reads.AskNode(node, enginereads.Event)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// hearReadsChanged hears the node-engine seat's event manifest-reads-changed on the assigner's one hub of its durable
|
||||
// consumer on EVENTS, and asks the node it names again; the hub also hands the indexer's views to the followers of
|
||||
// consumer on EVENTS, and keeps the words of the node it names; the hub also hands the indexer's views to the followers of
|
||||
// indexerViews, and starts once both are registered.
|
||||
func hearReadsChanged(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, reads *enginereads.Keeper, findings assign.Findings, live func(), logf func(string, ...any)) {
|
||||
func hearReadsChanged(ctx context.Context, js jetstream.JetStream, hub *process.EventHub, reads *engineReadsKept, findings assign.Findings, live func(), logf func(string, ...any)) {
|
||||
account := process.Account{Node: thisNode, Module: assignment.Module()}
|
||||
w := notHeardWriter(findings, logf)
|
||||
handle := readsChangedHandler(reads, logf)
|
||||
err := hub.Handle(enginereads.SubjectPrefix+"*", handle)
|
||||
// At each bind every node the assigner holds a statement of is asked again: an event said before its consumer existed
|
||||
// is not delivered to it (issue 1966 note 155: ace's bootstrapped statement stood; ADR 0668 §4).
|
||||
hub.OnBound(reads.AtBind(func() []string { return slices.Sorted(maps.Keys(reads.Statements())) }))
|
||||
if err == nil {
|
||||
err = hub.Start(ctx, js, account)
|
||||
}
|
||||
if err != nil {
|
||||
w.NotHeard(process.EventsStream, enginereads.Event+" is not heard until the assigner starts again (a node is asked again when it comes back online)", err)
|
||||
w.NotHeard(process.EventsStream, enginereads.Event+" is not heard until the assigner starts again, so no node's new words are kept", err)
|
||||
return
|
||||
}
|
||||
w.Heard(process.EventsStream)
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"maps"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"slices"
|
||||
"sort"
|
||||
@@ -26,8 +25,6 @@ import (
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-sdk/graph"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
@@ -57,8 +54,6 @@ type passer struct {
|
||||
whys map[string]string
|
||||
// findings is the assigner's findings bucket (its said conditions, ADR 0564 §2).
|
||||
findings assign.Findings
|
||||
// calls is the wait of the kept manifest-reads calls (enginereads.ReadsCalls, ADR 0668 §3), made at the first pass.
|
||||
calls *enginereads.ReadsCalls
|
||||
// unplaced are the nodes whose unplaced finding this process wrote.
|
||||
unplaced map[string]bool
|
||||
// answeredAt is, by the id of a backup ask, how many values the have had when its answer was written into the want.
|
||||
@@ -98,10 +93,9 @@ type passer struct {
|
||||
intended map[string]map[string]map[string]bool
|
||||
// laidHere are the proposals whose candidates this process wrote (design 47 G A-317), by id.
|
||||
laidHere map[string]bool
|
||||
// reads keeps what each node's engine states it reads, asked through the node-engine seat's verb (ADR 0618 §3); nil reads
|
||||
// each node's record, as the rules did before the verb. online is, by node, whether the last pass saw it online.
|
||||
reads *enginereads.Keeper
|
||||
online map[string]bool
|
||||
// reads is the assigner's state engine-reads, what each node's engine said it reads (enginewords.go, design 47 H-A-104);
|
||||
// nil reads none.
|
||||
reads *engineReadsKept
|
||||
// outside is the judging of waits on an outside cause (outside.go, design 47 G §706); publishWants says the seat mesh-wants'
|
||||
// wait-held and wait-ended. Nil: no wait is made.
|
||||
outside *outsideActs
|
||||
@@ -126,86 +120,15 @@ func (p *passer) wantsSay(event string, body any, id string) error {
|
||||
return p.publishWants(event, body, id)
|
||||
}
|
||||
|
||||
// engineReads sets what the rules read of each node's engine (ADR 0618 §3, §4): the statement the verb manifest-reads
|
||||
// answered, or its record where the verb was not answered (the fallback) or not asked yet. A node is asked when the
|
||||
// assigner first hears of it, and again when it comes back online, since an engine built before the event says none.
|
||||
// engineReads sets what the rules read of each node's engine: the words its node-engine said in the last
|
||||
// manifest-reads-changed the assigner heard of it, as its state engine-reads keeps them (design 47 H-A-104). A node with no
|
||||
// key states nothing, so its wants are held awaiting its hand placement (§772).
|
||||
func (p *passer) engineReads(n assign.Nodes) {
|
||||
if p.reads == nil {
|
||||
return
|
||||
}
|
||||
known := p.w.knownNodes()
|
||||
first := p.online == nil
|
||||
for _, node := range known {
|
||||
switch {
|
||||
case first:
|
||||
p.reads.AskNode(node, "the assigner started")
|
||||
case !p.reads.Known(node):
|
||||
p.reads.AskNode(node, "first heard of")
|
||||
case !first && n.Online[node] && !p.online[node]:
|
||||
p.reads.AskNode(node, "back online")
|
||||
for _, node := range p.reads.nodes() {
|
||||
if words, ok := p.reads.words(node); ok {
|
||||
n.Reads[node] = words
|
||||
}
|
||||
}
|
||||
p.online = maps.Clone(n.Online)
|
||||
if p.online == nil {
|
||||
p.online = map[string]bool{}
|
||||
}
|
||||
for node, reads := range p.reads.Reads(known) {
|
||||
if p.reads.Known(node) {
|
||||
n.Reads[node] = reads
|
||||
}
|
||||
}
|
||||
p.unansweredReads(known, n.Online)
|
||||
}
|
||||
|
||||
// unansweredReads keeps the wait of every node whose engine has not answered manifest-reads while its machine is heard
|
||||
// (online), through the node-engine seat's client (enginereads.ReadsCalls, shared with the builder and the indexer): it
|
||||
// is asked again once its last ask ended, so the kept call goes on until answered; past
|
||||
// meshconditionsclient.CallUnansweredAfter it is the finding call-unanswered in the assigner's findings, cleared once
|
||||
// answered (novox/hq ADR 0668 §3). Each pass is the event, and the assigner's own presence says heard: no timer, no bound.
|
||||
func (p *passer) unansweredReads(known []string, online map[string]bool) {
|
||||
if p.reads.Call == nil {
|
||||
return
|
||||
}
|
||||
if p.findings.Put == nil || p.findings.Delete == nil {
|
||||
for _, node := range known {
|
||||
if s, ok := p.reads.Statements()[node]; ok && !s.Answered && online[node] && !p.reads.Asking(node) {
|
||||
p.reads.AskNode(node, "unanswered since "+s.UnansweredSince.UTC().Format(time.RFC3339)+", its machine heard")
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
if p.calls == nil || p.calls.Keeper != p.reads {
|
||||
p.calls = &enginereads.ReadsCalls{Keeper: p.reads, Bound: mcc.CallUnansweredAfter, Now: p.clock, Logf: p.logf,
|
||||
Raise: func(_ context.Context, u enginereads.UnansweredCall) error {
|
||||
o := assign.CallUnanswered(enginereads.CallID(u.Node), enginereads.Invoke, u.Node, u.Words, "assigner."+mcc.KindCallUnanswered, u.Since)
|
||||
o.Needs = "assigner_engine_reads with ask_again asks the node again now and shows its statement."
|
||||
return p.findings.Put(o.Finding(u.At))
|
||||
},
|
||||
Clear: func(_ context.Context, node string) error {
|
||||
return p.findings.Delete(mcc.Key(mcc.ScopeCall, enginereads.CallID(node), mcc.KindCallUnanswered))
|
||||
}}
|
||||
if p.findings.Keys != nil {
|
||||
// What a pass before or another process wrote: every such finding of the assigner's is weighed to be cleared.
|
||||
prefix, suffix := mcc.ScopeCall+"."+enginereads.CallID(""), "."+mcc.KindCallUnanswered
|
||||
p.calls.Held = func(context.Context) ([]string, error) {
|
||||
keys, err := p.findings.Keys()
|
||||
var nodes []string
|
||||
for _, key := range keys {
|
||||
if rest, ok := strings.CutPrefix(key, prefix); ok {
|
||||
if node, ok := strings.CutSuffix(rest, suffix); ok {
|
||||
nodes = append(nodes, node)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nodes, err
|
||||
}
|
||||
} else {
|
||||
p.calls.Held = func(context.Context) ([]string, error) { return nil, nil }
|
||||
}
|
||||
}
|
||||
p.calls.Nodes = func() []string { return known }
|
||||
p.calls.Heard = func(_ context.Context, node string) (bool, error) { return online[node], nil }
|
||||
p.calls.Step(context.Background())
|
||||
}
|
||||
|
||||
func (p *passer) clock() time.Time {
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
mpr "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshpipelinerunner"
|
||||
"maps"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"mesh-delivery-assigner/internal/puts"
|
||||
"os"
|
||||
@@ -33,12 +32,10 @@ import (
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-sdk/go/busretry"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mcc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
plc "mesh-delivery-assigner/internal/assignment"
|
||||
@@ -138,130 +135,84 @@ func TestAnAnswerCountsOnlyForTheAskingItAnswered(t *testing.T) {
|
||||
}
|
||||
t.Run("a put of the bus acts on the operator's warrant alone", putOfTheBus)
|
||||
t.Run("a placement's build names the want's version", aPlacementsBuildNamesTheWantsVersion)
|
||||
// ADR 0614 §4: the rules read what each node's engine states it reads from its record in `<node>.node`; a node with no
|
||||
// record, or one that states nothing, is read as stating nothing (the rules then read the baseline).
|
||||
t.Run("the rules read what each node's engine reads", func(t *testing.T) {
|
||||
// Design 47 H-A-104: what each node's engine reads is the assigner's own state engine-reads, key the node, value its
|
||||
// words and when it said them, written when it hears manifest-reads-changed: no verb is asked, no node's record is read
|
||||
// in its place, and a restart reads the state back with nothing lost.
|
||||
t.Run("the rules read the state engine-reads that manifest-reads-changed writes", func(t *testing.T) {
|
||||
w := newWorld()
|
||||
for node, reads := range map[string][]string{"ace": {"word:module", "word:tool-runner"}, "g14": nil} {
|
||||
raw, err := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: node, ManifestReads: reads}})
|
||||
if err != nil {
|
||||
raw, err := nodeengineclient.EncodeHaveNode(nodeengine.HaveByNodeNodeValue{Record: &nodeengine.NodeRecord{Node: "g14", ManifestReads: []string{nodeengine.ManifestReadWordWantVersion}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := nodeengineclient.HaveNodeKey("g14")
|
||||
w.put(buckets.Have, key, raw)
|
||||
state := map[string][]byte{}
|
||||
failing, woken := false, 0
|
||||
kept := &engineReadsKept{changed: func() { woken++ }, put: func(node string, v []byte) error {
|
||||
if failing {
|
||||
return errors.New("the bucket is full")
|
||||
}
|
||||
state[node] = v
|
||||
return nil
|
||||
}}
|
||||
say := func(node, at string, words ...string) *readsMsg {
|
||||
body, _ := json.Marshal(nodeengine.ManifestReadsChangedEvent{V: 1, Node: node, Why: enginereads.ReadsStarted, At: at, ManifestReads: words})
|
||||
m := &readsMsg{subject: enginereads.SubjectPrefix + node, data: body}
|
||||
readsChangedHandler(kept, t.Logf)(m)
|
||||
return m
|
||||
}
|
||||
if m := say("ace", "2026-11-20T03:00:00Z", nodeengine.ManifestReadWordWantVersion, "seat:a"); !m.acked || m.naked {
|
||||
t.Errorf("the event written was acked %v, naked %v", m.acked, m.naked)
|
||||
}
|
||||
var v engineWords
|
||||
if err := json.Unmarshal(state["ace"], &v); err != nil || !slices.Equal(v.Words, []string{nodeengine.ManifestReadWordWantVersion, "seat:a"}) ||
|
||||
!v.At.Equal(time.Date(2026, 11, 20, 3, 0, 0, 0, time.UTC)) || woken != 1 {
|
||||
t.Errorf("the state holds %s (%v), woken %d", state["ace"], err, woken)
|
||||
}
|
||||
p := &passer{w: w, reads: kept}
|
||||
n := w.nodes(nil, nil)
|
||||
p.engineReads(n)
|
||||
if !slices.Equal(n.Reads["ace"], []string{nodeengine.ManifestReadWordWantVersion, "seat:a"}) || n.Reads["g14"] != nil {
|
||||
t.Errorf("the rules read %v: g14's record is no statement of what its engine reads", n.Reads)
|
||||
}
|
||||
// A restart: the state's watch hands every key back; the rules read the same, and the assigner's own write changes nothing.
|
||||
again := &engineReadsKept{}
|
||||
for node, value := range state {
|
||||
if err := again.take(node, value, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key, _ := nodeengineclient.HaveNodeKey(node)
|
||||
w.put(buckets.Have, key, raw)
|
||||
}
|
||||
n := w.nodes(nil, nil)
|
||||
if !slices.Equal(n.Reads["ace"], []string{"word:module", "word:tool-runner"}) || len(n.Reads["g14"]) != 0 || len(n.Reads["novox"]) != 0 {
|
||||
t.Errorf("the nodes' reads are %v", n.Reads)
|
||||
}
|
||||
// ADR 0618 §3: with the verb asked, a node's reads are what its engine answered; where the engine answers it does not
|
||||
// serve the verb, its record is read once in its place (the fallback, issue 774); a node asked again when it comes
|
||||
// back online, since an engine built before the event says none.
|
||||
asked := map[string]int{}
|
||||
call := func(node, key string, _ any) (json.RawMessage, error) {
|
||||
asked[node]++
|
||||
if node == "g14" || key != enginereads.Invoke {
|
||||
return nil, &busretry.Answered{Words: "the seat node-engine serves no verb manifest-reads"}
|
||||
}
|
||||
return json.Marshal(nodeengine.ManifestReadsAnswer{V: 1, Node: node, ManifestReads: []string{"form:new"}, StatedAt: time.Now().Format(time.RFC3339Nano)})
|
||||
}
|
||||
p := &passer{w: w, reads: &enginereads.Keeper{Call: call, Fallback: w.recordReads, Sync: true}}
|
||||
on := false
|
||||
w.heardPresence("ace", !on, time.Now())
|
||||
n = w.nodes(nil, nil)
|
||||
p.engineReads(n)
|
||||
if !slices.Equal(n.Reads["ace"], []string{"form:new"}) || len(n.Reads["g14"]) != 0 || asked["ace"] != 1 {
|
||||
t.Errorf("with the verb the nodes' reads are %v (asked %v)", n.Reads, asked)
|
||||
}
|
||||
if s := p.reads.Statements(); s["ace"].From != enginereads.FromVerb || s["g14"].From != enginereads.FromRecord {
|
||||
t.Errorf("the statements are %+v", s)
|
||||
}
|
||||
p.engineReads(w.nodes(nil, nil))
|
||||
on = true
|
||||
w.heardPresence("ace", !on, time.Now())
|
||||
p.engineReads(w.nodes(nil, nil))
|
||||
if asked["ace"] != 2 {
|
||||
t.Errorf("ace, back online, was asked %d times, want 2", asked["ace"])
|
||||
}
|
||||
// Issue 1975: ace's engine answered its bootstrapped seats, then stated the full list and said so; an answer older
|
||||
// than that event does not stand: the rules read what the event says, and the node is asked until it is as new.
|
||||
// The engine's verb keeps answering the bootstrapped statement (its stated-at is older than the event); the event is
|
||||
// handed to the assigner's own handler (readsChangedHandler, what hearReadsChanged registers) as the bus delivers it.
|
||||
bootstrapped := func(node, _ string, _ any) (json.RawMessage, error) {
|
||||
return json.Marshal(nodeengine.ManifestReadsAnswer{V: 1, Node: node, ManifestReads: []string{"seat:a"}, StatedAt: "2026-11-12T01:14:36.498Z"})
|
||||
}
|
||||
rk := &enginereads.Keeper{Call: bootstrapped, Sync: true}
|
||||
rk.AskNode("ace", "the assigner started")
|
||||
body, _ := json.Marshal(nodeengine.ManifestReadsChangedEvent{V: 1, Node: "ace", Why: enginereads.ReadsStarted, At: "2026-11-12T01:14:39.002Z", ManifestReads: []string{"seat:a", "seat:b"}})
|
||||
readsChangedHandler(rk, t.Logf)(readsMsg{subject: enginereads.SubjectPrefix + "ace", data: body})
|
||||
rp := &passer{w: w, reads: rk, online: map[string]bool{"ace": true}}
|
||||
rn := w.nodes(nil, nil)
|
||||
rp.engineReads(rn)
|
||||
if !slices.Equal(rn.Reads["ace"], []string{"seat:a", "seat:b"}) {
|
||||
t.Errorf("the rules read the statement older than the event heard: %v", rn.Reads["ace"])
|
||||
(&passer{w: w, reads: again}).engineReads(rn)
|
||||
if !slices.Equal(rn.Reads["ace"], n.Reads["ace"]) {
|
||||
t.Errorf("after a restart the rules read %v", rn.Reads)
|
||||
}
|
||||
// ADR 0668 §3 (review round 1): a node whose engine answers nothing while its machine is heard is asked again at each
|
||||
// pass once its ask ended, and past five minutes is the finding call-unanswered, cleared once it answers; none is
|
||||
// written while its machine is away.
|
||||
clock := time.Date(2026, 11, 10, 3, 0, 0, 0, time.UTC)
|
||||
held := map[string][]byte{}
|
||||
answers := false
|
||||
silent := func(node, _ string, _ any) (json.RawMessage, error) {
|
||||
asked[node]++
|
||||
if answers {
|
||||
return json.Marshal(nodeengine.ManifestReadsAnswer{V: 1, Node: node, ManifestReads: []string{"form:new"}, StatedAt: clock.Format(time.RFC3339Nano)})
|
||||
}
|
||||
return nil, context.DeadlineExceeded
|
||||
if err := kept.take("ace", state["ace"], false); err != nil || woken != 1 {
|
||||
t.Errorf("the assigner's own write handed back woke a pass (%d, %v)", woken, err)
|
||||
}
|
||||
q := &passer{w: w, logf: t.Logf, now: func() time.Time { return clock }, findings: assign.Findings{
|
||||
Put: putInto(held),
|
||||
Delete: func(k string) error { delete(held, k); return nil },
|
||||
Keys: func() ([]string, error) { return slices.Collect(maps.Keys(held)), nil }},
|
||||
reads: &enginereads.Keeper{Call: silent, Sync: true, Now: func() time.Time { return clock }}}
|
||||
key := mcc.Key(mcc.ScopeCall, enginereads.CallID("ace"), mcc.KindCallUnanswered)
|
||||
w.heardPresence("g14", true, time.Now()) // g14 away throughout: a machine is present unless the watchman said it away (§389)
|
||||
before := asked["ace"]
|
||||
q.engineReads(w.nodes(nil, nil))
|
||||
clock = clock.Add(6 * time.Minute)
|
||||
off := false
|
||||
w.heardPresence("ace", !off, time.Now())
|
||||
q.engineReads(w.nodes(nil, nil))
|
||||
if len(held) != 0 {
|
||||
t.Errorf("a finding was written while ace is away: %v", slices.Collect(maps.Keys(held)))
|
||||
// An event whose words are not written is not acknowledged, so the bus delivers it again; one that does not read is
|
||||
// acknowledged and not kept.
|
||||
failing = true
|
||||
if m := say("ace", "2026-11-20T03:05:00Z"); m.acked || !m.naked {
|
||||
t.Errorf("an event not written was acked %v, naked %v", m.acked, m.naked)
|
||||
}
|
||||
w.heardPresence("ace", !on, time.Now())
|
||||
q.engineReads(w.nodes(nil, nil))
|
||||
if _, ok := held[key]; !ok || asked["ace"]-before < 3 {
|
||||
t.Errorf("ace unanswered six minutes, heard: findings %v, asked %d times", slices.Collect(maps.Keys(held)), asked["ace"]-before)
|
||||
failing = false
|
||||
if m := say("ace", "yesterday"); !m.acked {
|
||||
t.Error("an event that does not read was not acknowledged")
|
||||
}
|
||||
answers = true
|
||||
q.engineReads(w.nodes(nil, nil))
|
||||
q.engineReads(w.nodes(nil, nil))
|
||||
if _, ok := held[key]; ok {
|
||||
t.Errorf("the finding stays after ace answered")
|
||||
if words, _ := kept.words("ace"); !slices.Contains(words, "seat:a") {
|
||||
t.Errorf("an event not written or not read changed the words to %v", words)
|
||||
}
|
||||
// The contract carries no checks, so its reader judges what the engine says (mesh-contracts seats/nodeengine): an
|
||||
// answer or an event that does not hold is no statement, and an event's node is its subject's last token.
|
||||
// assigner_engine_reads lists the state.
|
||||
got, err := readEngineReads(memStore{engineReadsState: &memKV{values: state}}, "")
|
||||
if listed, _ := json.Marshal(got); err != nil || !strings.Contains(string(listed), `"ace":{"words":["want:version","seat:a"]`) {
|
||||
t.Errorf("assigner_engine_reads answers %s, %v", listed, err)
|
||||
}
|
||||
if _, err := readEngineReads(memStore{engineReadsState: &memKV{values: state}}, "g14"); err == nil {
|
||||
t.Error("assigner_engine_reads answers a node it holds no key of")
|
||||
}
|
||||
// The event's node is its subject's last token, and what does not hold is no event.
|
||||
now := time.Now().Format(time.RFC3339Nano)
|
||||
for _, row := range []struct {
|
||||
name string
|
||||
raw string
|
||||
stays bool
|
||||
}{
|
||||
{"a whole answer", `{"v":1,"node":"ace","manifest-reads":["form:new"],"stated-at":"` + now + `"}`, true},
|
||||
{"an engine stating nothing", `{"v":1,"node":"ace","stated-at":"` + now + `"}`, true},
|
||||
{"another layout", `{"v":2,"node":"ace","stated-at":"` + now + `"}`, false},
|
||||
{"no node", `{"v":1,"node":"","stated-at":"` + now + `"}`, false},
|
||||
{"a node that is no token", `{"v":1,"node":"a.ce","stated-at":"` + now + `"}`, false},
|
||||
{"another node's answer", `{"v":1,"node":"g14","stated-at":"` + now + `"}`, false},
|
||||
{"an empty capability", `{"v":1,"node":"ace","manifest-reads":[""],"stated-at":"` + now + `"}`, false},
|
||||
{"a time that is no time", `{"v":1,"node":"ace","stated-at":"yesterday"}`, false},
|
||||
} {
|
||||
s := enginereads.Ask(func(string, string, any) (json.RawMessage, error) { return json.RawMessage(row.raw), nil }, "ace", time.Now())
|
||||
if (s.From == enginereads.FromVerb) != row.stays || !s.Answered {
|
||||
t.Errorf("answer %s: %+v", row.name, s)
|
||||
}
|
||||
}
|
||||
for _, row := range []struct {
|
||||
name, subject, raw string
|
||||
ours, fails bool
|
||||
@@ -343,9 +294,6 @@ func (b *fakeBus) CallOn(node, key string, body any) (json.RawMessage, error) {
|
||||
return b.Call(node+"/"+key, body)
|
||||
}
|
||||
func (b *fakeBus) CallKept(key string, body any) (json.RawMessage, error) { return b.Call(key, body) }
|
||||
func (b *fakeBus) CallKeptOn(node, key string, body any) (json.RawMessage, error) {
|
||||
return b.CallOn(node, key, body)
|
||||
}
|
||||
func (b *fakeBus) CallFile(key string, body any) (json.RawMessage, error) { return b.Call(key, body) }
|
||||
func (b *fakeBus) Emit(eventType string, body any) error {
|
||||
b.events = append(b.events, fmt.Sprintf("%s %+v", eventType, body))
|
||||
@@ -1032,13 +980,15 @@ func readAskID(t *testing.T, p *passer, key string) string {
|
||||
// readsMsg is a message of the events stream as the assigner's handler reads it.
|
||||
type readsMsg struct {
|
||||
jetstream.Msg
|
||||
subject string
|
||||
data []byte
|
||||
subject string
|
||||
data []byte
|
||||
acked, naked bool
|
||||
}
|
||||
|
||||
func (m readsMsg) Ack() error { return nil }
|
||||
func (m readsMsg) Subject() string { return m.subject }
|
||||
func (m readsMsg) Data() []byte { return m.data }
|
||||
func (m *readsMsg) Ack() error { m.acked = true; return nil }
|
||||
func (m *readsMsg) Nak() error { m.naked = true; return nil }
|
||||
func (m *readsMsg) Subject() string { return m.subject }
|
||||
func (m *readsMsg) Data() []byte { return m.data }
|
||||
|
||||
// A2/AC1 (design 47 H §750, §764; H-A-70): given an entry of the assignment graph whose acting part's new version does not run on
|
||||
// its node, when the assigner passes, then it keeps no wait for the part and says no wait-held: no want is held for an acting part.
|
||||
|
||||
@@ -42,8 +42,6 @@ type caller interface {
|
||||
CallOn(node, key string, body any) (json.RawMessage, error)
|
||||
// CallKept is Call for a kept call, safe to call again (process.Conn.CallKept; ADR 0668).
|
||||
CallKept(key string, body any) (json.RawMessage, error)
|
||||
// CallKeptOn is CallOn for a kept call, safe to call again (process.Conn.CallKeptOn; ADR 0668).
|
||||
CallKeptOn(node, key string, body any) (json.RawMessage, error)
|
||||
// CallFile is Call of a verb marked chunks, its answer read whole (process.Conn.CallFile; design 47 G A-291).
|
||||
CallFile(key string, body any) (json.RawMessage, error)
|
||||
Emit(eventType string, body any) error
|
||||
|
||||
@@ -25,8 +25,6 @@ import (
|
||||
"git.novox.be/novox/mesh-sdk/go/seatlist"
|
||||
"git.novox.be/novox/mesh-sdk/go/stateobjects"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient/enginereads"
|
||||
)
|
||||
|
||||
// The assigner's tools (novox/hq ADR 0546), served through the tool runner from its bundle: a process never serves (ADR
|
||||
@@ -34,9 +32,6 @@ import (
|
||||
// every step as the event put. They are the module's own tools; the seat mesh-module-assigner's verbs are served beside them
|
||||
// (seat_tools.go, ADR 0567 §10).
|
||||
|
||||
// engineReadsTool lists what each node's engine reads, as the assigner uses it, and asks one node's engine again.
|
||||
const engineReadsTool = "assigner_engine_reads"
|
||||
|
||||
// putsState is the assigner's state of puts, as the runtime names a module's own state.
|
||||
const putsState = "puts"
|
||||
|
||||
@@ -415,8 +410,8 @@ func serveTools() error {
|
||||
tools = append(tools, inputsRead(keeper), inputsDrop(keeper, stdio.Ask))
|
||||
tools = append(tools, compositionsTools(stdio.Ask)...) // the composition runs' read and drop, through their verbs (§728)
|
||||
tools = append(tools, viewTools(os.Getenv, keptSource, time.Now)...)
|
||||
// What each node's engine reads, as the assigner uses it (ADR 0618 §3): its own state engine-reads, asked again through it.
|
||||
tools = append(tools, enginereads.Tool(engineReadsTool, "the assigner", enginereads.KeptState(stdio.State(engineReadsState)), false, time.Now))
|
||||
// What each node's engine reads, as the assigner uses it: its own state engine-reads (design 47 H-A-104).
|
||||
tools = append(tools, engineReadsTools(runtimeStore{})...)
|
||||
tools = append(tools, writtenTools(runtimeStore{})...)
|
||||
tools = append(tools, whyTools(runtimeStore{})...)
|
||||
tools = append(tools, askAgainTools(runtimeStore{}, stdio.Ask, time.Now)...)
|
||||
|
||||
@@ -392,11 +392,7 @@ func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[stri
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if isNode {
|
||||
// What the node's engine reads (ADR 0614 §4); a record that states none is read as the baseline.
|
||||
if h, err := nodeengineclient.DecodeHaveNode(value); err == nil && h.Record != nil {
|
||||
n.Reads[node] = h.Record.ManifestReads
|
||||
}
|
||||
if isNode { // what the node's engine reads is the state engine-reads' alone (design 47 H-A-104)
|
||||
continue
|
||||
}
|
||||
if h, err := nodeengineclient.DecodeHave(value); err == nil {
|
||||
@@ -442,13 +438,6 @@ func (w *world) has(bucket, key string) bool {
|
||||
return ok
|
||||
}
|
||||
|
||||
// recordReads is what node's record `<node>.node` states its engine reads (ADR 0614 §4), and whether the assigner holds
|
||||
// a record of it: the fallback where the engine does not answer the verb manifest-reads (enginereads.FromRecord).
|
||||
func (w *world) recordReads(node string) ([]string, bool) {
|
||||
r, ok := w.records()[node]
|
||||
return r.ManifestReads, ok
|
||||
}
|
||||
|
||||
// knownNodes is every node the assigner hears of: placed, with a record, or named by machine-away or machine-back.
|
||||
func (w *world) knownNodes() []string {
|
||||
set := map[string]bool{}
|
||||
|
||||
@@ -149,7 +149,6 @@ func TestManifest(t *testing.T) {
|
||||
"seat:node-build-go.drop", // §720, A-469s: a composition run past its bound drops its ask: the composition run's ask on the Go build seat's queue check (accept.check.<asker>)
|
||||
"seat:node-build-go.output", // A-254: a composition run's answer kept as the build seat's object
|
||||
"seat:node-engine.compose-candidate", // §354, A-226: each machine's composition input
|
||||
"seat:node-engine.manifest-reads", // ADR 0618 §3
|
||||
"seat:operator-channel.asks"}; !slices.Equal(slices.Sorted(slices.Values(m.Invokes)), want) { // ADR 0668 §4: the ended asks at bind
|
||||
t.Errorf("invokes = %v, want %v", m.Invokes, want)
|
||||
}
|
||||
|
||||
@@ -80,7 +80,6 @@
|
||||
"seat:node-build-go.output",
|
||||
"seat:node-build-go.drop",
|
||||
"seat:node-engine.compose-candidate",
|
||||
"seat:node-engine.manifest-reads",
|
||||
"seat:operator-channel.asks",
|
||||
"seat:node-build-go.accept.check"
|
||||
],
|
||||
|
||||
Reference in New Issue
Block a user