Merge branch 'feat/47-G-fix11-b2l' into feat/47-G-join11

# Conflicts:
#	modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended.go
#	modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go
#	modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go
#	modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go
#	modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go
#	modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go
#	modules/mesh-delivery-assigner/internal/assign/wants_test.go
#	modules/mesh-delivery-assigner/manifest_test.go
#	modules/mesh-delivery-assigner/module.json
This commit is contained in:
2026-11-12 10:36:04 +01:00
51 changed files with 2476 additions and 118 deletions
@@ -0,0 +1,152 @@
package main
import (
"context"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/acting"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
)
// The waits for an acting part's new version in the steps the assigner holds (design 47 G §266, §268, §276): an entry of the assignment
// graph whose acting edge names composing or placing waits, its want held, until the part's version runs on the edge's node. The wait is
// kept in the state `waits` under `acting.<node>.<module>.<part>`, said as `wait-held` when it starts and `wait-ended` (ran, released,
// replaced) when it ends; `assigner_waits` reads it and `assigner_release` with how `now` lets the entry move with the part's running
// version. A node away starts no wait: its clock starts when the node is present (§268).
// actingOf is the Module.Acting of module: the held acting edges of its entry on a node, those whose wait was not released left out.
func (p *passer) actingOf(module string, released map[string]bool) func(node string) []assign.ActingEdge {
return func(node string) []assign.ActingEdge {
var out []assign.ActingEdge
for _, e := range p.entryEdges(node, module) {
if acting.Held(e.Step) && !released[acting.Key(node, module, e.Module)+"@"+e.Fp] {
out = append(out, assign.ActingEdge{Node: e.Node, Part: e.Module, Fp: e.Fp, Step: e.Step})
}
}
return out
}
}
// entryEdges are the acting edges of the entry of module on node in the revision the pass computed.
func (p *passer) entryEdges(node, module string) []seat.Edge {
g := p.graph
if g == nil {
return nil
}
for _, ref := range g.computed.Revision.Entries {
if ref.Node == node && ref.Module == module {
var out []seat.Edge
for _, e := range g.computed.Entries[ref.Digest].Edges {
if e.Kind == "acting" {
out = append(out, e)
}
}
return out
}
}
return nil
}
// actingReleased is the waits released by hand and not yet run, by `<key>@<part's fingerprint>`.
func (p *passer) actingReleased() map[string]bool {
out := map[string]bool{}
for key, raw := range p.w.read(buckets.Waits) {
if w, err := acting.Decode(raw); err == nil && w.Released && (acting.Is(key) || acting.IsOrder(key)) {
out[key+"@"+w.Fp] = true
}
}
return out
}
// actingWaits keeps the acting waits after the pass's wants: it starts the wait of each entry held for a part's version, ends the
// wait of one that ran, was replaced or was released, and says each.
func (p *passer) actingWaits(ctx context.Context, n assign.Nodes, wanted map[string]map[string]meshwants.Want) {
g := p.graph
if g == nil || p.kvs[buckets.Waits] == nil {
return
}
now := p.clock()
kept := p.w.read(buckets.Waits)
standing := map[string]bool{}
for _, ref := range g.computed.Revision.Entries {
entry := g.computed.Entries[ref.Digest]
if w, ok := wanted[ref.Module][ref.Node]; ok && w.Fp == entry.Fp {
continue // the entry's version is wanted already: nothing moves, so nothing waits
}
for _, e := range entry.Edges {
if e.Kind != "acting" || !acting.Held(e.Step) {
continue
}
key := acting.Key(ref.Node, ref.Module, e.Module)
if n.Runs(assign.ActingEdge{Node: e.Node, Part: e.Module, Fp: e.Fp, Step: e.Step}) {
continue
}
standing[key] = true
if raw := kept[key]; raw != nil {
if w, err := acting.Decode(raw); err == nil && w.Entry == entry.Fp && w.Fp == e.Fp {
p.actingReleaseSaid(ctx, key, w)
continue
}
}
if !n.Online[ref.Node] {
continue // a node away holds only itself, and its clock has not started (§268)
}
w := acting.Wait{Step: e.Step, Node: ref.Node, Module: ref.Module, Part: e.Module, PartNode: e.Node, Fp: e.Fp, Entry: entry.Fp,
Since: now, ClockFrom: now, Bound: acting.BoundOf(e).String()}
if err := p.wantsSay(seat.EventWaitHeld, acting.Body(w, assignment.Module), key+"."+e.Fp+"."+entry.Fp); err != nil {
p.logf("the wait of %s on %s for %s is not said, and is said at the next pass: %v", ref.Module, ref.Node, e.Module, err)
continue
}
p.put(ctx, buckets.Waits, key, func() ([]byte, error) { return acting.Encode(w) })
}
}
for key, raw := range kept {
if !acting.Is(key) || standing[key] {
continue
}
w, err := acting.Decode(raw)
if err != nil {
continue
}
how := acting.HowRan
if w.Released {
how = acting.HowReleased
} else if e, ok := p.edgeOf(w); !ok || !n.Runs(e) {
how = acting.HowReplaced
}
if !w.EndSaid {
body := acting.Body(w, assignment.Module)
body.How = how
if err := p.wantsSay(seat.EventWaitEnded, body, key+"."+w.Fp+"."+w.Entry+"."+how); err != nil {
p.logf("the end of the wait of %s on %s for %s is not said, and is said at the next pass: %v", w.Module, w.Node, w.Part, err)
continue
}
}
p.remove(ctx, buckets.Waits, key)
}
}
// actingReleaseSaid says wait-ended of a wait released by hand, once, and keeps its record so the entry is not held again.
func (p *passer) actingReleaseSaid(ctx context.Context, key string, w acting.Wait) {
if !w.Released || w.EndSaid {
return
}
body := acting.Body(w, assignment.Module)
body.How = acting.HowReleased
body.By = w.By
if err := p.wantsSay(seat.EventWaitEnded, body, key+"."+w.Fp+"."+w.Entry+"."+acting.HowReleased); err != nil {
p.logf("the release of the wait of %s on %s for %s is not said, and is said at the next pass: %v", w.Module, w.Node, w.Part, err)
return
}
w.EndSaid = true
p.put(ctx, buckets.Waits, key, func() ([]byte, error) { return acting.Encode(w) })
}
// edgeOf is the edge a kept wait stands on, as an acting edge.
func (p *passer) edgeOf(w acting.Wait) (assign.ActingEdge, bool) {
return assign.ActingEdge{Node: w.PartNode, Part: w.Part, Fp: w.Fp, Step: w.Step}, true
}
@@ -0,0 +1,118 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/acting"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/buckets"
)
// Design 47 G §266, §268, §276 (AS9): an entry with an acting edge in composing waits for the part's version to run on the edge's node;
// the wait is kept, said as wait-held with its bound and clock-from, read by assigner_waits, released by assigner_release with how
// `now` and ended when the version runs.
func TestAnEntryWaitsForItsActingPartAndTheWaitIsKeptSaidAndEnded(t *testing.T) {
now := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
p := newPasser(kvSet{buckets.Waits: &memKV{values: map[string][]byte{}}}, now)
var events []string
var bodies []meshwants.Wait
p.publishWants = func(event string, body any, id string) error {
events = append(events, event)
bodies = append(bodies, body.(meshwants.Wait))
return nil
}
entry := meshwants.AssignmentEntry{Node: "ace", Module: "nats", Fp: fpTest, Edges: []meshwants.Edge{
{Kind: "acting", Node: "ace", Module: "mesh-host", Fp: "src2:new", Seat: "node-engine", Step: "compose", Bound: "15m"},
{Kind: "acting", Node: "ace", Module: "indexer", Fp: "src2:idx", Seat: "mesh-catalog", Step: "take-in"}}}
p.graph = &graphKeeper{}
p.graph.computed.Revision.Entries = []meshwants.NodeEntryRef{{Node: "ace", Module: "nats", Digest: "d1"}}
p.graph.computed.Entries = map[string]meshwants.AssignmentEntry{"d1": entry}
n := assign.Nodes{Online: map[string]bool{"ace": true}, Running: map[string]map[string]nodeengine.Running{"ace": {"mesh-host": {Fingerprint: "src2:old"}}}}
wanted := map[string]map[string]mwc.Want{"nats": {"ace": {Fp: "src2:older"}}}
ctx := context.Background()
if got := p.actingOf("nats", nil)("ace"); len(got) != 1 || got[0].Part != "mesh-host" {
t.Fatalf("the assigner holds composing only: %+v", got)
}
p.actingWaits(ctx, n, wanted)
p.actingWaits(ctx, n, wanted) // said once
if len(events) != 1 || events[0] != meshwants.EventWaitHeld {
t.Fatalf("events %v", events)
}
b := bodies[0]
if b.Step != "compose" || b.Part != "mesh-host" || b.Fp != "src2:new" || b.Bound != "15m0s" || b.ClockFrom == nil || !b.ClockFrom.Equal(now) || b.Owner == "" {
t.Errorf("wait-held %+v", b)
}
key := acting.Key("ace", "nats", "mesh-host")
if _, err := acting.Decode(p.w.read(buckets.Waits)[key]); err != nil {
t.Fatalf("the wait is kept: %v", err)
}
// a release by hand lets the entry move, and ends the wait once
w, _ := acting.Decode(p.w.read(buckets.Waits)[key])
w.Released = true
raw, _ := acting.Encode(w)
p.w.put(buckets.Waits, key, raw)
if got := p.actingOf("nats", p.actingReleased())("ace"); len(got) != 0 {
t.Errorf("a released wait still holds: %+v", got)
}
p.actingWaits(ctx, n, wanted)
p.actingWaits(ctx, n, wanted)
if len(events) != 2 || events[1] != meshwants.EventWaitEnded || bodies[1].How != acting.HowReleased {
t.Fatalf("events %v bodies %+v", events, bodies)
}
// the version runs: the record goes, said nothing more
n.Running["ace"]["mesh-host"] = nodeengine.Running{Fingerprint: "src2:new"}
p.actingWaits(ctx, n, wanted)
if len(p.w.read(buckets.Waits)) != 0 || len(events) != 2 {
t.Errorf("kept %v events %v", p.w.read(buckets.Waits), events)
}
}
func TestAnActingWaitOfANodeAwayStartsNoClockAndAnEntryWantedAlreadyWaitsNot(t *testing.T) {
p := newPasser(kvSet{buckets.Waits: &memKV{values: map[string][]byte{}}}, time.Now())
p.publishWants = func(string, any, string) error { t.Fatal("said"); return nil }
p.graph = &graphKeeper{}
p.graph.computed.Revision.Entries = []meshwants.NodeEntryRef{{Node: "ace", Module: "nats", Digest: "d1"}}
p.graph.computed.Entries = map[string]meshwants.AssignmentEntry{"d1": {Node: "ace", Module: "nats", Fp: "src2:a", Edges: []meshwants.Edge{
{Kind: "acting", Node: "ace", Module: "mesh-host", Fp: "src2:new", Step: "compose"}}}}
n := assign.Nodes{Online: map[string]bool{}}
p.actingWaits(context.Background(), n, map[string]map[string]mwc.Want{})
n.Online["ace"] = true
p.actingWaits(context.Background(), n, map[string]map[string]mwc.Want{"nats": {"ace": {Fp: "src2:a"}}})
if len(p.w.read(buckets.Waits)) != 0 || strings.Contains(p.graph.computed.Entries["d1"].Fp, "x") {
t.Errorf("kept %v", p.w.read(buckets.Waits))
}
}
func TestTheToolsReadAndReleaseAnActingWait(t *testing.T) {
var acts []string
s := newPutTools(memStore{waitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveryWaitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveredAgainState: &memKV{values: map[string][]byte{}}}, func(key string, body any) (json.RawMessage, error) {
acts = append(acts, key)
return json.RawMessage(`{"ok":true,"log":"logged"}`), nil
})
w := acting.Wait{Step: "compose", Node: "ace", Module: "nats", Part: "mesh-host", PartNode: "ace", Fp: "src2:new", Entry: "src2:a", Since: time.Now(), ClockFrom: time.Now(), Bound: "30m0s"}
raw, _ := acting.Encode(w)
s.st.put(waitsState, acting.Key("ace", "nats", "mesh-host"), raw)
got, err := s.doWaits(map[string]any{"module": "nats"})
if list := got.(map[string]any)["waits"].([]any); err != nil || len(list) != 1 || list[0].(actingSaid).BoundEnds == "" {
t.Fatalf("assigner_waits = %+v %v", got, err)
}
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "now"}, "x"); err == nil {
t.Error("a release with no why is taken")
}
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "now", "why": "the new engine is known good"}, "shanks.node-tools"); err != nil || len(acts) == 0 {
t.Fatalf("%v %v", err, acts)
}
raw, _, _, _ = s.st.get(waitsState, acting.Key("ace", "nats", "mesh-host"))
if r, _ := acting.Decode(raw); !r.Released || !strings.Contains(r.By, "known good") {
t.Errorf("wait = %+v", r)
}
}
@@ -154,3 +154,17 @@ func hearBuilds(ctx context.Context, js jetstream.JetStream, ends *builtEnds, wa
cc.Stop()
}()
}
// hearTaken registers the hub's taker of the indexer's take-in.taken, each waking the pass (orderwaits.go, §298, A-260); before
// the hub starts.
func hearTaken(hub *process.EventHub, taken *takenSources, wake func(), logf func(string, ...any)) error {
return hub.Handle(takenSubject, func(m jetstream.Msg) {
if err := taken.put(m.Data()); err != nil {
logf("%s is not kept: %v", m.Subject(), err)
}
if err := m.Ack(); err != nil {
logf("acknowledging %s: %v", m.Subject(), err)
}
wake()
})
}
@@ -0,0 +1,82 @@
package main
import (
"context"
"fmt"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assigngraph"
)
// A store filling is said before it is full (design 47 G §181, in the assigner's part): the assignment graph revisions' object store
// says `filling` of the seat mesh-wants when it passes 80 % of its cap, and `full` when the next revision does not fit: the assigner then keeps
// its last revision and writes no new want, and says `room` when the store is back below 80 % and goes on. Nothing is dropped to make room.
// fillOf is the store's fill now: its bytes and cap, when the bucket says them and has a cap.
func (g *graphKeeper) fillOf(ctx context.Context) (used, capacity int64, ok bool) {
f, is := g.objects.(assigngraph.Filler)
if !is {
return 0, 0, false
}
used, capacity, err := f.Fill(ctx)
return used, capacity, err == nil && capacity > 0
}
func (p *passer) sayFill(event, about string, used, capacity int64, refused string) {
at := p.clock().UTC()
body := meshwants.Fill{Store: assigngraph.RevisionsObjects, Bytes: used, Cap: capacity, Refused: refused, At: &at}
if err := p.wantsSay(event, body, event+"-"+about); err != nil {
p.logf("%s of %s is not said: %v", event, assigngraph.RevisionsObjects, err)
}
}
// fillAtStart judges the store once at the process's first pass, when no event is pending: a store past 80 % says filling (full when at
// its cap), so a fill reached while the process was down is said too.
func (p *passer) fillAtStart(ctx context.Context) {
g := p.graph
used, capacity, ok := g.fillOf(ctx)
if !ok {
return
}
switch {
case used >= capacity:
g.full = true
p.sayFill(meshwants.EventFull, "start", used, capacity, "the next revision")
case used*5 >= capacity*4:
p.sayFill(meshwants.EventFilling, "start", used, capacity, "")
}
}
// fullBefore says the next revision does not fit its store, and keeps nothing: it says `full` once, and `room` once the store is below
// 80 % again. It answers true while the store is full.
func (p *passer) fullBefore(ctx context.Context, c assigngraph.Computed) bool {
g := p.graph
used, capacity, ok := g.fillOf(ctx)
if !ok {
return false
}
if g.full && used*5 < capacity*4 {
g.full = false
p.sayFill(meshwants.EventRoom, fmt.Sprint(used), used, capacity, "")
}
if used+assigngraph.NewBytes(ctx, g.objects, c) <= capacity {
g.full = false
return false
}
if !g.full {
g.full = true
p.sayFill(meshwants.EventFull, fmt.Sprint(used), used, capacity, "the next revision")
}
p.logf("the assignment graph revision is not kept: %s is full (%d of %d bytes), the last revision stays and no want is written until there is room", assigngraph.RevisionsObjects, used, capacity)
return true
}
// fillingAfter says `filling` when keeping a revision took the store past 80 % of its cap.
func (p *passer) fillingAfter(ctx context.Context, before int64) {
g := p.graph
used, capacity, ok := g.fillOf(ctx)
if ok && before*5 < capacity*4 && used*5 >= capacity*4 {
p.sayFill(meshwants.EventFilling, fmt.Sprint(used), used, capacity, "")
}
}
@@ -0,0 +1,83 @@
package main
import (
"context"
"testing"
"time"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"mesh-delivery-assigner/internal/assigngraph"
)
// filled is an object bucket with a fill: the bytes it holds count beside what is stored.
type filled struct {
objs map[string][]byte
base int64
cap int64
}
func (f *filled) Put(_ context.Context, name string, data []byte) error {
f.objs[name] = data
return nil
}
func (f *filled) Get(_ context.Context, name string) ([]byte, error) {
if d, ok := f.objs[name]; ok {
return d, nil
}
return nil, assigngraph.ErrNoObject
}
func (f *filled) Names(context.Context) ([]string, error) {
out := []string{}
for n := range f.objs {
out = append(out, n)
}
return out, nil
}
func (f *filled) Fill(context.Context) (int64, int64, error) {
used := f.base
for _, d := range f.objs {
used += int64(len(d))
}
return used, f.cap, nil
}
// Design 47 G §181 (AS13): the assignment graph revisions' store says filling past 80 % of its cap, refuses a revision that does not fit
// with `full` (the last revision stays, no want is written), and says `room` below 80 % again; at the start a store already past 80 % is said.
func TestTheAssignmentGraphStoreSaysFillingFullAndRoom(t *testing.T) {
ctx := context.Background()
p := newPasser(kvSet{}, time.Now())
var events []string
p.publishWants = func(event string, body any, id string) error {
if _, ok := body.(meshwants.Fill); ok {
events = append(events, event)
}
return nil
}
o := &filled{objs: map[string][]byte{}, base: 500, cap: 1000}
p.graph = &graphKeeper{objects: o, halves: func() map[string]mcc.Half { return nil }}
p.keepGraph(ctx) // the first pass judges at the start: 500 of 1000 is below 80 %
if len(events) != 0 {
t.Fatalf("events %v", events)
}
o.base = 700
p.graph.started = false
p.keepGraph(ctx)
if len(events) != 1 || events[0] != meshwants.EventFilling {
t.Fatalf("at the start past 80 %%: %v", events)
}
c := assigngraph.Computed{Revision: meshwants.AssignmentRevision{Entries: []meshwants.NodeEntryRef{}}}
o.base = 990
if !p.fullBefore(ctx, c) || !p.graph.full || events[len(events)-1] != meshwants.EventFull {
t.Fatalf("a revision that does not fit: events %v", events)
}
p.fullBefore(ctx, c)
if n := len(events); n != 2 {
t.Errorf("full is said once: %v", events)
}
o.base = 100
if p.fullBefore(ctx, c) || p.graph.full || events[len(events)-1] != meshwants.EventRoom {
t.Errorf("room: events %v", events)
}
}
@@ -34,8 +34,14 @@ type graphKeeper struct {
source int64
ok bool
said string
// cycled is the cycles last said (assigngraph.Cycles).
cycled string
// full says the next revision did not fit the store, so no want is written (graphfill.go); started says the first pass judged the fill.
full, started bool
// pins removes a revision that ceased to be current once nothing pins it (pins.go); nil keeps every revision.
pins *pinKeeper
// computed is the revision computed by the last pass, with its entries: the edges the wants wait on (actingwaits.go).
computed assigngraph.Computed
}
// keepGraph computes the revision of this pass and keeps it when the pass's inputs differ from the last computed.
@@ -45,15 +51,30 @@ func (p *passer) keepGraph(ctx context.Context) {
return
}
c := assigngraph.Compute(p.graphInputs())
g.computed = c
for _, u := range c.Uncomputed {
if g.said != u {
g.said = u
p.logf("the assignment graph leaves out %s", u)
}
}
if cycles := strings.Join(c.Cycles, "; "); cycles != g.cycled {
g.cycled = cycles
if cycles != "" {
p.logf("the assignment graph holds a cycle among the waits within one step, and its entries wait for each other: %s", cycles)
}
}
if !g.started {
g.started = true
p.fillAtStart(ctx)
}
if g.has && g.ok && assigngraph.Same(g.last, c.Revision) {
return
}
used, _, _ := g.fillOf(ctx)
if p.fullBefore(ctx, c) {
return
}
rev, fresh, err := assigngraph.Keep(ctx, g.objects, c, meshwants.EventAssignmentRevision, p.clock())
if err != nil {
p.logf("the assignment graph revision is not kept, so no want is said as wanted until it is: %v", err)
@@ -69,6 +90,7 @@ func (p *passer) keepGraph(ctx context.Context) {
if !fresh {
return
}
p.fillingAfter(ctx, used)
if g.pins != nil {
g.pins.judge() // the revision before ceased to be current
}
@@ -82,6 +104,11 @@ func (p *passer) keepGraph(ctx context.Context) {
// node's capability reading and the first branch of each registration.
func (p *passer) graphInputs() assigngraph.Inputs {
in := assigngraph.Inputs{Halves: p.graph.halves(), Placements: p.w.assignments(), Capabilities: p.capabilityDigests()}
// The seat definitions read through the seat mesh-catalog's verb seat-definitions say which seat delivers a requirement
// (design 47 G §708).
if r := p.seats.Reader(); r != nil {
in.Definitions = r.Definitions()
}
// With no pipeline ends kept, the wants are the trunk-only form (pass.go): a placement naming no branch runs the trunk's.
first := p.w.trunkName()
in.FirstBranch = func(string) string { return first }
@@ -138,6 +138,8 @@ func fromEntry(e meshcatalog.CatalogEntry) (meshcatalog.ModulesByModuleValue, gr
return v, n, &parsed{module: e.Module, err: "it does not decode: " + err.Error()}
}
n.Serves = graph.ServesOf(man)
// The entry's requires name the optional ones too; the node means those required, not optionally (A-229).
n.Requires = slices.DeleteFunc(slices.Clone(n.Requires), man.IsOptional)
return v, n, &parsed{module: e.Module, manifest: man}
}
@@ -29,6 +29,7 @@ import (
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/process"
@@ -576,6 +577,9 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
return p.keepCandidate(ctx, iv.call, ev)
}
iv.candidates.Store(&keep)
// A held revert's running entry and its want's, asked of the seat mesh-catalog's verb entry (design 47 G §710, A-217).
p.entries = meshcatalogclient.NewEntries(iv.h.call)
p.entries.Changed = func(meshcatalogclient.EntryRow) { wake() }
// The pins of the revisions (pins.go, design 47 G §579): the gate's pinned, freed, heads-replaced and pins-stated
// on the hub, registered before it starts.
if kv := kvs[pinsBucket]; kv != nil {
@@ -618,6 +622,9 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
}
hearDelivery(ctx, js, w, wake, findings, logger.Printf) // version-released, version-delivered, machine-away and machine-back
hearBuilds(ctx, js, &w.ends, wake, findings, logger.Printf)
if err := hearTaken(hub, &w.taken, wake, logger.Printf); err != nil {
logger.Printf("the indexer's take-ins are not heard, so an entry held by an order edge waits until the assigner starts again: %v", err)
}
hearReadsChanged(ctx, js, hub, reads, findings, gate.EventsLive, logger.Printf)
hearRunning(ctx, js, w, wake, findings, logger.Printf)
}
@@ -99,3 +99,20 @@ func (b *busObjects) Delete(ctx context.Context, name string) error {
}
return nil
}
// Fill is the bytes the bucket holds and its cap (assigngraph.Filler).
func (b *busObjects) Fill(ctx context.Context) (int64, int64, error) {
o, err := b.open(ctx)
if err != nil {
return 0, 0, err
}
st, err := o.Status(ctx)
if err != nil {
return 0, 0, err
}
info, err := b.js.Stream(ctx, "OBJ_"+st.Bucket())
if err != nil {
return int64(st.Size()), 0, err
}
return int64(st.Size()), info.CachedInfo().Config.MaxBytes, nil
}
@@ -0,0 +1,296 @@
package main
import (
"context"
"encoding/json"
"fmt"
"strconv"
"sync"
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/process"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"mesh-delivery-assigner/internal/acting"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
)
// The three order edges of a delivery that crosses a form change (design 47 G §298, A-245, A-258, A-259, A-260). Each is an acting wait of the
// assigner, kept in the state `waits` under `order.<node>.<module>.<part>` beside the acting waits, in the step placing, said as wait-held
// when a want is held for it and wait-ended (ran, released, replaced) when it moves; assigner_waits reads them and assigner_release with
// how `now` releases them, as it does an acting wait. The holders of the seats are read from the seat holders the assignments compose
// (assignment.SeatHoldersOf), never named:
// - the node-backup holder's and the bus module's (the holder of the seat mesh-broker) new versions run before a module whose kept data's
// form changes is moved, since the restore point needs both (OrderBackup);
// - at a revert a node's node-engine goes back last on that node, after every module it restores there runs the reverted build, which it
// starts only once it has said `restored` (OrderEngineLast);
// - the reverted assigner and gate are wanted only once the indexer's take-in.taken of the trunk branch names a source revision at least
// the trunk's head, the highest `src` of the trunk branch's module values (OrderTakenIn, A-260).
// takenSubject is the indexer's event take-in.taken on its own stream, as the manifest's consumes names it.
var takenSubject = process.ModuleStreamSubject(buckets.IndexerModule, "take-in.taken")
// takenSources are the newest source revisions a take-in.taken named, by watched branch, as heard (hearBuildsAndReleases).
type takenSources struct {
mu sync.Mutex
own map[string]int64
}
// put takes one take-in.taken in. A hand drop, and a taken with no source revision (one said before G), take nothing in.
func (t *takenSources) put(data []byte) error {
var e meshcatalog.TakeInTaken
if err := json.Unmarshal(data, &e); err != nil {
return fmt.Errorf("a take-in.taken that is none: %w", err)
}
if e.Dropped || e.Source == 0 || e.Branch == "" {
return nil
}
t.mu.Lock()
defer t.mu.Unlock()
if t.own == nil {
t.own = map[string]int64{}
}
t.own[e.Branch] = max(t.own[e.Branch], e.Source)
return nil
}
// of is the newest source revision of branch taken in; 0 before one is heard.
func (t *takenSources) of(branch string) int64 {
t.mu.Lock()
defer t.mu.Unlock()
return t.own[branch]
}
// orderEdge is one order edge held on module's want on node: part's version fp must run on partNode (OrderBackup, OrderEngineLast), or the
// trunk's source revision source be taken in (OrderTakenIn).
type orderEdge struct {
node, module, part, partNode, fp, entry, order string
source int64
}
func (e orderEdge) key() string { return acting.OrderKey(e.node, e.module, e.part) }
func (e orderEdge) acting() assign.ActingEdge {
return assign.ActingEdge{Node: e.partNode, Part: e.part, Fp: e.fp, Step: seatdef.StepPlace}
}
// orderRuns says an order edge is met: its part runs the version, or the source revision is taken in.
func (p *passer) orderRuns(n assign.Nodes, e orderEdge) bool {
if e.order == acting.OrderTakenIn {
return p.w.taken.of(p.w.trunkName()) >= e.source
}
return n.Runs(e.acting())
}
// seatHolders are the holders of each seat as the assignments compose them, by seat name.
func (p *passer) holdersBySeat() map[string][]mma.SeatHoldersHoldersItem {
out := map[string][]mma.SeatHoldersHoldersItem{}
for _, sh := range assignment.SeatHoldersOf(p.w.assignments()) {
out[sh.Seat] = append(out[sh.Seat], sh.Holders...)
}
return out
}
// orderEdgesOf computes §298's edges on the wants of module m on each of its nodes, into p.orders. wanted is m's wants by node; trunk the
// trunk's module values. A node that has no running build heard, runs m at its trunk version already, or is wanted that version already
// has none: nothing moves. It answers, by node, whether an edge still stands that cannot be recorded as a wait because the catalog entry
// it reads is not answered yet (asked, and its answer wakes a pass): the want is held meanwhile, never written across a form change unread.
func (p *passer) orderEdgesOf(ctx context.Context, m assign.Module, wanted map[string]mwc.Want, trunk map[string]meshcatalog.ModulesByModuleValue) map[string]bool {
unread := map[string]bool{}
p.orderVisited[m.Name] = true
if m.FP == "" || p.graph == nil || p.entries == nil {
return unread
}
var holders map[string][]mma.SeatHoldersHoldersItem
for _, node := range m.Nodes {
if w, ok := wanted[node]; ok && !w.Removed && w.Fp == m.FP {
continue // the version is wanted already: nothing moves, so nothing waits
}
run, heard := p.w.runningOf(node, m.Name)
if !heard || run.Stopped || run.Fingerprint == "" || run.Fingerprint == m.FP {
continue
}
if holders == nil {
holders = p.holdersBySeat()
}
held := func(seatName string) bool {
for _, h := range holders[seatName] {
if h.Module == m.Name && h.Node == node {
return true
}
}
return false
}
add := func(e orderEdge) {
e.node, e.module, e.entry = node, m.Name, m.FP
p.orders[e.key()] = e
}
// 1. a kept data's form changes: the restore point needs the new node-backup holder and bus module.
changes, known := p.formChange(ctx, m.Name, run.Fingerprint, m.FP)
if !known {
unread[node] = true
}
if len(changes) > 0 {
for seatName, onNode := range map[string]bool{catalogue.BackupSeat: true, meshbroker.Seat: false} {
for _, h := range holders[seatName] {
if h.Module == m.Name || (onNode && h.Node != node) || trunk[h.Module].Fp2 == "" {
continue
}
add(orderEdge{part: h.Module, partNode: h.Node, fp: trunk[h.Module].Fp2, order: acting.OrderBackup})
}
}
}
if !held(nodeengine.Seat) && !held(mma.Seat) && !held(meshdelivery.Seat) {
continue
}
reverted, err := assigngraph.Reverted(ctx, p.graph.objects, node, m.Name, run.Fingerprint, m.FP)
if err != nil {
p.logf("whether the want of %s on %s at %s is a revert is not read: %v", m.Name, node, m.FP, err)
continue
}
if !reverted {
continue
}
// 2. at a revert the node-engine goes back last on its node: after every module that is set back there across a form change runs.
if held(nodeengine.Seat) {
for module, byNode := range p.w.wantsOf() {
// What it is set back to is its trunk version, which the rules write as its want in this pass.
target := trunk[module].Fp2
if w, ok := byNode[node]; module == m.Name || !ok || w.Removed || target == "" {
continue
}
xrun, heard := p.w.runningOf(node, module)
if !heard || xrun.Stopped || xrun.Fingerprint == "" || xrun.Fingerprint == target {
continue
}
if back, err := assigngraph.Reverted(ctx, p.graph.objects, node, module, xrun.Fingerprint, target); err != nil || !back {
continue
}
xchanges, known := p.formChange(ctx, module, xrun.Fingerprint, target)
if !known {
unread[node] = true
}
if len(xchanges) > 0 {
add(orderEdge{part: module, partNode: node, fp: target, order: acting.OrderEngineLast})
}
}
}
// 3. the reverted assigner and gate wait for the indexer to take in the trunk's head.
if held(mma.Seat) || held(meshdelivery.Seat) {
var head int64
for _, v := range p.w.trunkOf(p.w.trunkName()) {
head = max(head, v.Src)
}
if head > 0 {
add(orderEdge{part: buckets.IndexerModule, fp: strconv.FormatInt(head, 10), source: head, order: acting.OrderTakenIn})
}
}
}
return unread
}
// orderActing is the order edges of module on node that still hold its want, as acting edges: those not met, and not released by hand. A
// node whose catalog entry is not read yet holds its want too, on a stand-in edge no part runs.
func (p *passer) orderActing(module string, released map[string]bool, unread map[string]bool, n func() assign.Nodes) func(node string) []assign.ActingEdge {
return func(node string) []assign.ActingEdge {
var out []assign.ActingEdge
nodes := n()
for _, e := range p.orders {
if e.module == module && e.node == node && !released[e.key()+"@"+e.fp] && !p.orderRuns(nodes, e) {
out = append(out, assign.ActingEdge{Node: e.partNode, Part: e.part, Fp: e.fp, Step: seatdef.StepPlace})
}
}
if unread[node] {
out = append(out, assign.ActingEdge{Node: node, Part: "the catalog entry", Step: seatdef.StepPlace})
}
return out
}
}
// orderWaits keeps the order waits after the pass's wants, as actingWaits keeps the acting waits: it starts the wait of each edge not met,
// ends the wait of one that ran, was replaced or was released, and says each.
func (p *passer) orderWaits(ctx context.Context, n assign.Nodes, wanted map[string]map[string]mwc.Want) {
if p.kvs[buckets.Waits] == nil {
return
}
now := p.clock()
kept := p.w.read(buckets.Waits)
trunk := p.w.trunk()
if p.orderSaid == nil {
p.orderSaid = map[string]bool{}
}
standing := map[string]bool{}
for key, e := range p.orders {
if w, ok := wanted[e.module][e.node]; ok && !w.Removed && w.Fp == e.entry {
continue
}
if p.orderRuns(n, e) {
continue
}
standing[key] = true
if raw := kept[key]; raw != nil {
if w, err := acting.Decode(raw); err == nil && w.Entry == e.entry && w.Fp == e.fp {
if !w.Released && !p.orderSaid[key] {
// A wait still held after a restart is said again, as wait-held (A-260).
if err := p.wantsSay(seat.EventWaitHeld, acting.Body(w, assignment.Module), key+"."+e.fp+"."+e.entry+".again"); err != nil {
p.logf("the wait of %s on %s for %s is not said again, and is said at the next pass: %v", e.module, e.node, e.part, err)
} else {
p.orderSaid[key] = true
}
}
p.actingReleaseSaid(ctx, key, w)
continue
}
}
if !n.Online[e.node] {
continue // a node away holds only itself, and its clock has not started (§268)
}
w := acting.Wait{Step: seatdef.StepPlace, Node: e.node, Module: e.module, Part: e.part, PartNode: e.partNode, Fp: e.fp, Entry: e.entry, Order: e.order,
Since: now, ClockFrom: now, Bound: acting.BoundOf(seat.Edge{Step: seatdef.StepPlace}).String()}
if err := p.wantsSay(seat.EventWaitHeld, acting.Body(w, assignment.Module), key+"."+e.fp+"."+e.entry); err != nil {
p.logf("the wait of %s on %s for %s is not said, and is said at the next pass: %v", e.module, e.node, e.part, err)
continue
}
p.put(ctx, buckets.Waits, key, func() ([]byte, error) { return acting.Encode(w) })
p.orderSaid[key] = true
}
for key, raw := range kept {
if !acting.IsOrder(key) || standing[key] {
continue
}
w, err := acting.Decode(raw)
if err != nil {
continue
}
if _, there := trunk[w.Module]; there && !p.orderVisited[w.Module] {
continue // the module was not looked at in this pass (no pipeline passed for it yet): its wait stands as kept
}
how := acting.HowRan
switch e, there := p.orders[key]; {
case w.Released:
how = acting.HowReleased
case there && e.fp != w.Fp, !there && wanted[w.Module][w.Node].Fp != w.Entry:
how = acting.HowReplaced
}
if !w.EndSaid {
body := acting.Body(w, assignment.Module)
body.How = how
if err := p.wantsSay(seat.EventWaitEnded, body, key+"."+w.Fp+"."+w.Entry+"."+how); err != nil {
p.logf("the end of the wait of %s on %s for %s is not said, and is said at the next pass: %v", w.Module, w.Node, w.Part, err)
continue
}
}
p.remove(ctx, buckets.Waits, key)
delete(p.orderSaid, key)
}
}
@@ -0,0 +1,351 @@
package main
import (
"context"
"encoding/json"
"fmt"
"strings"
"testing"
"time"
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/seatdef"
"mesh-delivery-assigner/internal/acting"
"mesh-delivery-assigner/internal/assigngraph"
plc "mesh-delivery-assigner/internal/assignment"
"mesh-delivery-assigner/internal/buckets"
)
// orderFP is a fingerprint of the form the trunk keeps, made of one letter.
func orderFP(c string) string {
return meshcatalogclient.SourcePrefix2 + strings.Repeat(fmt.Sprintf("%x", c[0]&15), 64)
}
// orderMod is one module of an order test: the fingerprint it runs on ace and the one the trunk holds, the form of its kept data at each
// fingerprint (0: it keeps none), the seats it holds and the source revision of its trunk value.
type orderMod struct {
name string
run, trunk string
forms map[string]int
holds []string
src int64
}
type orderFixture struct {
t *testing.T
p *passer
wants *memKV
events []string
bodies []meshwants.Wait
answers chan struct{}
}
// newOrderFixture is a pass on node ace, which composes its own modules, with each module running its fingerprint healthy, built and
// granted at the trunk's, an older want, the catalog's entries answered and each held seat as the assignments say it. history lists the
// fingerprints of each module by assignment graph revision, oldest first (the objects Reverted reads).
func newOrderFixture(t *testing.T, mods []orderMod, history ...map[string]string) *orderFixture {
t.Helper()
ctx := context.Background()
f := &orderFixture{t: t, wants: &memKV{values: map[string][]byte{}}, answers: make(chan struct{}, 64)}
f.p = newPasser(kvSet{buckets.Wants: f.wants, buckets.Asks: &memKV{values: map[string][]byte{}}, buckets.Waits: &memKV{values: map[string][]byte{}}}, time.Now())
p := f.p
p.logf = t.Logf
p.say = func(oc.Condition) error { return nil }
p.publishWants = func(event string, body any, id string) error {
f.events = append(f.events, event)
f.bodies = append(f.bodies, body.(meshwants.Wait))
return nil
}
digest := "sha256:" + strings.Repeat("c", 64)
pl := mma.Assignments{Node: "ace"}
forms := map[string]int{} // by fingerprint: each module's own
modules := []string{}
for _, m := range mods {
modules = append(modules, m.name)
pl.Modules = append(pl.Modules, mma.PlacedModule{Module: m.name})
for _, seat := range m.holds {
scope := "node"
if seat != catalogue.BackupSeat && seat != nodeengine.Seat {
scope = "mesh"
}
pl.Holds = append(pl.Holds, mma.Holding{Claim: seat, Module: m.name, Scope: scope})
}
for fp, form := range m.forms {
forms[fp] = form
}
mdigest := fmt.Sprintf("%064x", len(modules))
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: m.trunk, Manifest: mdigest, Repository: "novox/" + m.name, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath, Src: m.src})
putModuleRaw(p.w, trunkBranch, m.name, mod)
p.w.manifests[mdigest] = parsed{module: m.name, manifest: catalogue.Manifest{Module: m.name}}
endDone(t, p.w, m.name, m.trunk, mdigest)
gkey, _ := grantsstate.PartGrantKey(m.name, m.trunk)
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
p.w.put(buckets.Grants, gkey, grant)
f.runs(m.name, m.run)
wkey, _ := mwc.WantKey("ace", m.name)
old, _ := mwc.EncodeWant(mwc.Want{Fp: m.run, Self: true})
p.w.put(buckets.Wants, wkey, old)
f.wants.values[wkey] = old
}
composes(t, p.w, "ace", digest)
values, err := mmac.AssignmentsValues(pl)
if err != nil {
t.Fatal(err)
}
for key, raw := range values {
p.w.put(plc.AssignmentsBucket, key, raw)
}
p.w.heardPresence("ace", true, time.Now())
objects := memObjects{}
for i, at := range history {
half := meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: int64(i + 1), Source: int64(i + 1)},
Entries: map[string]meshcatalog.CatalogEntry{}}
for _, module := range modules {
digest := fmt.Sprintf("sha256:%064x", (i+1)*1000+len(half.Entries))
half.Revision.Entries = append(half.Revision.Entries, meshcatalog.EntryRef{Module: module, Digest: digest})
half.Entries[module] = meshcatalog.CatalogEntry{Module: module, Fp: at[module]}
}
placed := mma.Assignments{Node: "ace"}
for _, module := range modules {
placed.Modules = append(placed.Modules, mma.PlacedModule{Module: module})
}
in := assigngraph.Inputs{Halves: map[string]meshcatalogclient.Half{trunkBranch: half}, FirstBranch: func(string) string { return trunkBranch },
Placements: map[string]mma.Assignments{"ace": placed}}
if _, _, err := assigngraph.Keep(ctx, objects, assigngraph.Compute(in), "test", time.Now()); err != nil {
t.Fatal(err)
}
}
p.graph = &graphKeeper{objects: objects}
p.entries = meshcatalogclient.NewEntries(func(_ string, body any) (json.RawMessage, error) {
in := body.(meshcatalog.EntryInput)
entry := meshcatalog.CatalogEntry{Module: "m", Fp: in.Fingerprint, Manifest: map[string]any{"module": "m"}}
if form := forms[in.Fingerprint]; form > 0 {
entry.Manifest = map[string]any{"module": "m", "state": []any{map[string]any{"name": "sessions", "form": form}}}
}
return json.Marshal(meshcatalog.EntryAnswer{Entry: entry})
})
p.entries.Changed = func(meshcatalogclient.EntryRow) { f.answers <- struct{}{} }
return f
}
// runs says module runs fp healthy on ace, heard as the node-engine seat's running and in the have.
func (f *orderFixture) runs(module, fp string) {
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
f.p.w.put(buckets.Have, "ace."+module, have)
f.p.w.running["ace."+module] = nodeengine.Running{Node: "ace", Module: module, Fingerprint: fp}
}
// pass makes a pass, and after it every catalog entry it asked is answered.
func (f *orderFixture) pass() {
f.t.Helper()
f.p.wants(context.Background())
asked := 0
for _, row := range f.p.entries.Rows() {
if row.State == meshcatalogclient.EntryAsked {
asked++
}
}
for range asked {
select {
case <-f.answers:
case <-time.After(10 * time.Second): // a bound on a failure only: the answers come at once
f.t.Fatal("an entry asked was not answered")
}
}
}
// wantAt is the fingerprint module is wanted at on ace.
func (f *orderFixture) wantAt(module string) string {
f.t.Helper()
key, _ := mwc.WantKey("ace", module)
w, err := mwc.DecodeWant(f.wants.values[key])
if err != nil {
f.t.Fatal(err)
}
return w.Fp
}
func (f *orderFixture) waits() []string {
var out []string
for key := range f.p.w.read(buckets.Waits) {
if acting.IsOrder(key) {
out = append(out, key)
}
}
return out
}
// Design 47 G §298 first edge, A-258, A-259: a module whose kept data's form changes is not moved until the new version of the node-backup
// holder and of the holder of the seat mesh-broker run; the wait is kept under `order.` in the state waits, said as wait-held, and ends
// ran (wait-ended) when the holders run; a module whose kept data keeps its form is moved at once.
func TestAModuleWhoseKeptDataChangesFormWaitsForTheNewBackupHolderAndTheBusModule(t *testing.T) {
f := newOrderFixture(t, []orderMod{
{name: "web", run: orderFP("a"), trunk: orderFP("b"), forms: map[string]int{orderFP("a"): 1, orderFP("b"): 2}},
{name: "plain", run: orderFP("c"), trunk: orderFP("d"), forms: map[string]int{orderFP("c"): 1, orderFP("d"): 1}},
{name: "backup", run: orderFP("e"), trunk: orderFP("f"), holds: []string{catalogue.BackupSeat}},
{name: "bus", run: orderFP("g"), trunk: orderFP("h"), holds: []string{meshbroker.Seat}},
})
f.pass() // the entries are asked: no want crosses a form change unread
if got := f.wantAt("web"); got != orderFP("a") {
t.Fatalf("web is wanted at %s before the entries are read", got)
}
f.pass()
if got := f.wantAt("web"); got != orderFP("a") {
t.Fatalf("web moved across a form change at %s while the new node-backup holder and bus module do not run", got)
}
if got := f.wantAt("plain"); got != orderFP("d") {
t.Errorf("a module whose kept data keeps its form is wanted at %s, not moved", got)
}
if len(f.waits()) != 2 || len(f.events) != 2 || f.events[0] != meshwants.EventWaitHeld || f.bodies[0].Step != seatdef.StepPlace || f.bodies[0].Module != "web" {
t.Fatalf("waits %v events %v bodies %+v: want two wait-held in the step place, `order.ace.web.backup` and `order.ace.web.bus`", f.waits(), f.events, f.bodies)
}
if _, err := acting.Decode(f.p.w.read(buckets.Waits)[acting.OrderKey("ace", "web", "backup")]); err != nil {
t.Fatalf("the wait is not kept under its key: %v", err)
}
f.pass()
if len(f.events) != 2 {
t.Fatalf("a pass that changed nothing said %v", f.events)
}
f.runs("backup", orderFP("f"))
f.pass()
if got := f.wantAt("web"); got != orderFP("a") || len(f.events) != 3 || f.events[2] != meshwants.EventWaitEnded || f.bodies[2].How != acting.HowRan {
t.Fatalf("with the bus module still old: web at %s, events %v", got, f.events)
}
f.runs("bus", orderFP("h"))
f.pass()
f.pass()
if got := f.wantAt("web"); got != orderFP("b") || len(f.waits()) != 0 || len(f.events) != 4 {
t.Fatalf("with both running: web at %s, waits %v, events %v", got, f.waits(), f.events)
}
}
// §298 first edge, the hand: assigner_waits reads an order wait and assigner_release with how `now` lets the module move.
func TestAnOrderWaitIsReadAndReleasedByTheExistingTools(t *testing.T) {
f := newOrderFixture(t, []orderMod{
{name: "web", run: orderFP("a"), trunk: orderFP("b"), forms: map[string]int{orderFP("a"): 1, orderFP("b"): 2}},
{name: "backup", run: orderFP("e"), trunk: orderFP("f"), holds: []string{catalogue.BackupSeat}},
})
f.pass()
f.pass()
var acts []string
s := newPutTools(memStore{waitsState: f.p.kvs[buckets.Waits].(*memKV), buckets.DeliveryWaitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveredAgainState: &memKV{values: map[string][]byte{}}}, func(key string, body any) (json.RawMessage, error) {
acts = append(acts, key)
return json.RawMessage(`{"ok":true,"log":"logged"}`), nil
})
got, err := s.doWaits(map[string]any{"module": "web"})
if list := got.(map[string]any)["waits"].([]any); err != nil || len(list) != 1 || list[0].(actingSaid).Key != "order.ace.web.backup" {
t.Fatalf("assigner_waits = %+v %v", got, err)
}
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "web", "how": "now", "why": "the restore point is taken by hand"}, "shanks.node-tools"); err != nil || len(acts) == 0 {
t.Fatalf("%v %v", err, acts)
}
f.p.w.put(buckets.Waits, "order.ace.web.backup", f.p.kvs[buckets.Waits].(*memKV).values["order.ace.web.backup"])
f.pass()
f.pass()
if got := f.wantAt("web"); got != orderFP("b") || f.bodies[len(f.bodies)-1].How != acting.HowReleased {
t.Fatalf("after the release: web at %s, last %+v", got, f.bodies[len(f.bodies)-1])
}
}
// §298 second edge, A-259: at a revert the node-engine goes back last on its node, after every module that is set back there across a
// form change runs the reverted build.
func TestAtARevertTheNodeEngineGoesBackLast(t *testing.T) {
old, now := map[string]string{"web": orderFP("a"), "engine": orderFP("c")}, map[string]string{"web": orderFP("b"), "engine": orderFP("d")}
f := newOrderFixture(t, []orderMod{
{name: "web", run: orderFP("b"), trunk: orderFP("a"), forms: map[string]int{orderFP("a"): 1, orderFP("b"): 2}},
{name: "engine", run: orderFP("d"), trunk: orderFP("c"), holds: []string{nodeengine.Seat}},
}, old, now)
f.pass()
f.pass()
if f.wantAt("web") != orderFP("a") {
t.Fatalf("web is wanted at %s: the module set back is not held", f.wantAt("web"))
}
if got := f.wantAt("engine"); got != orderFP("d") {
t.Fatalf("the node-engine is set back to %s before web runs its reverted build", got)
}
if w := f.waits(); len(w) != 1 || w[0] != "order.ace.engine.web" || f.bodies[len(f.bodies)-1].Module != "engine" {
t.Fatalf("waits %v bodies %+v", w, f.bodies)
}
f.runs("web", orderFP("a"))
f.pass()
f.pass()
if got := f.wantAt("engine"); got != orderFP("c") || len(f.waits()) != 0 {
t.Fatalf("with web back: the node-engine is wanted at %s, waits %v", got, f.waits())
}
}
// §298 third edge, A-259, A-260: the reverted assigner and gate are wanted only once the indexer's take-in.taken of the trunk branch names a
// source revision at least the trunk's head (the highest src of the trunk's module values).
func TestTheRevertedAssignerAndGateWaitForTheIndexerToTakeInTheTrunksHead(t *testing.T) {
old, now := map[string]string{"asg": orderFP("a"), "gate": orderFP("c"), "idx": orderFP("e")}, map[string]string{"asg": orderFP("b"), "gate": orderFP("d"), "idx": orderFP("f")}
f := newOrderFixture(t, []orderMod{
{name: "asg", run: orderFP("b"), trunk: orderFP("a"), holds: []string{mma.Seat}, src: 6},
{name: "gate", run: orderFP("d"), trunk: orderFP("c"), holds: []string{meshdelivery.Seat}, src: 7},
{name: "idx", run: orderFP("e"), trunk: orderFP("e"), src: 7},
}, old, now)
f.pass()
f.pass()
if f.wantAt("asg") != orderFP("b") || f.wantAt("gate") != orderFP("d") {
t.Fatalf("the reverted assigner (%s) and gate (%s) are wanted before the head is taken in", f.wantAt("asg"), f.wantAt("gate"))
}
if len(f.waits()) != 2 || f.bodies[0].Part != "mesh-delivery-indexer" {
t.Fatalf("waits %v bodies %+v", f.waits(), f.bodies)
}
taken := func(branch string, source int64) {
raw, _ := json.Marshal(meshcatalog.TakeInTaken{Kind: "taken", Key: "m.1", Branch: branch, Source: source})
if err := f.p.w.taken.put(raw); err != nil {
t.Fatal(err)
}
}
taken("beta", 9) // another branch, and
taken(trunkBranch, 6)
f.pass()
if f.wantAt("asg") != orderFP("b") {
t.Fatal("a source revision before the trunk's head was taken in, and the assigner was wanted")
}
taken(trunkBranch, 7)
f.pass()
f.pass()
if f.wantAt("asg") != orderFP("a") || f.wantAt("gate") != orderFP("c") || len(f.waits()) != 0 {
t.Fatalf("after the head is taken in: asg %s gate %s waits %v", f.wantAt("asg"), f.wantAt("gate"), f.waits())
}
}
// A-260: an order wait still held after a restart is said again as wait-held, by a pass that remembers nothing, and ends at the next
// take-in.taken that names the trunk's head.
func TestAnOrderWaitHeldAfterARestartIsSaidAgain(t *testing.T) {
old, now := map[string]string{"asg": orderFP("a")}, map[string]string{"asg": orderFP("b")}
f := newOrderFixture(t, []orderMod{{name: "asg", run: orderFP("b"), trunk: orderFP("a"), holds: []string{mma.Seat}, src: 7}}, old, now)
f.pass()
f.pass()
if len(f.events) != 1 {
t.Fatalf("events %v", f.events)
}
f.p.orderSaid = nil // a process started again
f.pass()
if len(f.events) != 2 || f.events[1] != meshwants.EventWaitHeld || len(f.waits()) != 1 {
t.Fatalf("events %v waits %v", f.events, f.waits())
}
raw, _ := json.Marshal(meshcatalog.TakeInTaken{Kind: "taken", Key: "m.1", Branch: trunkBranch, Source: 8})
if err := f.p.w.taken.put(raw); err != nil {
t.Fatal(err)
}
f.pass()
f.pass()
if f.wantAt("asg") != orderFP("a") || len(f.waits()) != 0 || f.events[len(f.events)-1] != meshwants.EventWaitEnded {
t.Fatalf("asg at %s, waits %v, events %v", f.wantAt("asg"), f.waits(), f.events)
}
}
@@ -267,10 +267,10 @@ func TestTheToolsReadAndReleaseAnOutsideWait(t *testing.T) {
s.st.put(waitsState, k, v)
}
got, err := s.doWaits(map[string]any{"module": "nats"})
if err != nil || len(got.(map[string]any)["waits"].([]waitSaid)) != 1 {
if err != nil || len(got.(map[string]any)["waits"].([]any)) != 1 {
t.Fatalf("assigner_waits = %+v %v", got, err)
}
if w := got.(map[string]any)["waits"].([]waitSaid)[0]; w.On != "unreachable:hub" || w.BoundEnds == "" {
if w := got.(map[string]any)["waits"].([]any)[0].(waitSaid); w.On != "unreachable:hub" || w.BoundEnds == "" {
t.Errorf("wait = %+v", w)
}
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "sideways", "why": "x"}, "shanks.node-tools"); err == nil {
@@ -14,6 +14,7 @@ import (
stdio "git.novox.be/novox/mesh-sdk/go"
"git.novox.be/novox/mesh-sdk/go/keys"
"mesh-delivery-assigner/internal/acting"
"mesh-delivery-assigner/internal/buckets"
"mesh-delivery-assigner/internal/outside"
)
@@ -36,6 +37,13 @@ type waitSaid struct {
BoundEnds string `json:"bound-ends,omitempty"`
}
// actingSaid is one wait for an acting part's new version as assigner_waits answers it (actingwaits.go).
type actingSaid struct {
Key string `json:"key"`
acting.Wait
BoundEnds string `json:"bound-ends,omitempty"`
}
// doWaits is the tool assigner_waits.
func (s *putTools) doWaits(args map[string]any) (any, error) {
keys, err := s.st.keys(waitsState)
@@ -43,12 +51,22 @@ func (s *putTools) doWaits(args map[string]any) (any, error) {
return nil, fmt.Errorf("the waits cannot be listed: %w", err)
}
sort.Strings(keys)
out := []waitSaid{}
out := []any{}
for _, key := range keys {
raw, _, found, err := s.st.get(waitsState, key)
if err != nil || !found {
continue
}
if acting.Is(key) || acting.IsOrder(key) {
if a, err := acting.Decode(raw); err == nil && (arg(args, "node") == "" || a.Node == arg(args, "node")) && (arg(args, "module") == "" || a.Module == arg(args, "module")) {
said := actingSaid{Key: key, Wait: a}
if d, err := time.ParseDuration(a.Bound); err == nil && !a.Released {
said.BoundEnds = a.ClockFrom.Add(d).UTC().Format("2006-01-02T15:04:05Z")
}
out = append(out, said)
}
continue
}
w, err := outside.Decode(raw)
if err != nil || (arg(args, "node") != "" && w.Node != arg(args, "node")) || (arg(args, "module") != "" && w.Module != arg(args, "module")) {
continue
@@ -172,6 +190,9 @@ func (s *putTools) doReleaseDelivery(args map[string]any, by string) (any, error
// module, which the assigner's process takes at its next pass.
func (s *putTools) doReleaseWait(args map[string]any, by string) (any, error) {
node, module, how, why := arg(args, "node"), arg(args, "module"), arg(args, "how"), arg(args, "why")
if how == acting.HowNow {
return s.doReleaseActing(node, module, why, by)
}
if !slices.Contains([]string{outside.HowGoOn, outside.HowFailed, outside.HowRejudge}, how) {
return nil, fmt.Errorf("how is %s, %s or %s, not %q", outside.HowGoOn, outside.HowFailed, outside.HowRejudge, how)
}
@@ -217,9 +238,61 @@ func (s *putTools) doReleaseWait(args map[string]any, by string) (any, error) {
return map[string]any{"released": key, "how": how, "by": by, "hand-act": taken.Log}, nil
}
// doReleaseActing is assigner_release with how `now`: the entry of module on node moves with its acting parts' running versions. The
// hand act is recorded first; the release is written on each standing wait of the entry, and the process says wait-ended at its next pass.
func (s *putTools) doReleaseActing(node, module, why, by string) (any, error) {
if why == "" {
return nil, errors.New("release says why: it is kept in the hand-act log")
}
keys, err := s.st.keys(waitsState)
if err != nil {
return nil, fmt.Errorf("the waits cannot be listed: %w", err)
}
type one struct {
key string
w acting.Wait
}
var held []one
prefix, orderPrefix := acting.Prefix+node+"."+module+".", acting.OrderPrefix+node+"."+module+"."
for _, k := range keys {
if !strings.HasPrefix(k, prefix) && !strings.HasPrefix(k, orderPrefix) {
continue
}
raw, _, found, err := s.st.get(waitsState, k)
if err != nil || !found {
continue
}
if w, err := acting.Decode(raw); err == nil && !w.Released {
held = append(held, one{k, w})
}
}
if len(held) == 0 {
return nil, fmt.Errorf("%s holds no wait of %s for an acting part: assigner_waits lists them", node, module)
}
rec := meshhandactsclient.Recorder{Ask: s.ask}
taken, err := rec.Record(meshhandactsclient.Request{What: fmt.Sprintf("assigner_release %s %s %s", node, module, acting.HowNow), Why: why,
Cause: "acting-wait", Verb: "mesh-wants.assigner_release"})
if err != nil {
return nil, fmt.Errorf("the hand act could not be recorded, so nothing was released: %w", err)
}
released := []string{}
for _, h := range held {
h.w.Released, h.w.By, h.w.Why = true, by+": "+why, why
data, err := acting.Encode(h.w)
if err != nil {
return nil, err
}
if err := s.st.put(waitsState, h.key, data); err != nil {
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
}
released = append(released, h.key)
}
return map[string]any{"released": released, "how": acting.HowNow, "by": by, "hand-act": taken.Log}, nil
}
// waitsTool is assigner_waits as the runtime serves it.
func (s *putTools) waitsTool() stdio.Tool {
return stdio.Tool{Name: toolWaits, Description: "Every wait of the delivery on an outside cause and where it stands: a moved module's failing health check on a " +
return stdio.Tool{Name: toolWaits, Description: "Every wait of the delivery and where it stands: an entry waiting for an acting part's new version in composing or placing (§266, §268), with its part, version, step, clock-from and bound; and a moved module's failing health check on a " +
"machine whose open conditions name a cause outside the delivery (design 47 G §706), with its module, machine, fingerprint, the " +
"condition's key, what it stands `on`, its clock-from, bound, its ask and its state; a settle shows as settling with its end. Narrowed by " +
"machine or module. Read-only.",
@@ -14,6 +14,7 @@ import (
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
@@ -103,6 +104,14 @@ type passer struct {
publishWants publisher
// graph keeps the assignment graph revision the wants follow (graphkept.go, design 47 G §292, A-67); nil: none is kept.
graph *graphKeeper
// orders are §298's order edges computed in this pass, by their wait's key, and orderVisited the modules they were computed for (orderwaits.go).
orders map[string]orderEdge
orderVisited map[string]bool
// orderSaid are the order waits whose wait-held this process said.
orderSaid map[string]bool
// entries are the catalog entries by fingerprint asked of the seat mesh-catalog's verb `entry` (design 47 G §614, A-217): a
// held revert's running entry and its want's (revert.go); nil: no revert is asked about.
entries *meshcatalogclient.Entries
}
// wantsSay says one event of the seat mesh-wants; with no bus to say it on, nothing is said.
@@ -207,10 +216,12 @@ func (p *passer) pass(ctx context.Context) {
p.keepGraph(ctx)
p.judgeDelivered()
p.deliveredAgain(ctx)
p.wants(ctx)
if p.graph == nil || !p.graph.full { // a full assignment graph store keeps the last revision and writes no new want (design 47 G §181)
p.wants(ctx)
}
p.puts(ctx)
p.nodeWants(ctx)
p.backups(ctx)
p.backups(ctx, p.reverts(ctx))
}
// nodeWants writes each placed node's own want, `want.<node>.node`: the node composes its own (ADR 0547 §2, ADR 0571 §6).
@@ -286,6 +297,10 @@ func (p *passer) wants(ctx context.Context) {
defer func() { p.sayNeither(neither, stale, left) }()
// Where grants are read from, once for the pass: the grants part's (issue 774, ADR 0571 clause 9).
grants := p.w.grantsBucket()
released := p.actingReleased()
p.orders, p.orderVisited = map[string]orderEdge{}, map[string]bool{}
defer func() { p.actingWaits(ctx, viewOf(p.w.trunkName()).nodes, wanted) }()
defer func() { p.orderWaits(ctx, viewOf(p.w.trunkName()).nodes, wanted) }()
for name := range names {
groups := p.groups(name, nodesOf[name], assignedBranches[name])
for gi, g := range groups {
@@ -356,6 +371,9 @@ func (p *passer) wants(ctx context.Context) {
m.MachineHeld = p.outside.Held
nodes = p.outsideReview(ctx, name, fp, m.Nodes, nodes, manifests[name].ClaimsSeat(mcseat.Seat), p.movedTo(wanted, nodes))
}
unread := p.orderEdgesOf(ctx, m, wantedOf[name], trunk)
actingOf, orderOf := p.actingOf(name, released), p.orderActing(name, released, unread, func() assign.Nodes { return nodes })
m.Acting = func(node string) []assign.ActingEdge { return append(actingOf(node), orderOf(node)...) }
out := assign.Wants(m, nodes, wantedOf[name], last)
for _, node := range m.Nodes {
for _, prov := range m.MissingProvisions(node) {
@@ -744,7 +762,7 @@ func (p *passer) lift(ctx context.Context, module, fp string) {
// staleAsk says why the hold, restore or record ask at key (read as the SDK reads it) is about something the trunk has left
// behind, or "" when it is not: a module no longer on the trunk, a fingerprint that is no longer the module's trunk fingerprint,
// or, for a record ask, a composition the module's node no longer waits on. A backup ask is stale when the module's want on its
// or, for a record ask, a composition the module's node no longer waits on. A backup ask, and a revert ask (design 47 G §710), is stale when the module's want on its
// node names another fingerprint or is removed (the move it is about was superseded; issue 1966 note 165); a want not written
// yet, or at the same fingerprint in another composition, leaves it (backups asks that again under the next number). An ask of
// another kind is never stale here. A module of the trunk whose fingerprint is not read yet, and a have not read yet, are not stale: nothing known is
@@ -755,7 +773,7 @@ func (p *passer) staleAsk(key string, trunk map[string]meshcatalog.ModulesByModu
return ""
}
switch k.Kind {
case asks.AskKindBackup:
case asks.AskKindBackup, asks.AskKindRevert:
wkey, err := mwc.WantKey(k.Node, k.Module)
if err != nil {
return ""
@@ -876,6 +894,10 @@ func (p *passer) staleAsks(ctx context.Context, trunk map[string]meshcatalog.Mod
}
delete(p.said, assign.BackupCondition(k.Node, k.Module, k.FP, "", false).Key)
}
// A revert ask's finding goes with it: a newer want replaced the held revert (design 47 G §710, §363).
if kerr == nil && k.Kind == asks.AskKindRevert {
p.revertFinding(assign.RevertHeldFinding(k.Node, k.Module, "", "", ""), false)
}
p.logf("the ask %s is about a fingerprint or composition the trunk has left behind and is taken away: %s", key, why)
p.remove(ctx, buckets.Asks, key)
// The held record stays: it is the memory that this fingerprint failed (ADR 0408 §4), and a revert returns the trunk to
@@ -995,7 +1017,9 @@ func (p *passer) recordAsk(ctx context.Context, module, fp string, need assign.R
// before the `@`). Each answer is another want value (the asking's number is in the want's words), so a backup that fails again
// after "back up again" and is answered "again" again changes the key and the node retries. That the backup failed again is seen
// in a new have value after the answer was written.
func (p *passer) backups(ctx context.Context) {
//
// A have that holds a revert (reverts, which answers its `<node>.<module>` in reverting) is left to the revert ask.
func (p *passer) backups(ctx context.Context, reverting map[string]bool) {
wanted := p.w.wantsOf()
for key, value := range p.w.read(buckets.Have) {
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
@@ -1003,7 +1027,7 @@ func (p *passer) backups(ctx context.Context) {
continue
}
have, err := nodeengineclient.DecodeHave(value)
if err != nil || have.Phase != nodeengineclient.PhaseHeld {
if err != nil || have.Phase != nodeengineclient.PhaseHeld || reverting[node+"."+module] {
continue
}
want, ok := wanted[module][node]
@@ -334,8 +334,9 @@ func TestTheCatalogHalfIsTakenInPerBranch(t *testing.T) {
woken := 0
h := &halves{w: w, wake: func() { woken++ }, logf: t.Logf}
entry := func(fp string) json.RawMessage {
raw, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "web", Fp: fp, Repository: "novox/web", Commit: "c1", Manifest: map[string]any{"module": "web", "version": "1"},
Invokes: []string{"seat:x.v"}})
raw, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "web", Fp: fp, Repository: "novox/web", Commit: "c1",
Manifest: map[string]any{"module": "web", "version": "1", "requires": []any{"database", map[string]any{"name": "cache", "optional": true}}},
Invokes: []string{"seat:x.v"}, Requires: []string{"cache", "database"}})
return raw
}
h.changed("main", "web", entry("src2:1"), false)
@@ -347,7 +348,8 @@ func TestTheCatalogHalfIsTakenInPerBranch(t *testing.T) {
if got := w.manifest(v.Manifest); got.err != "" || got.manifest.Module != "web" {
t.Errorf("the entry's manifest is not read by its digest: %+v", got)
}
if n := w.graphOf("main")["web"]; n.FP != "src2:1" || !slices.Equal(n.Invokes, []string{"seat:x.v"}) {
// The entry's requires name the optional cache too; the graph node means those required, not optionally (A-229).
if n := w.graphOf("main")["web"]; n.FP != "src2:1" || !slices.Equal(n.Invokes, []string{"seat:x.v"}) || !slices.Equal(n.Requires, []string{"database"}) {
t.Errorf("graph node %+v", n)
}
h.changed("beta", "web", nil, true)
@@ -0,0 +1,177 @@
package main
import (
"context"
"strconv"
"strings"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/views"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/buckets"
)
// revertReasons are the reasons a node's have gives when it holds a revert (design 47 G §300, §567).
var revertReasons = []string{"restore point missing", "restore point damaged"}
// revertReason is the reason of a have's words when they say a held revert, or "".
func revertReason(words string) string {
for _, r := range revertReasons {
if strings.HasPrefix(words, r) {
return r
}
}
return ""
}
// reverts sees to each revert a node holds with no restore point (design 47 G §300, §710): the have's phase held with the
// reason `restore point missing` or `restore point damaged`, for a want the assignment graph tells a revert (assigngraph.Reverted,
// A-217) across a kept data's form (assigngraph.FormChanges, the running entry and the want's read through the catalog seat's verb
// `entry`). It asks the operator hold or live, raises the finding revert-held, and on live writes the want again with `revert`
// live; hold writes nothing. The finding is cleared once the reverted build runs. It answers the `<node>.<module>` of each have
// that holds a revert, which backups leaves to it.
func (p *passer) reverts(ctx context.Context) map[string]bool {
taken := map[string]bool{}
wanted := p.w.wantsOf()
for key, value := range p.w.read(buckets.Have) {
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
if err != nil || isNode {
continue
}
have, err := nodeengineclient.DecodeHave(value)
if err != nil {
continue
}
want, ok := wanted[module][node]
if !ok || want.Removed {
continue
}
akey, err := asks.RevertAskKey(node, module, want.Fp)
if err != nil {
continue
}
reason := revertReason(have.Detail)
if have.Phase != nodeengineclient.PhaseHeld || reason == "" {
if have.Fp == want.Fp && p.w.has(buckets.Asks, akey) {
p.revertFinding(assign.RevertHeldFinding(node, module, "", "", ""), false) // the reverted build runs
}
continue
}
taken[node+"."+module] = true
kept, form, ok := p.revertCrosses(ctx, node, module, want.Fp)
if !ok {
continue
}
current, currentAsk, there := assign.DecodeAsk(akey, p.w.read(buckets.Asks)[akey])
if there != nil {
now := p.clock()
a := assign.RevertAsk(akey, node, module, want.Fp, kept, form, have.Detail, 0, now)
p.put(ctx, buckets.Asks, akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
p.revertFinding(assign.RevertHeldFinding(node, module, kept, form, reason), true)
continue
}
p.revertFinding(assign.RevertHeldFinding(node, module, kept, form, reason), true)
answer, answered := p.w.answer(akey)
if router, _, _ := strings.Cut(answer.Warrant, "@"); !answered || router != currentAsk.ID || answer.Option != assign.OptionRevertLive {
continue // not answered, or answered hold: the hold stands and nothing is written
}
words := assign.RevertAnswerWords(answer.Option, current.Asking)
if want.Revert != mwc.RevertLive || want.Words != words {
want.Revert, want.Words = mwc.RevertLive, words
p.putWant(ctx, node, module, want)
}
}
return taken
}
// revertCrosses reads whether the want of module on node at wantFp is a revert across a kept data's form: the kept data's names
// and the want's forms of them, said as the ask and the finding name them. ok is false while it is not known (the entries not
// answered yet, a pass with the assignment graph not kept) or when the want is no revert across a form.
func (p *passer) revertCrosses(ctx context.Context, node, module, wantFp string) (kept, form string, ok bool) {
if p.graph == nil || p.entries == nil {
return "", "", false
}
run, heard := p.w.runningOf(node, module)
if !heard || run.Stopped || run.Fingerprint == "" {
return "", "", false
}
reverted, err := assigngraph.Reverted(ctx, p.graph.objects, node, module, run.Fingerprint, wantFp)
if err != nil {
p.logf("whether the want of %s on %s at %s is a revert is not read: %v", module, node, wantFp, err)
return "", "", false
}
if !reverted {
return "", "", false
}
changes, known := p.formChange(ctx, module, run.Fingerprint, wantFp)
if !known {
return "", "", false // asked: its answer wakes a pass
}
var names, forms []string
for _, c := range changes {
f := 0
if c.Applied != nil {
f = c.Applied.Form
}
names, forms = append(names, c.Name), append(forms, strconv.Itoa(f))
}
if len(names) == 0 {
return "", "", false
}
return strings.Join(names, ", "), strings.Join(forms, ", "), true
}
// revertFinding writes the finding revert-held, or clears it, once per process each way.
func (p *passer) revertFinding(o assign.Observed, raised bool) {
key := views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)
said := key + ".raised"
if !raised {
said = key + ".cleared"
}
if p.said[said] {
return
}
var err error
switch {
case raised && p.findings.Put != nil:
err = p.findings.Put(o.Finding(p.clock()))
case !raised && p.findings.Delete != nil:
err = p.findings.Delete(key)
default:
return
}
if err != nil {
p.logf("the finding %s is not written: %v", key, err)
return
}
p.said[said] = true
delete(p.said, map[bool]string{true: key + ".cleared", false: key + ".raised"}[raised])
}
// formChange reads the kept data whose form differs between module's entry at runFp and at wantFp (design 47 G §295 to §298, A-217), as
// the catalog seat's verb `entry` answers the two entries; known is false while either is asked, and woken by its answer.
func (p *passer) formChange(ctx context.Context, module, runFp, wantFp string) (changes []catalogue.FormChange, known bool) {
if p.entries == nil {
return nil, false
}
running, rs := p.entries.Need(ctx, runFp)
applied, as := p.entries.Need(ctx, wantFp)
if rs == mcc.EntryAsked || as == mcc.EntryAsked {
return nil, false
}
if rs != mcc.EntryAnswered || as != mcc.EntryAnswered {
return nil, true // the catalog holds no such entry: no kept data's form is read from it, so none changes
}
changes, err := assigngraph.FormChanges(running, applied)
if err != nil {
p.logf("the forms of %s's kept data between %s and %s are not read: %v", module, runFp, wantFp, err)
return nil, false
}
return changes, true
}
@@ -0,0 +1,136 @@
package main
import (
"context"
"encoding/json"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
"mesh-delivery-assigner/internal/assign"
"mesh-delivery-assigner/internal/assigngraph"
"mesh-delivery-assigner/internal/buckets"
)
// Design 47 G §300, §710, A-217: a node holding a revert with no restore point gets the revert ask, keyed by the reverted build,
// and the finding revert-held, never a backup ask; hold writes no want; live writes the want with `revert` live and its words;
// the finding clears once the reverted build runs; a newer want takes the ask away and clears the finding.
func TestAHeldRevertIsAskedAndALiveAnswerIsWrittenIntoTheWant(t *testing.T) {
ctx := context.Background()
at := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
fpOld := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
p := newPasser(kvSet{buckets.Asks: asksKV, buckets.Wants: wantsKV}, at)
p.logf = t.Logf
findings := map[string][]byte{}
p.findings = assign.Findings{Put: putInto(findings), Delete: func(k string) error { delete(findings, k); return nil }}
// Two assignment graph revisions: web on ace at fpOld, then at fpTest, which it runs.
objects := memObjects{}
for i, fp := range []string{fpOld, fpTest} {
half := meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: int64(i + 1), Source: int64(i + 1),
Entries: []meshcatalog.EntryRef{{Module: "web", Digest: "sha256:" + strings.Repeat(string(rune('c'+i)), 64)}}},
Entries: map[string]meshcatalog.CatalogEntry{"web": {Module: "web", Fp: fp}}}
in := assigngraph.Inputs{Halves: map[string]meshcatalogclient.Half{trunkBranch: half}, FirstBranch: func(string) string { return trunkBranch },
Placements: map[string]meshmoduleassigner.Assignments{"ace": {Node: "ace", Modules: []meshmoduleassigner.PlacedModule{{Module: "web"}}}}}
if _, _, err := assigngraph.Keep(ctx, objects, assigngraph.Compute(in), "test", at); err != nil {
t.Fatal(err)
}
}
p.graph = &graphKeeper{objects: objects}
// The entries by fingerprint, as the catalog seat's verb entry answers them: the kept data sessions at form 1, then form 2.
entry := func(fp string, form int) meshcatalog.CatalogEntry {
return meshcatalog.CatalogEntry{Module: "web", Fp: fp, Manifest: map[string]any{"module": "web",
"state": []any{map[string]any{"name": "sessions", "form": form}}}}
}
answered := make(chan struct{}, 4)
p.entries = meshcatalogclient.NewEntries(func(_ string, body any) (json.RawMessage, error) {
in := body.(meshcatalog.EntryInput)
return json.Marshal(meshcatalog.EntryAnswer{Entry: map[string]meshcatalog.CatalogEntry{fpOld: entry(fpOld, 1), fpTest: entry(fpTest, 2)}[in.Fingerprint]})
})
p.entries.Changed = func(meshcatalogclient.EntryRow) { answered <- struct{}{} }
p.w.running["ace.web"] = nodeengine.Running{Node: "ace", Module: "web", Fingerprint: fpTest}
wkey, _ := mwc.WantKey("ace", "web")
want := mwc.Want{Fp: fpOld, Self: true}
raw, _ := mwc.EncodeWant(want)
p.w.put(buckets.Wants, wkey, raw)
hkey, _ := nodeengineclient.HaveKey("ace", "web")
have := func(fp, phase, detail string) {
raw, err := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: phase, Detail: detail, Health: nodeengineclient.HealthOK})
if err != nil {
t.Fatal(err)
}
p.w.put(buckets.Have, hkey, raw)
}
have(fpTest, nodeengineclient.PhaseHeld, "restore point missing: web's sessions has none of form 1 taken before "+fpOld)
pass := func() { p.backups(ctx, p.reverts(ctx)) }
pass() // the entries are asked; nothing is asked of the operator until they answer
for range 2 {
select {
case <-answered:
case <-time.After(10 * time.Second): // a bound on a failure only: the answers come at once
t.Fatalf("the entries were not asked: asks %v", asksKV.values)
}
}
pass()
rkey, _ := asks.RevertAskKey("ace", "web", fpOld)
bkey, _ := asks.BackupAskKey("ace", "web", fpOld)
if !p.w.has(buckets.Asks, rkey) || p.w.has(buckets.Asks, bkey) {
t.Fatalf("asks %v: want the revert ask %s and no backup ask", asksKV.values, rkey)
}
_, ask, err := assign.DecodeAsk(rkey, p.w.read(buckets.Asks)[rkey])
if err != nil || !strings.Contains(ask.Explanation, "sessions") || !strings.Contains(ask.Explanation, "form 1") {
t.Fatalf("the revert ask = %+v, %v", ask, err)
}
fkey := "delivery.ace.web.revert-held"
if _, raised := findings[fkey]; !raised {
t.Fatalf("findings %v: want %s", findings, fkey)
}
answer := func(option string) {
v, _ := mdc.EncodeAnswer(mdc.Answer{Option: option, Warrant: ask.ID + "@2026-11-12T10:05:00Z", Words: "the operator"})
p.w.put(buckets.Answers, rkey, v)
}
puts := wantsKV.puts
answer(assign.OptionRevertHold)
pass()
if wantsKV.puts != puts {
t.Fatal("an answer hold wrote a want")
}
answer(assign.OptionRevertLive)
pass()
got, err := mwc.DecodeWant(p.w.read(buckets.Wants)[wkey])
if err != nil || got.Fp != fpOld || got.Revert != mwc.RevertLive || got.Words != `the operator answered "live" to the revert ask (asking 1)` {
t.Fatalf("the want after live = %+v, %v", got, err)
}
have(fpOld, nodeengineclient.PhaseRunning, "") // the reverted build runs: the finding clears
pass()
if _, raised := findings[fkey]; raised {
t.Fatal("the finding stands while the reverted build runs")
}
// A newer want replaces the held revert: the ask goes and the finding with it.
have(fpTest, nodeengineclient.PhaseHeld, "restore point damaged: the point does not read")
pass()
if _, raised := findings[fkey]; !raised {
t.Fatal("the finding is not raised again for a hold again")
}
raw, _ = mwc.EncodeWant(mwc.Want{Fp: fpTest, Self: true})
p.w.put(buckets.Wants, wkey, raw)
p.staleAsks(ctx, map[string]meshcatalog.ModulesByModuleValue{"web": {Fp2: fpTest}})
if _, raised := findings[fkey]; raised || p.w.has(buckets.Asks, rkey) {
t.Fatalf("a newer want left the ask (%v) or the finding (%v)", p.w.has(buckets.Asks, rkey), raised)
}
}
@@ -348,7 +348,7 @@ func (s *putTools) tools() []stdio.Tool {
"rules write the machine's want again from their next pass. Nothing changes on the machine until they move it.",
Input: schema(map[string]string{"node": "the machine", "module": "the module", "why": "why, kept with the put",
"how": "with how, the wait of the delivery on an outside cause (assigner_waits) is released, not a put: go-on (the module counts healthy there), " +
"failed (the failing check is the delivery's) or rejudge (judge it again now); written in the hand-act log first",
"failed (the failing check is the delivery's), rejudge (judge it again now), or now (the entry waiting for an acting part's new version moves with the running one); written in the hand-act log first",
"step": "delivery: the module's wait for its dependency's delivery (assigner_waits) is released, its want written now; no machine is named; written in the hand-act log first"},
[]string{"module", "why"}),
Run: func(a map[string]any) (any, error) { return s.doRelease(a, stdio.CallerOf(a)) }},
@@ -47,6 +47,8 @@ type world struct {
delivered deliveredVersions
// edges are, by watched branch and module, the edges into the module's artifacts its catalog entry names (§374's kinds).
edges map[string]map[string][]meshcatalog.Edge
// taken is the newest source revision of each watched branch the indexer's take-in.taken named, as heard (orderwaits.go, §298, A-260).
taken takenSources
// assignments is the seat's state assignments assembled node by node from its keys in either form (mesh-sdk
// mmac.Keys; research 072 row 37): the one reader of its keys in the assigner.
assignmentKeys *mmac.Keys