Merge branch 'feat/47-G-fix11-b2l' into feat/47-G-join11
# Conflicts: # modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/buildended.go # modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/graphkept.go # modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go # modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go # modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/tools.go # modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go # modules/mesh-delivery-assigner/internal/assign/wants_test.go # modules/mesh-delivery-assigner/manifest_test.go # modules/mesh-delivery-assigner/module.json
This commit is contained in:
@@ -0,0 +1,152 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"mesh-delivery-assigner/internal/acting"
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// The waits for an acting part's new version in the steps the assigner holds (design 47 G §266, §268, §276): an entry of the assignment
|
||||
// graph whose acting edge names composing or placing waits, its want held, until the part's version runs on the edge's node. The wait is
|
||||
// kept in the state `waits` under `acting.<node>.<module>.<part>`, said as `wait-held` when it starts and `wait-ended` (ran, released,
|
||||
// replaced) when it ends; `assigner_waits` reads it and `assigner_release` with how `now` lets the entry move with the part's running
|
||||
// version. A node away starts no wait: its clock starts when the node is present (§268).
|
||||
|
||||
// actingOf is the Module.Acting of module: the held acting edges of its entry on a node, those whose wait was not released left out.
|
||||
func (p *passer) actingOf(module string, released map[string]bool) func(node string) []assign.ActingEdge {
|
||||
return func(node string) []assign.ActingEdge {
|
||||
var out []assign.ActingEdge
|
||||
for _, e := range p.entryEdges(node, module) {
|
||||
if acting.Held(e.Step) && !released[acting.Key(node, module, e.Module)+"@"+e.Fp] {
|
||||
out = append(out, assign.ActingEdge{Node: e.Node, Part: e.Module, Fp: e.Fp, Step: e.Step})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
// entryEdges are the acting edges of the entry of module on node in the revision the pass computed.
|
||||
func (p *passer) entryEdges(node, module string) []seat.Edge {
|
||||
g := p.graph
|
||||
if g == nil {
|
||||
return nil
|
||||
}
|
||||
for _, ref := range g.computed.Revision.Entries {
|
||||
if ref.Node == node && ref.Module == module {
|
||||
var out []seat.Edge
|
||||
for _, e := range g.computed.Entries[ref.Digest].Edges {
|
||||
if e.Kind == "acting" {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// actingReleased is the waits released by hand and not yet run, by `<key>@<part's fingerprint>`.
|
||||
func (p *passer) actingReleased() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for key, raw := range p.w.read(buckets.Waits) {
|
||||
if w, err := acting.Decode(raw); err == nil && w.Released && (acting.Is(key) || acting.IsOrder(key)) {
|
||||
out[key+"@"+w.Fp] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// actingWaits keeps the acting waits after the pass's wants: it starts the wait of each entry held for a part's version, ends the
|
||||
// wait of one that ran, was replaced or was released, and says each.
|
||||
func (p *passer) actingWaits(ctx context.Context, n assign.Nodes, wanted map[string]map[string]meshwants.Want) {
|
||||
g := p.graph
|
||||
if g == nil || p.kvs[buckets.Waits] == nil {
|
||||
return
|
||||
}
|
||||
now := p.clock()
|
||||
kept := p.w.read(buckets.Waits)
|
||||
standing := map[string]bool{}
|
||||
for _, ref := range g.computed.Revision.Entries {
|
||||
entry := g.computed.Entries[ref.Digest]
|
||||
if w, ok := wanted[ref.Module][ref.Node]; ok && w.Fp == entry.Fp {
|
||||
continue // the entry's version is wanted already: nothing moves, so nothing waits
|
||||
}
|
||||
for _, e := range entry.Edges {
|
||||
if e.Kind != "acting" || !acting.Held(e.Step) {
|
||||
continue
|
||||
}
|
||||
key := acting.Key(ref.Node, ref.Module, e.Module)
|
||||
if n.Runs(assign.ActingEdge{Node: e.Node, Part: e.Module, Fp: e.Fp, Step: e.Step}) {
|
||||
continue
|
||||
}
|
||||
standing[key] = true
|
||||
if raw := kept[key]; raw != nil {
|
||||
if w, err := acting.Decode(raw); err == nil && w.Entry == entry.Fp && w.Fp == e.Fp {
|
||||
p.actingReleaseSaid(ctx, key, w)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if !n.Online[ref.Node] {
|
||||
continue // a node away holds only itself, and its clock has not started (§268)
|
||||
}
|
||||
w := acting.Wait{Step: e.Step, Node: ref.Node, Module: ref.Module, Part: e.Module, PartNode: e.Node, Fp: e.Fp, Entry: entry.Fp,
|
||||
Since: now, ClockFrom: now, Bound: acting.BoundOf(e).String()}
|
||||
if err := p.wantsSay(seat.EventWaitHeld, acting.Body(w, assignment.Module), key+"."+e.Fp+"."+entry.Fp); err != nil {
|
||||
p.logf("the wait of %s on %s for %s is not said, and is said at the next pass: %v", ref.Module, ref.Node, e.Module, err)
|
||||
continue
|
||||
}
|
||||
p.put(ctx, buckets.Waits, key, func() ([]byte, error) { return acting.Encode(w) })
|
||||
}
|
||||
}
|
||||
for key, raw := range kept {
|
||||
if !acting.Is(key) || standing[key] {
|
||||
continue
|
||||
}
|
||||
w, err := acting.Decode(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
how := acting.HowRan
|
||||
if w.Released {
|
||||
how = acting.HowReleased
|
||||
} else if e, ok := p.edgeOf(w); !ok || !n.Runs(e) {
|
||||
how = acting.HowReplaced
|
||||
}
|
||||
if !w.EndSaid {
|
||||
body := acting.Body(w, assignment.Module)
|
||||
body.How = how
|
||||
if err := p.wantsSay(seat.EventWaitEnded, body, key+"."+w.Fp+"."+w.Entry+"."+how); err != nil {
|
||||
p.logf("the end of the wait of %s on %s for %s is not said, and is said at the next pass: %v", w.Module, w.Node, w.Part, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.remove(ctx, buckets.Waits, key)
|
||||
}
|
||||
}
|
||||
|
||||
// actingReleaseSaid says wait-ended of a wait released by hand, once, and keeps its record so the entry is not held again.
|
||||
func (p *passer) actingReleaseSaid(ctx context.Context, key string, w acting.Wait) {
|
||||
if !w.Released || w.EndSaid {
|
||||
return
|
||||
}
|
||||
body := acting.Body(w, assignment.Module)
|
||||
body.How = acting.HowReleased
|
||||
body.By = w.By
|
||||
if err := p.wantsSay(seat.EventWaitEnded, body, key+"."+w.Fp+"."+w.Entry+"."+acting.HowReleased); err != nil {
|
||||
p.logf("the release of the wait of %s on %s for %s is not said, and is said at the next pass: %v", w.Module, w.Node, w.Part, err)
|
||||
return
|
||||
}
|
||||
w.EndSaid = true
|
||||
p.put(ctx, buckets.Waits, key, func() ([]byte, error) { return acting.Encode(w) })
|
||||
}
|
||||
|
||||
// edgeOf is the edge a kept wait stands on, as an acting edge.
|
||||
func (p *passer) edgeOf(w acting.Wait) (assign.ActingEdge, bool) {
|
||||
return assign.ActingEdge{Node: w.PartNode, Part: w.Part, Fp: w.Fp, Step: w.Step}, true
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"mesh-delivery-assigner/internal/acting"
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// Design 47 G §266, §268, §276 (AS9): an entry with an acting edge in composing waits for the part's version to run on the edge's node;
|
||||
// the wait is kept, said as wait-held with its bound and clock-from, read by assigner_waits, released by assigner_release with how
|
||||
// `now` and ended when the version runs.
|
||||
func TestAnEntryWaitsForItsActingPartAndTheWaitIsKeptSaidAndEnded(t *testing.T) {
|
||||
now := time.Date(2026, 11, 12, 9, 0, 0, 0, time.UTC)
|
||||
p := newPasser(kvSet{buckets.Waits: &memKV{values: map[string][]byte{}}}, now)
|
||||
var events []string
|
||||
var bodies []meshwants.Wait
|
||||
p.publishWants = func(event string, body any, id string) error {
|
||||
events = append(events, event)
|
||||
bodies = append(bodies, body.(meshwants.Wait))
|
||||
return nil
|
||||
}
|
||||
entry := meshwants.AssignmentEntry{Node: "ace", Module: "nats", Fp: fpTest, Edges: []meshwants.Edge{
|
||||
{Kind: "acting", Node: "ace", Module: "mesh-host", Fp: "src2:new", Seat: "node-engine", Step: "compose", Bound: "15m"},
|
||||
{Kind: "acting", Node: "ace", Module: "indexer", Fp: "src2:idx", Seat: "mesh-catalog", Step: "take-in"}}}
|
||||
p.graph = &graphKeeper{}
|
||||
p.graph.computed.Revision.Entries = []meshwants.NodeEntryRef{{Node: "ace", Module: "nats", Digest: "d1"}}
|
||||
p.graph.computed.Entries = map[string]meshwants.AssignmentEntry{"d1": entry}
|
||||
n := assign.Nodes{Online: map[string]bool{"ace": true}, Running: map[string]map[string]nodeengine.Running{"ace": {"mesh-host": {Fingerprint: "src2:old"}}}}
|
||||
wanted := map[string]map[string]mwc.Want{"nats": {"ace": {Fp: "src2:older"}}}
|
||||
ctx := context.Background()
|
||||
|
||||
if got := p.actingOf("nats", nil)("ace"); len(got) != 1 || got[0].Part != "mesh-host" {
|
||||
t.Fatalf("the assigner holds composing only: %+v", got)
|
||||
}
|
||||
p.actingWaits(ctx, n, wanted)
|
||||
p.actingWaits(ctx, n, wanted) // said once
|
||||
if len(events) != 1 || events[0] != meshwants.EventWaitHeld {
|
||||
t.Fatalf("events %v", events)
|
||||
}
|
||||
b := bodies[0]
|
||||
if b.Step != "compose" || b.Part != "mesh-host" || b.Fp != "src2:new" || b.Bound != "15m0s" || b.ClockFrom == nil || !b.ClockFrom.Equal(now) || b.Owner == "" {
|
||||
t.Errorf("wait-held %+v", b)
|
||||
}
|
||||
key := acting.Key("ace", "nats", "mesh-host")
|
||||
if _, err := acting.Decode(p.w.read(buckets.Waits)[key]); err != nil {
|
||||
t.Fatalf("the wait is kept: %v", err)
|
||||
}
|
||||
// a release by hand lets the entry move, and ends the wait once
|
||||
w, _ := acting.Decode(p.w.read(buckets.Waits)[key])
|
||||
w.Released = true
|
||||
raw, _ := acting.Encode(w)
|
||||
p.w.put(buckets.Waits, key, raw)
|
||||
if got := p.actingOf("nats", p.actingReleased())("ace"); len(got) != 0 {
|
||||
t.Errorf("a released wait still holds: %+v", got)
|
||||
}
|
||||
p.actingWaits(ctx, n, wanted)
|
||||
p.actingWaits(ctx, n, wanted)
|
||||
if len(events) != 2 || events[1] != meshwants.EventWaitEnded || bodies[1].How != acting.HowReleased {
|
||||
t.Fatalf("events %v bodies %+v", events, bodies)
|
||||
}
|
||||
// the version runs: the record goes, said nothing more
|
||||
n.Running["ace"]["mesh-host"] = nodeengine.Running{Fingerprint: "src2:new"}
|
||||
p.actingWaits(ctx, n, wanted)
|
||||
if len(p.w.read(buckets.Waits)) != 0 || len(events) != 2 {
|
||||
t.Errorf("kept %v events %v", p.w.read(buckets.Waits), events)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnActingWaitOfANodeAwayStartsNoClockAndAnEntryWantedAlreadyWaitsNot(t *testing.T) {
|
||||
p := newPasser(kvSet{buckets.Waits: &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
p.publishWants = func(string, any, string) error { t.Fatal("said"); return nil }
|
||||
p.graph = &graphKeeper{}
|
||||
p.graph.computed.Revision.Entries = []meshwants.NodeEntryRef{{Node: "ace", Module: "nats", Digest: "d1"}}
|
||||
p.graph.computed.Entries = map[string]meshwants.AssignmentEntry{"d1": {Node: "ace", Module: "nats", Fp: "src2:a", Edges: []meshwants.Edge{
|
||||
{Kind: "acting", Node: "ace", Module: "mesh-host", Fp: "src2:new", Step: "compose"}}}}
|
||||
n := assign.Nodes{Online: map[string]bool{}}
|
||||
p.actingWaits(context.Background(), n, map[string]map[string]mwc.Want{})
|
||||
n.Online["ace"] = true
|
||||
p.actingWaits(context.Background(), n, map[string]map[string]mwc.Want{"nats": {"ace": {Fp: "src2:a"}}})
|
||||
if len(p.w.read(buckets.Waits)) != 0 || strings.Contains(p.graph.computed.Entries["d1"].Fp, "x") {
|
||||
t.Errorf("kept %v", p.w.read(buckets.Waits))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheToolsReadAndReleaseAnActingWait(t *testing.T) {
|
||||
var acts []string
|
||||
s := newPutTools(memStore{waitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveryWaitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveredAgainState: &memKV{values: map[string][]byte{}}}, func(key string, body any) (json.RawMessage, error) {
|
||||
acts = append(acts, key)
|
||||
return json.RawMessage(`{"ok":true,"log":"logged"}`), nil
|
||||
})
|
||||
w := acting.Wait{Step: "compose", Node: "ace", Module: "nats", Part: "mesh-host", PartNode: "ace", Fp: "src2:new", Entry: "src2:a", Since: time.Now(), ClockFrom: time.Now(), Bound: "30m0s"}
|
||||
raw, _ := acting.Encode(w)
|
||||
s.st.put(waitsState, acting.Key("ace", "nats", "mesh-host"), raw)
|
||||
got, err := s.doWaits(map[string]any{"module": "nats"})
|
||||
if list := got.(map[string]any)["waits"].([]any); err != nil || len(list) != 1 || list[0].(actingSaid).BoundEnds == "" {
|
||||
t.Fatalf("assigner_waits = %+v %v", got, err)
|
||||
}
|
||||
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "now"}, "x"); err == nil {
|
||||
t.Error("a release with no why is taken")
|
||||
}
|
||||
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "now", "why": "the new engine is known good"}, "shanks.node-tools"); err != nil || len(acts) == 0 {
|
||||
t.Fatalf("%v %v", err, acts)
|
||||
}
|
||||
raw, _, _, _ = s.st.get(waitsState, acting.Key("ace", "nats", "mesh-host"))
|
||||
if r, _ := acting.Decode(raw); !r.Released || !strings.Contains(r.By, "known good") {
|
||||
t.Errorf("wait = %+v", r)
|
||||
}
|
||||
}
|
||||
@@ -154,3 +154,17 @@ func hearBuilds(ctx context.Context, js jetstream.JetStream, ends *builtEnds, wa
|
||||
cc.Stop()
|
||||
}()
|
||||
}
|
||||
|
||||
// hearTaken registers the hub's taker of the indexer's take-in.taken, each waking the pass (orderwaits.go, §298, A-260); before
|
||||
// the hub starts.
|
||||
func hearTaken(hub *process.EventHub, taken *takenSources, wake func(), logf func(string, ...any)) error {
|
||||
return hub.Handle(takenSubject, func(m jetstream.Msg) {
|
||||
if err := taken.put(m.Data()); err != nil {
|
||||
logf("%s is not kept: %v", m.Subject(), err)
|
||||
}
|
||||
if err := m.Ack(); err != nil {
|
||||
logf("acknowledging %s: %v", m.Subject(), err)
|
||||
}
|
||||
wake()
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
)
|
||||
|
||||
// A store filling is said before it is full (design 47 G §181, in the assigner's part): the assignment graph revisions' object store
|
||||
// says `filling` of the seat mesh-wants when it passes 80 % of its cap, and `full` when the next revision does not fit: the assigner then keeps
|
||||
// its last revision and writes no new want, and says `room` when the store is back below 80 % and goes on. Nothing is dropped to make room.
|
||||
|
||||
// fillOf is the store's fill now: its bytes and cap, when the bucket says them and has a cap.
|
||||
func (g *graphKeeper) fillOf(ctx context.Context) (used, capacity int64, ok bool) {
|
||||
f, is := g.objects.(assigngraph.Filler)
|
||||
if !is {
|
||||
return 0, 0, false
|
||||
}
|
||||
used, capacity, err := f.Fill(ctx)
|
||||
return used, capacity, err == nil && capacity > 0
|
||||
}
|
||||
|
||||
func (p *passer) sayFill(event, about string, used, capacity int64, refused string) {
|
||||
at := p.clock().UTC()
|
||||
body := meshwants.Fill{Store: assigngraph.RevisionsObjects, Bytes: used, Cap: capacity, Refused: refused, At: &at}
|
||||
if err := p.wantsSay(event, body, event+"-"+about); err != nil {
|
||||
p.logf("%s of %s is not said: %v", event, assigngraph.RevisionsObjects, err)
|
||||
}
|
||||
}
|
||||
|
||||
// fillAtStart judges the store once at the process's first pass, when no event is pending: a store past 80 % says filling (full when at
|
||||
// its cap), so a fill reached while the process was down is said too.
|
||||
func (p *passer) fillAtStart(ctx context.Context) {
|
||||
g := p.graph
|
||||
used, capacity, ok := g.fillOf(ctx)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case used >= capacity:
|
||||
g.full = true
|
||||
p.sayFill(meshwants.EventFull, "start", used, capacity, "the next revision")
|
||||
case used*5 >= capacity*4:
|
||||
p.sayFill(meshwants.EventFilling, "start", used, capacity, "")
|
||||
}
|
||||
}
|
||||
|
||||
// fullBefore says the next revision does not fit its store, and keeps nothing: it says `full` once, and `room` once the store is below
|
||||
// 80 % again. It answers true while the store is full.
|
||||
func (p *passer) fullBefore(ctx context.Context, c assigngraph.Computed) bool {
|
||||
g := p.graph
|
||||
used, capacity, ok := g.fillOf(ctx)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if g.full && used*5 < capacity*4 {
|
||||
g.full = false
|
||||
p.sayFill(meshwants.EventRoom, fmt.Sprint(used), used, capacity, "")
|
||||
}
|
||||
if used+assigngraph.NewBytes(ctx, g.objects, c) <= capacity {
|
||||
g.full = false
|
||||
return false
|
||||
}
|
||||
if !g.full {
|
||||
g.full = true
|
||||
p.sayFill(meshwants.EventFull, fmt.Sprint(used), used, capacity, "the next revision")
|
||||
}
|
||||
p.logf("the assignment graph revision is not kept: %s is full (%d of %d bytes), the last revision stays and no want is written until there is room", assigngraph.RevisionsObjects, used, capacity)
|
||||
return true
|
||||
}
|
||||
|
||||
// fillingAfter says `filling` when keeping a revision took the store past 80 % of its cap.
|
||||
func (p *passer) fillingAfter(ctx context.Context, before int64) {
|
||||
g := p.graph
|
||||
used, capacity, ok := g.fillOf(ctx)
|
||||
if ok && before*5 < capacity*4 && used*5 >= capacity*4 {
|
||||
p.sayFill(meshwants.EventFilling, fmt.Sprint(used), used, capacity, "")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
)
|
||||
|
||||
// filled is an object bucket with a fill: the bytes it holds count beside what is stored.
|
||||
type filled struct {
|
||||
objs map[string][]byte
|
||||
base int64
|
||||
cap int64
|
||||
}
|
||||
|
||||
func (f *filled) Put(_ context.Context, name string, data []byte) error {
|
||||
f.objs[name] = data
|
||||
return nil
|
||||
}
|
||||
func (f *filled) Get(_ context.Context, name string) ([]byte, error) {
|
||||
if d, ok := f.objs[name]; ok {
|
||||
return d, nil
|
||||
}
|
||||
return nil, assigngraph.ErrNoObject
|
||||
}
|
||||
func (f *filled) Names(context.Context) ([]string, error) {
|
||||
out := []string{}
|
||||
for n := range f.objs {
|
||||
out = append(out, n)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
func (f *filled) Fill(context.Context) (int64, int64, error) {
|
||||
used := f.base
|
||||
for _, d := range f.objs {
|
||||
used += int64(len(d))
|
||||
}
|
||||
return used, f.cap, nil
|
||||
}
|
||||
|
||||
// Design 47 G §181 (AS13): the assignment graph revisions' store says filling past 80 % of its cap, refuses a revision that does not fit
|
||||
// with `full` (the last revision stays, no want is written), and says `room` below 80 % again; at the start a store already past 80 % is said.
|
||||
func TestTheAssignmentGraphStoreSaysFillingFullAndRoom(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
p := newPasser(kvSet{}, time.Now())
|
||||
var events []string
|
||||
p.publishWants = func(event string, body any, id string) error {
|
||||
if _, ok := body.(meshwants.Fill); ok {
|
||||
events = append(events, event)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
o := &filled{objs: map[string][]byte{}, base: 500, cap: 1000}
|
||||
p.graph = &graphKeeper{objects: o, halves: func() map[string]mcc.Half { return nil }}
|
||||
p.keepGraph(ctx) // the first pass judges at the start: 500 of 1000 is below 80 %
|
||||
if len(events) != 0 {
|
||||
t.Fatalf("events %v", events)
|
||||
}
|
||||
o.base = 700
|
||||
p.graph.started = false
|
||||
p.keepGraph(ctx)
|
||||
if len(events) != 1 || events[0] != meshwants.EventFilling {
|
||||
t.Fatalf("at the start past 80 %%: %v", events)
|
||||
}
|
||||
c := assigngraph.Computed{Revision: meshwants.AssignmentRevision{Entries: []meshwants.NodeEntryRef{}}}
|
||||
o.base = 990
|
||||
if !p.fullBefore(ctx, c) || !p.graph.full || events[len(events)-1] != meshwants.EventFull {
|
||||
t.Fatalf("a revision that does not fit: events %v", events)
|
||||
}
|
||||
p.fullBefore(ctx, c)
|
||||
if n := len(events); n != 2 {
|
||||
t.Errorf("full is said once: %v", events)
|
||||
}
|
||||
o.base = 100
|
||||
if p.fullBefore(ctx, c) || p.graph.full || events[len(events)-1] != meshwants.EventRoom {
|
||||
t.Errorf("room: events %v", events)
|
||||
}
|
||||
}
|
||||
@@ -34,8 +34,14 @@ type graphKeeper struct {
|
||||
source int64
|
||||
ok bool
|
||||
said string
|
||||
// cycled is the cycles last said (assigngraph.Cycles).
|
||||
cycled string
|
||||
// full says the next revision did not fit the store, so no want is written (graphfill.go); started says the first pass judged the fill.
|
||||
full, started bool
|
||||
// pins removes a revision that ceased to be current once nothing pins it (pins.go); nil keeps every revision.
|
||||
pins *pinKeeper
|
||||
// computed is the revision computed by the last pass, with its entries: the edges the wants wait on (actingwaits.go).
|
||||
computed assigngraph.Computed
|
||||
}
|
||||
|
||||
// keepGraph computes the revision of this pass and keeps it when the pass's inputs differ from the last computed.
|
||||
@@ -45,15 +51,30 @@ func (p *passer) keepGraph(ctx context.Context) {
|
||||
return
|
||||
}
|
||||
c := assigngraph.Compute(p.graphInputs())
|
||||
g.computed = c
|
||||
for _, u := range c.Uncomputed {
|
||||
if g.said != u {
|
||||
g.said = u
|
||||
p.logf("the assignment graph leaves out %s", u)
|
||||
}
|
||||
}
|
||||
if cycles := strings.Join(c.Cycles, "; "); cycles != g.cycled {
|
||||
g.cycled = cycles
|
||||
if cycles != "" {
|
||||
p.logf("the assignment graph holds a cycle among the waits within one step, and its entries wait for each other: %s", cycles)
|
||||
}
|
||||
}
|
||||
if !g.started {
|
||||
g.started = true
|
||||
p.fillAtStart(ctx)
|
||||
}
|
||||
if g.has && g.ok && assigngraph.Same(g.last, c.Revision) {
|
||||
return
|
||||
}
|
||||
used, _, _ := g.fillOf(ctx)
|
||||
if p.fullBefore(ctx, c) {
|
||||
return
|
||||
}
|
||||
rev, fresh, err := assigngraph.Keep(ctx, g.objects, c, meshwants.EventAssignmentRevision, p.clock())
|
||||
if err != nil {
|
||||
p.logf("the assignment graph revision is not kept, so no want is said as wanted until it is: %v", err)
|
||||
@@ -69,6 +90,7 @@ func (p *passer) keepGraph(ctx context.Context) {
|
||||
if !fresh {
|
||||
return
|
||||
}
|
||||
p.fillingAfter(ctx, used)
|
||||
if g.pins != nil {
|
||||
g.pins.judge() // the revision before ceased to be current
|
||||
}
|
||||
@@ -82,6 +104,11 @@ func (p *passer) keepGraph(ctx context.Context) {
|
||||
// node's capability reading and the first branch of each registration.
|
||||
func (p *passer) graphInputs() assigngraph.Inputs {
|
||||
in := assigngraph.Inputs{Halves: p.graph.halves(), Placements: p.w.assignments(), Capabilities: p.capabilityDigests()}
|
||||
// The seat definitions read through the seat mesh-catalog's verb seat-definitions say which seat delivers a requirement
|
||||
// (design 47 G §708).
|
||||
if r := p.seats.Reader(); r != nil {
|
||||
in.Definitions = r.Definitions()
|
||||
}
|
||||
// With no pipeline ends kept, the wants are the trunk-only form (pass.go): a placement naming no branch runs the trunk's.
|
||||
first := p.w.trunkName()
|
||||
in.FirstBranch = func(string) string { return first }
|
||||
|
||||
@@ -138,6 +138,8 @@ func fromEntry(e meshcatalog.CatalogEntry) (meshcatalog.ModulesByModuleValue, gr
|
||||
return v, n, &parsed{module: e.Module, err: "it does not decode: " + err.Error()}
|
||||
}
|
||||
n.Serves = graph.ServesOf(man)
|
||||
// The entry's requires name the optional ones too; the node means those required, not optionally (A-229).
|
||||
n.Requires = slices.DeleteFunc(slices.Clone(n.Requires), man.IsOptional)
|
||||
return v, n, &parsed{module: e.Module, manifest: man}
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
mcseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions"
|
||||
occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
@@ -576,6 +577,9 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
return p.keepCandidate(ctx, iv.call, ev)
|
||||
}
|
||||
iv.candidates.Store(&keep)
|
||||
// A held revert's running entry and its want's, asked of the seat mesh-catalog's verb entry (design 47 G §710, A-217).
|
||||
p.entries = meshcatalogclient.NewEntries(iv.h.call)
|
||||
p.entries.Changed = func(meshcatalogclient.EntryRow) { wake() }
|
||||
// The pins of the revisions (pins.go, design 47 G §579): the gate's pinned, freed, heads-replaced and pins-stated
|
||||
// on the hub, registered before it starts.
|
||||
if kv := kvs[pinsBucket]; kv != nil {
|
||||
@@ -618,6 +622,9 @@ func follow(ctx context.Context, js jetstream.JetStream, nc process.Subscriber,
|
||||
}
|
||||
hearDelivery(ctx, js, w, wake, findings, logger.Printf) // version-released, version-delivered, machine-away and machine-back
|
||||
hearBuilds(ctx, js, &w.ends, wake, findings, logger.Printf)
|
||||
if err := hearTaken(hub, &w.taken, wake, logger.Printf); err != nil {
|
||||
logger.Printf("the indexer's take-ins are not heard, so an entry held by an order edge waits until the assigner starts again: %v", err)
|
||||
}
|
||||
hearReadsChanged(ctx, js, hub, reads, findings, gate.EventsLive, logger.Printf)
|
||||
hearRunning(ctx, js, w, wake, findings, logger.Printf)
|
||||
}
|
||||
|
||||
@@ -99,3 +99,20 @@ func (b *busObjects) Delete(ctx context.Context, name string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Fill is the bytes the bucket holds and its cap (assigngraph.Filler).
|
||||
func (b *busObjects) Fill(ctx context.Context) (int64, int64, error) {
|
||||
o, err := b.open(ctx)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
st, err := o.Status(ctx)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
info, err := b.js.Stream(ctx, "OBJ_"+st.Bucket())
|
||||
if err != nil {
|
||||
return int64(st.Size()), 0, err
|
||||
}
|
||||
return int64(st.Size()), info.CachedInfo().Config.MaxBytes, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,296 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
seat "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/process"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
|
||||
"mesh-delivery-assigner/internal/acting"
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
"mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// The three order edges of a delivery that crosses a form change (design 47 G §298, A-245, A-258, A-259, A-260). Each is an acting wait of the
|
||||
// assigner, kept in the state `waits` under `order.<node>.<module>.<part>` beside the acting waits, in the step placing, said as wait-held
|
||||
// when a want is held for it and wait-ended (ran, released, replaced) when it moves; assigner_waits reads them and assigner_release with
|
||||
// how `now` releases them, as it does an acting wait. The holders of the seats are read from the seat holders the assignments compose
|
||||
// (assignment.SeatHoldersOf), never named:
|
||||
// - the node-backup holder's and the bus module's (the holder of the seat mesh-broker) new versions run before a module whose kept data's
|
||||
// form changes is moved, since the restore point needs both (OrderBackup);
|
||||
// - at a revert a node's node-engine goes back last on that node, after every module it restores there runs the reverted build, which it
|
||||
// starts only once it has said `restored` (OrderEngineLast);
|
||||
// - the reverted assigner and gate are wanted only once the indexer's take-in.taken of the trunk branch names a source revision at least
|
||||
// the trunk's head, the highest `src` of the trunk branch's module values (OrderTakenIn, A-260).
|
||||
|
||||
// takenSubject is the indexer's event take-in.taken on its own stream, as the manifest's consumes names it.
|
||||
var takenSubject = process.ModuleStreamSubject(buckets.IndexerModule, "take-in.taken")
|
||||
|
||||
// takenSources are the newest source revisions a take-in.taken named, by watched branch, as heard (hearBuildsAndReleases).
|
||||
type takenSources struct {
|
||||
mu sync.Mutex
|
||||
own map[string]int64
|
||||
}
|
||||
|
||||
// put takes one take-in.taken in. A hand drop, and a taken with no source revision (one said before G), take nothing in.
|
||||
func (t *takenSources) put(data []byte) error {
|
||||
var e meshcatalog.TakeInTaken
|
||||
if err := json.Unmarshal(data, &e); err != nil {
|
||||
return fmt.Errorf("a take-in.taken that is none: %w", err)
|
||||
}
|
||||
if e.Dropped || e.Source == 0 || e.Branch == "" {
|
||||
return nil
|
||||
}
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
if t.own == nil {
|
||||
t.own = map[string]int64{}
|
||||
}
|
||||
t.own[e.Branch] = max(t.own[e.Branch], e.Source)
|
||||
return nil
|
||||
}
|
||||
|
||||
// of is the newest source revision of branch taken in; 0 before one is heard.
|
||||
func (t *takenSources) of(branch string) int64 {
|
||||
t.mu.Lock()
|
||||
defer t.mu.Unlock()
|
||||
return t.own[branch]
|
||||
}
|
||||
|
||||
// orderEdge is one order edge held on module's want on node: part's version fp must run on partNode (OrderBackup, OrderEngineLast), or the
|
||||
// trunk's source revision source be taken in (OrderTakenIn).
|
||||
type orderEdge struct {
|
||||
node, module, part, partNode, fp, entry, order string
|
||||
source int64
|
||||
}
|
||||
|
||||
func (e orderEdge) key() string { return acting.OrderKey(e.node, e.module, e.part) }
|
||||
|
||||
func (e orderEdge) acting() assign.ActingEdge {
|
||||
return assign.ActingEdge{Node: e.partNode, Part: e.part, Fp: e.fp, Step: seatdef.StepPlace}
|
||||
}
|
||||
|
||||
// orderRuns says an order edge is met: its part runs the version, or the source revision is taken in.
|
||||
func (p *passer) orderRuns(n assign.Nodes, e orderEdge) bool {
|
||||
if e.order == acting.OrderTakenIn {
|
||||
return p.w.taken.of(p.w.trunkName()) >= e.source
|
||||
}
|
||||
return n.Runs(e.acting())
|
||||
}
|
||||
|
||||
// seatHolders are the holders of each seat as the assignments compose them, by seat name.
|
||||
func (p *passer) holdersBySeat() map[string][]mma.SeatHoldersHoldersItem {
|
||||
out := map[string][]mma.SeatHoldersHoldersItem{}
|
||||
for _, sh := range assignment.SeatHoldersOf(p.w.assignments()) {
|
||||
out[sh.Seat] = append(out[sh.Seat], sh.Holders...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// orderEdgesOf computes §298's edges on the wants of module m on each of its nodes, into p.orders. wanted is m's wants by node; trunk the
|
||||
// trunk's module values. A node that has no running build heard, runs m at its trunk version already, or is wanted that version already
|
||||
// has none: nothing moves. It answers, by node, whether an edge still stands that cannot be recorded as a wait because the catalog entry
|
||||
// it reads is not answered yet (asked, and its answer wakes a pass): the want is held meanwhile, never written across a form change unread.
|
||||
func (p *passer) orderEdgesOf(ctx context.Context, m assign.Module, wanted map[string]mwc.Want, trunk map[string]meshcatalog.ModulesByModuleValue) map[string]bool {
|
||||
unread := map[string]bool{}
|
||||
p.orderVisited[m.Name] = true
|
||||
if m.FP == "" || p.graph == nil || p.entries == nil {
|
||||
return unread
|
||||
}
|
||||
var holders map[string][]mma.SeatHoldersHoldersItem
|
||||
for _, node := range m.Nodes {
|
||||
if w, ok := wanted[node]; ok && !w.Removed && w.Fp == m.FP {
|
||||
continue // the version is wanted already: nothing moves, so nothing waits
|
||||
}
|
||||
run, heard := p.w.runningOf(node, m.Name)
|
||||
if !heard || run.Stopped || run.Fingerprint == "" || run.Fingerprint == m.FP {
|
||||
continue
|
||||
}
|
||||
if holders == nil {
|
||||
holders = p.holdersBySeat()
|
||||
}
|
||||
held := func(seatName string) bool {
|
||||
for _, h := range holders[seatName] {
|
||||
if h.Module == m.Name && h.Node == node {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
add := func(e orderEdge) {
|
||||
e.node, e.module, e.entry = node, m.Name, m.FP
|
||||
p.orders[e.key()] = e
|
||||
}
|
||||
// 1. a kept data's form changes: the restore point needs the new node-backup holder and bus module.
|
||||
changes, known := p.formChange(ctx, m.Name, run.Fingerprint, m.FP)
|
||||
if !known {
|
||||
unread[node] = true
|
||||
}
|
||||
if len(changes) > 0 {
|
||||
for seatName, onNode := range map[string]bool{catalogue.BackupSeat: true, meshbroker.Seat: false} {
|
||||
for _, h := range holders[seatName] {
|
||||
if h.Module == m.Name || (onNode && h.Node != node) || trunk[h.Module].Fp2 == "" {
|
||||
continue
|
||||
}
|
||||
add(orderEdge{part: h.Module, partNode: h.Node, fp: trunk[h.Module].Fp2, order: acting.OrderBackup})
|
||||
}
|
||||
}
|
||||
}
|
||||
if !held(nodeengine.Seat) && !held(mma.Seat) && !held(meshdelivery.Seat) {
|
||||
continue
|
||||
}
|
||||
reverted, err := assigngraph.Reverted(ctx, p.graph.objects, node, m.Name, run.Fingerprint, m.FP)
|
||||
if err != nil {
|
||||
p.logf("whether the want of %s on %s at %s is a revert is not read: %v", m.Name, node, m.FP, err)
|
||||
continue
|
||||
}
|
||||
if !reverted {
|
||||
continue
|
||||
}
|
||||
// 2. at a revert the node-engine goes back last on its node: after every module that is set back there across a form change runs.
|
||||
if held(nodeengine.Seat) {
|
||||
for module, byNode := range p.w.wantsOf() {
|
||||
// What it is set back to is its trunk version, which the rules write as its want in this pass.
|
||||
target := trunk[module].Fp2
|
||||
if w, ok := byNode[node]; module == m.Name || !ok || w.Removed || target == "" {
|
||||
continue
|
||||
}
|
||||
xrun, heard := p.w.runningOf(node, module)
|
||||
if !heard || xrun.Stopped || xrun.Fingerprint == "" || xrun.Fingerprint == target {
|
||||
continue
|
||||
}
|
||||
if back, err := assigngraph.Reverted(ctx, p.graph.objects, node, module, xrun.Fingerprint, target); err != nil || !back {
|
||||
continue
|
||||
}
|
||||
xchanges, known := p.formChange(ctx, module, xrun.Fingerprint, target)
|
||||
if !known {
|
||||
unread[node] = true
|
||||
}
|
||||
if len(xchanges) > 0 {
|
||||
add(orderEdge{part: module, partNode: node, fp: target, order: acting.OrderEngineLast})
|
||||
}
|
||||
}
|
||||
}
|
||||
// 3. the reverted assigner and gate wait for the indexer to take in the trunk's head.
|
||||
if held(mma.Seat) || held(meshdelivery.Seat) {
|
||||
var head int64
|
||||
for _, v := range p.w.trunkOf(p.w.trunkName()) {
|
||||
head = max(head, v.Src)
|
||||
}
|
||||
if head > 0 {
|
||||
add(orderEdge{part: buckets.IndexerModule, fp: strconv.FormatInt(head, 10), source: head, order: acting.OrderTakenIn})
|
||||
}
|
||||
}
|
||||
}
|
||||
return unread
|
||||
}
|
||||
|
||||
// orderActing is the order edges of module on node that still hold its want, as acting edges: those not met, and not released by hand. A
|
||||
// node whose catalog entry is not read yet holds its want too, on a stand-in edge no part runs.
|
||||
func (p *passer) orderActing(module string, released map[string]bool, unread map[string]bool, n func() assign.Nodes) func(node string) []assign.ActingEdge {
|
||||
return func(node string) []assign.ActingEdge {
|
||||
var out []assign.ActingEdge
|
||||
nodes := n()
|
||||
for _, e := range p.orders {
|
||||
if e.module == module && e.node == node && !released[e.key()+"@"+e.fp] && !p.orderRuns(nodes, e) {
|
||||
out = append(out, assign.ActingEdge{Node: e.partNode, Part: e.part, Fp: e.fp, Step: seatdef.StepPlace})
|
||||
}
|
||||
}
|
||||
if unread[node] {
|
||||
out = append(out, assign.ActingEdge{Node: node, Part: "the catalog entry", Step: seatdef.StepPlace})
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
// orderWaits keeps the order waits after the pass's wants, as actingWaits keeps the acting waits: it starts the wait of each edge not met,
|
||||
// ends the wait of one that ran, was replaced or was released, and says each.
|
||||
func (p *passer) orderWaits(ctx context.Context, n assign.Nodes, wanted map[string]map[string]mwc.Want) {
|
||||
if p.kvs[buckets.Waits] == nil {
|
||||
return
|
||||
}
|
||||
now := p.clock()
|
||||
kept := p.w.read(buckets.Waits)
|
||||
trunk := p.w.trunk()
|
||||
if p.orderSaid == nil {
|
||||
p.orderSaid = map[string]bool{}
|
||||
}
|
||||
standing := map[string]bool{}
|
||||
for key, e := range p.orders {
|
||||
if w, ok := wanted[e.module][e.node]; ok && !w.Removed && w.Fp == e.entry {
|
||||
continue
|
||||
}
|
||||
if p.orderRuns(n, e) {
|
||||
continue
|
||||
}
|
||||
standing[key] = true
|
||||
if raw := kept[key]; raw != nil {
|
||||
if w, err := acting.Decode(raw); err == nil && w.Entry == e.entry && w.Fp == e.fp {
|
||||
if !w.Released && !p.orderSaid[key] {
|
||||
// A wait still held after a restart is said again, as wait-held (A-260).
|
||||
if err := p.wantsSay(seat.EventWaitHeld, acting.Body(w, assignment.Module), key+"."+e.fp+"."+e.entry+".again"); err != nil {
|
||||
p.logf("the wait of %s on %s for %s is not said again, and is said at the next pass: %v", e.module, e.node, e.part, err)
|
||||
} else {
|
||||
p.orderSaid[key] = true
|
||||
}
|
||||
}
|
||||
p.actingReleaseSaid(ctx, key, w)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if !n.Online[e.node] {
|
||||
continue // a node away holds only itself, and its clock has not started (§268)
|
||||
}
|
||||
w := acting.Wait{Step: seatdef.StepPlace, Node: e.node, Module: e.module, Part: e.part, PartNode: e.partNode, Fp: e.fp, Entry: e.entry, Order: e.order,
|
||||
Since: now, ClockFrom: now, Bound: acting.BoundOf(seat.Edge{Step: seatdef.StepPlace}).String()}
|
||||
if err := p.wantsSay(seat.EventWaitHeld, acting.Body(w, assignment.Module), key+"."+e.fp+"."+e.entry); err != nil {
|
||||
p.logf("the wait of %s on %s for %s is not said, and is said at the next pass: %v", e.module, e.node, e.part, err)
|
||||
continue
|
||||
}
|
||||
p.put(ctx, buckets.Waits, key, func() ([]byte, error) { return acting.Encode(w) })
|
||||
p.orderSaid[key] = true
|
||||
}
|
||||
for key, raw := range kept {
|
||||
if !acting.IsOrder(key) || standing[key] {
|
||||
continue
|
||||
}
|
||||
w, err := acting.Decode(raw)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if _, there := trunk[w.Module]; there && !p.orderVisited[w.Module] {
|
||||
continue // the module was not looked at in this pass (no pipeline passed for it yet): its wait stands as kept
|
||||
}
|
||||
how := acting.HowRan
|
||||
switch e, there := p.orders[key]; {
|
||||
case w.Released:
|
||||
how = acting.HowReleased
|
||||
case there && e.fp != w.Fp, !there && wanted[w.Module][w.Node].Fp != w.Entry:
|
||||
how = acting.HowReplaced
|
||||
}
|
||||
if !w.EndSaid {
|
||||
body := acting.Body(w, assignment.Module)
|
||||
body.How = how
|
||||
if err := p.wantsSay(seat.EventWaitEnded, body, key+"."+w.Fp+"."+w.Entry+"."+how); err != nil {
|
||||
p.logf("the end of the wait of %s on %s for %s is not said, and is said at the next pass: %v", w.Module, w.Node, w.Part, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
p.remove(ctx, buckets.Waits, key)
|
||||
delete(p.orderSaid, key)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,351 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
mmac "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshbroker"
|
||||
mma "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshdelivery"
|
||||
meshwants "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshwants"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/seatdef"
|
||||
|
||||
"mesh-delivery-assigner/internal/acting"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
plc "mesh-delivery-assigner/internal/assignment"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// orderFP is a fingerprint of the form the trunk keeps, made of one letter.
|
||||
func orderFP(c string) string {
|
||||
return meshcatalogclient.SourcePrefix2 + strings.Repeat(fmt.Sprintf("%x", c[0]&15), 64)
|
||||
}
|
||||
|
||||
// orderMod is one module of an order test: the fingerprint it runs on ace and the one the trunk holds, the form of its kept data at each
|
||||
// fingerprint (0: it keeps none), the seats it holds and the source revision of its trunk value.
|
||||
type orderMod struct {
|
||||
name string
|
||||
run, trunk string
|
||||
forms map[string]int
|
||||
holds []string
|
||||
src int64
|
||||
}
|
||||
|
||||
type orderFixture struct {
|
||||
t *testing.T
|
||||
p *passer
|
||||
wants *memKV
|
||||
events []string
|
||||
bodies []meshwants.Wait
|
||||
answers chan struct{}
|
||||
}
|
||||
|
||||
// newOrderFixture is a pass on node ace, which composes its own modules, with each module running its fingerprint healthy, built and
|
||||
// granted at the trunk's, an older want, the catalog's entries answered and each held seat as the assignments say it. history lists the
|
||||
// fingerprints of each module by assignment graph revision, oldest first (the objects Reverted reads).
|
||||
func newOrderFixture(t *testing.T, mods []orderMod, history ...map[string]string) *orderFixture {
|
||||
t.Helper()
|
||||
ctx := context.Background()
|
||||
f := &orderFixture{t: t, wants: &memKV{values: map[string][]byte{}}, answers: make(chan struct{}, 64)}
|
||||
f.p = newPasser(kvSet{buckets.Wants: f.wants, buckets.Asks: &memKV{values: map[string][]byte{}}, buckets.Waits: &memKV{values: map[string][]byte{}}}, time.Now())
|
||||
p := f.p
|
||||
p.logf = t.Logf
|
||||
p.say = func(oc.Condition) error { return nil }
|
||||
p.publishWants = func(event string, body any, id string) error {
|
||||
f.events = append(f.events, event)
|
||||
f.bodies = append(f.bodies, body.(meshwants.Wait))
|
||||
return nil
|
||||
}
|
||||
digest := "sha256:" + strings.Repeat("c", 64)
|
||||
pl := mma.Assignments{Node: "ace"}
|
||||
forms := map[string]int{} // by fingerprint: each module's own
|
||||
modules := []string{}
|
||||
for _, m := range mods {
|
||||
modules = append(modules, m.name)
|
||||
pl.Modules = append(pl.Modules, mma.PlacedModule{Module: m.name})
|
||||
for _, seat := range m.holds {
|
||||
scope := "node"
|
||||
if seat != catalogue.BackupSeat && seat != nodeengine.Seat {
|
||||
scope = "mesh"
|
||||
}
|
||||
pl.Holds = append(pl.Holds, mma.Holding{Claim: seat, Module: m.name, Scope: scope})
|
||||
}
|
||||
for fp, form := range m.forms {
|
||||
forms[fp] = form
|
||||
}
|
||||
mdigest := fmt.Sprintf("%064x", len(modules))
|
||||
mod, _ := meshcatalogclient.EncodeModule(meshcatalog.ModulesByModuleValue{Fp2: m.trunk, Manifest: mdigest, Repository: "novox/" + m.name, Commit: strings.Repeat("1", 40), Path: meshcatalogclient.RootPath, Src: m.src})
|
||||
putModuleRaw(p.w, trunkBranch, m.name, mod)
|
||||
p.w.manifests[mdigest] = parsed{module: m.name, manifest: catalogue.Manifest{Module: m.name}}
|
||||
endDone(t, p.w, m.name, m.trunk, mdigest)
|
||||
gkey, _ := grantsstate.PartGrantKey(m.name, m.trunk)
|
||||
grant, _ := grantsstate.EncodeGrant(grantsstate.Grant{})
|
||||
p.w.put(buckets.Grants, gkey, grant)
|
||||
f.runs(m.name, m.run)
|
||||
wkey, _ := mwc.WantKey("ace", m.name)
|
||||
old, _ := mwc.EncodeWant(mwc.Want{Fp: m.run, Self: true})
|
||||
p.w.put(buckets.Wants, wkey, old)
|
||||
f.wants.values[wkey] = old
|
||||
}
|
||||
composes(t, p.w, "ace", digest)
|
||||
values, err := mmac.AssignmentsValues(pl)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for key, raw := range values {
|
||||
p.w.put(plc.AssignmentsBucket, key, raw)
|
||||
}
|
||||
p.w.heardPresence("ace", true, time.Now())
|
||||
objects := memObjects{}
|
||||
for i, at := range history {
|
||||
half := meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: int64(i + 1), Source: int64(i + 1)},
|
||||
Entries: map[string]meshcatalog.CatalogEntry{}}
|
||||
for _, module := range modules {
|
||||
digest := fmt.Sprintf("sha256:%064x", (i+1)*1000+len(half.Entries))
|
||||
half.Revision.Entries = append(half.Revision.Entries, meshcatalog.EntryRef{Module: module, Digest: digest})
|
||||
half.Entries[module] = meshcatalog.CatalogEntry{Module: module, Fp: at[module]}
|
||||
}
|
||||
placed := mma.Assignments{Node: "ace"}
|
||||
for _, module := range modules {
|
||||
placed.Modules = append(placed.Modules, mma.PlacedModule{Module: module})
|
||||
}
|
||||
in := assigngraph.Inputs{Halves: map[string]meshcatalogclient.Half{trunkBranch: half}, FirstBranch: func(string) string { return trunkBranch },
|
||||
Placements: map[string]mma.Assignments{"ace": placed}}
|
||||
if _, _, err := assigngraph.Keep(ctx, objects, assigngraph.Compute(in), "test", time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
p.graph = &graphKeeper{objects: objects}
|
||||
p.entries = meshcatalogclient.NewEntries(func(_ string, body any) (json.RawMessage, error) {
|
||||
in := body.(meshcatalog.EntryInput)
|
||||
entry := meshcatalog.CatalogEntry{Module: "m", Fp: in.Fingerprint, Manifest: map[string]any{"module": "m"}}
|
||||
if form := forms[in.Fingerprint]; form > 0 {
|
||||
entry.Manifest = map[string]any{"module": "m", "state": []any{map[string]any{"name": "sessions", "form": form}}}
|
||||
}
|
||||
return json.Marshal(meshcatalog.EntryAnswer{Entry: entry})
|
||||
})
|
||||
p.entries.Changed = func(meshcatalogclient.EntryRow) { f.answers <- struct{}{} }
|
||||
return f
|
||||
}
|
||||
|
||||
// runs says module runs fp healthy on ace, heard as the node-engine seat's running and in the have.
|
||||
func (f *orderFixture) runs(module, fp string) {
|
||||
have, _ := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Composed: "sha256:" + strings.Repeat("c", 64), Phase: nodeengineclient.PhaseRunning, Health: nodeengineclient.HealthOK})
|
||||
f.p.w.put(buckets.Have, "ace."+module, have)
|
||||
f.p.w.running["ace."+module] = nodeengine.Running{Node: "ace", Module: module, Fingerprint: fp}
|
||||
}
|
||||
|
||||
// pass makes a pass, and after it every catalog entry it asked is answered.
|
||||
func (f *orderFixture) pass() {
|
||||
f.t.Helper()
|
||||
f.p.wants(context.Background())
|
||||
asked := 0
|
||||
for _, row := range f.p.entries.Rows() {
|
||||
if row.State == meshcatalogclient.EntryAsked {
|
||||
asked++
|
||||
}
|
||||
}
|
||||
for range asked {
|
||||
select {
|
||||
case <-f.answers:
|
||||
case <-time.After(10 * time.Second): // a bound on a failure only: the answers come at once
|
||||
f.t.Fatal("an entry asked was not answered")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// wantAt is the fingerprint module is wanted at on ace.
|
||||
func (f *orderFixture) wantAt(module string) string {
|
||||
f.t.Helper()
|
||||
key, _ := mwc.WantKey("ace", module)
|
||||
w, err := mwc.DecodeWant(f.wants.values[key])
|
||||
if err != nil {
|
||||
f.t.Fatal(err)
|
||||
}
|
||||
return w.Fp
|
||||
}
|
||||
|
||||
func (f *orderFixture) waits() []string {
|
||||
var out []string
|
||||
for key := range f.p.w.read(buckets.Waits) {
|
||||
if acting.IsOrder(key) {
|
||||
out = append(out, key)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Design 47 G §298 first edge, A-258, A-259: a module whose kept data's form changes is not moved until the new version of the node-backup
|
||||
// holder and of the holder of the seat mesh-broker run; the wait is kept under `order.` in the state waits, said as wait-held, and ends
|
||||
// ran (wait-ended) when the holders run; a module whose kept data keeps its form is moved at once.
|
||||
func TestAModuleWhoseKeptDataChangesFormWaitsForTheNewBackupHolderAndTheBusModule(t *testing.T) {
|
||||
f := newOrderFixture(t, []orderMod{
|
||||
{name: "web", run: orderFP("a"), trunk: orderFP("b"), forms: map[string]int{orderFP("a"): 1, orderFP("b"): 2}},
|
||||
{name: "plain", run: orderFP("c"), trunk: orderFP("d"), forms: map[string]int{orderFP("c"): 1, orderFP("d"): 1}},
|
||||
{name: "backup", run: orderFP("e"), trunk: orderFP("f"), holds: []string{catalogue.BackupSeat}},
|
||||
{name: "bus", run: orderFP("g"), trunk: orderFP("h"), holds: []string{meshbroker.Seat}},
|
||||
})
|
||||
f.pass() // the entries are asked: no want crosses a form change unread
|
||||
if got := f.wantAt("web"); got != orderFP("a") {
|
||||
t.Fatalf("web is wanted at %s before the entries are read", got)
|
||||
}
|
||||
f.pass()
|
||||
if got := f.wantAt("web"); got != orderFP("a") {
|
||||
t.Fatalf("web moved across a form change at %s while the new node-backup holder and bus module do not run", got)
|
||||
}
|
||||
if got := f.wantAt("plain"); got != orderFP("d") {
|
||||
t.Errorf("a module whose kept data keeps its form is wanted at %s, not moved", got)
|
||||
}
|
||||
if len(f.waits()) != 2 || len(f.events) != 2 || f.events[0] != meshwants.EventWaitHeld || f.bodies[0].Step != seatdef.StepPlace || f.bodies[0].Module != "web" {
|
||||
t.Fatalf("waits %v events %v bodies %+v: want two wait-held in the step place, `order.ace.web.backup` and `order.ace.web.bus`", f.waits(), f.events, f.bodies)
|
||||
}
|
||||
if _, err := acting.Decode(f.p.w.read(buckets.Waits)[acting.OrderKey("ace", "web", "backup")]); err != nil {
|
||||
t.Fatalf("the wait is not kept under its key: %v", err)
|
||||
}
|
||||
f.pass()
|
||||
if len(f.events) != 2 {
|
||||
t.Fatalf("a pass that changed nothing said %v", f.events)
|
||||
}
|
||||
f.runs("backup", orderFP("f"))
|
||||
f.pass()
|
||||
if got := f.wantAt("web"); got != orderFP("a") || len(f.events) != 3 || f.events[2] != meshwants.EventWaitEnded || f.bodies[2].How != acting.HowRan {
|
||||
t.Fatalf("with the bus module still old: web at %s, events %v", got, f.events)
|
||||
}
|
||||
f.runs("bus", orderFP("h"))
|
||||
f.pass()
|
||||
f.pass()
|
||||
if got := f.wantAt("web"); got != orderFP("b") || len(f.waits()) != 0 || len(f.events) != 4 {
|
||||
t.Fatalf("with both running: web at %s, waits %v, events %v", got, f.waits(), f.events)
|
||||
}
|
||||
}
|
||||
|
||||
// §298 first edge, the hand: assigner_waits reads an order wait and assigner_release with how `now` lets the module move.
|
||||
func TestAnOrderWaitIsReadAndReleasedByTheExistingTools(t *testing.T) {
|
||||
f := newOrderFixture(t, []orderMod{
|
||||
{name: "web", run: orderFP("a"), trunk: orderFP("b"), forms: map[string]int{orderFP("a"): 1, orderFP("b"): 2}},
|
||||
{name: "backup", run: orderFP("e"), trunk: orderFP("f"), holds: []string{catalogue.BackupSeat}},
|
||||
})
|
||||
f.pass()
|
||||
f.pass()
|
||||
var acts []string
|
||||
s := newPutTools(memStore{waitsState: f.p.kvs[buckets.Waits].(*memKV), buckets.DeliveryWaitsState: &memKV{values: map[string][]byte{}}, buckets.DeliveredAgainState: &memKV{values: map[string][]byte{}}}, func(key string, body any) (json.RawMessage, error) {
|
||||
acts = append(acts, key)
|
||||
return json.RawMessage(`{"ok":true,"log":"logged"}`), nil
|
||||
})
|
||||
got, err := s.doWaits(map[string]any{"module": "web"})
|
||||
if list := got.(map[string]any)["waits"].([]any); err != nil || len(list) != 1 || list[0].(actingSaid).Key != "order.ace.web.backup" {
|
||||
t.Fatalf("assigner_waits = %+v %v", got, err)
|
||||
}
|
||||
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "web", "how": "now", "why": "the restore point is taken by hand"}, "shanks.node-tools"); err != nil || len(acts) == 0 {
|
||||
t.Fatalf("%v %v", err, acts)
|
||||
}
|
||||
f.p.w.put(buckets.Waits, "order.ace.web.backup", f.p.kvs[buckets.Waits].(*memKV).values["order.ace.web.backup"])
|
||||
f.pass()
|
||||
f.pass()
|
||||
if got := f.wantAt("web"); got != orderFP("b") || f.bodies[len(f.bodies)-1].How != acting.HowReleased {
|
||||
t.Fatalf("after the release: web at %s, last %+v", got, f.bodies[len(f.bodies)-1])
|
||||
}
|
||||
}
|
||||
|
||||
// §298 second edge, A-259: at a revert the node-engine goes back last on its node, after every module that is set back there across a
|
||||
// form change runs the reverted build.
|
||||
func TestAtARevertTheNodeEngineGoesBackLast(t *testing.T) {
|
||||
old, now := map[string]string{"web": orderFP("a"), "engine": orderFP("c")}, map[string]string{"web": orderFP("b"), "engine": orderFP("d")}
|
||||
f := newOrderFixture(t, []orderMod{
|
||||
{name: "web", run: orderFP("b"), trunk: orderFP("a"), forms: map[string]int{orderFP("a"): 1, orderFP("b"): 2}},
|
||||
{name: "engine", run: orderFP("d"), trunk: orderFP("c"), holds: []string{nodeengine.Seat}},
|
||||
}, old, now)
|
||||
f.pass()
|
||||
f.pass()
|
||||
if f.wantAt("web") != orderFP("a") {
|
||||
t.Fatalf("web is wanted at %s: the module set back is not held", f.wantAt("web"))
|
||||
}
|
||||
if got := f.wantAt("engine"); got != orderFP("d") {
|
||||
t.Fatalf("the node-engine is set back to %s before web runs its reverted build", got)
|
||||
}
|
||||
if w := f.waits(); len(w) != 1 || w[0] != "order.ace.engine.web" || f.bodies[len(f.bodies)-1].Module != "engine" {
|
||||
t.Fatalf("waits %v bodies %+v", w, f.bodies)
|
||||
}
|
||||
f.runs("web", orderFP("a"))
|
||||
f.pass()
|
||||
f.pass()
|
||||
if got := f.wantAt("engine"); got != orderFP("c") || len(f.waits()) != 0 {
|
||||
t.Fatalf("with web back: the node-engine is wanted at %s, waits %v", got, f.waits())
|
||||
}
|
||||
}
|
||||
|
||||
// §298 third edge, A-259, A-260: the reverted assigner and gate are wanted only once the indexer's take-in.taken of the trunk branch names a
|
||||
// source revision at least the trunk's head (the highest src of the trunk's module values).
|
||||
func TestTheRevertedAssignerAndGateWaitForTheIndexerToTakeInTheTrunksHead(t *testing.T) {
|
||||
old, now := map[string]string{"asg": orderFP("a"), "gate": orderFP("c"), "idx": orderFP("e")}, map[string]string{"asg": orderFP("b"), "gate": orderFP("d"), "idx": orderFP("f")}
|
||||
f := newOrderFixture(t, []orderMod{
|
||||
{name: "asg", run: orderFP("b"), trunk: orderFP("a"), holds: []string{mma.Seat}, src: 6},
|
||||
{name: "gate", run: orderFP("d"), trunk: orderFP("c"), holds: []string{meshdelivery.Seat}, src: 7},
|
||||
{name: "idx", run: orderFP("e"), trunk: orderFP("e"), src: 7},
|
||||
}, old, now)
|
||||
f.pass()
|
||||
f.pass()
|
||||
if f.wantAt("asg") != orderFP("b") || f.wantAt("gate") != orderFP("d") {
|
||||
t.Fatalf("the reverted assigner (%s) and gate (%s) are wanted before the head is taken in", f.wantAt("asg"), f.wantAt("gate"))
|
||||
}
|
||||
if len(f.waits()) != 2 || f.bodies[0].Part != "mesh-delivery-indexer" {
|
||||
t.Fatalf("waits %v bodies %+v", f.waits(), f.bodies)
|
||||
}
|
||||
taken := func(branch string, source int64) {
|
||||
raw, _ := json.Marshal(meshcatalog.TakeInTaken{Kind: "taken", Key: "m.1", Branch: branch, Source: source})
|
||||
if err := f.p.w.taken.put(raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
taken("beta", 9) // another branch, and
|
||||
taken(trunkBranch, 6)
|
||||
f.pass()
|
||||
if f.wantAt("asg") != orderFP("b") {
|
||||
t.Fatal("a source revision before the trunk's head was taken in, and the assigner was wanted")
|
||||
}
|
||||
taken(trunkBranch, 7)
|
||||
f.pass()
|
||||
f.pass()
|
||||
if f.wantAt("asg") != orderFP("a") || f.wantAt("gate") != orderFP("c") || len(f.waits()) != 0 {
|
||||
t.Fatalf("after the head is taken in: asg %s gate %s waits %v", f.wantAt("asg"), f.wantAt("gate"), f.waits())
|
||||
}
|
||||
}
|
||||
|
||||
// A-260: an order wait still held after a restart is said again as wait-held, by a pass that remembers nothing, and ends at the next
|
||||
// take-in.taken that names the trunk's head.
|
||||
func TestAnOrderWaitHeldAfterARestartIsSaidAgain(t *testing.T) {
|
||||
old, now := map[string]string{"asg": orderFP("a")}, map[string]string{"asg": orderFP("b")}
|
||||
f := newOrderFixture(t, []orderMod{{name: "asg", run: orderFP("b"), trunk: orderFP("a"), holds: []string{mma.Seat}, src: 7}}, old, now)
|
||||
f.pass()
|
||||
f.pass()
|
||||
if len(f.events) != 1 {
|
||||
t.Fatalf("events %v", f.events)
|
||||
}
|
||||
f.p.orderSaid = nil // a process started again
|
||||
f.pass()
|
||||
if len(f.events) != 2 || f.events[1] != meshwants.EventWaitHeld || len(f.waits()) != 1 {
|
||||
t.Fatalf("events %v waits %v", f.events, f.waits())
|
||||
}
|
||||
raw, _ := json.Marshal(meshcatalog.TakeInTaken{Kind: "taken", Key: "m.1", Branch: trunkBranch, Source: 8})
|
||||
if err := f.p.w.taken.put(raw); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f.pass()
|
||||
f.pass()
|
||||
if f.wantAt("asg") != orderFP("a") || len(f.waits()) != 0 || f.events[len(f.events)-1] != meshwants.EventWaitEnded {
|
||||
t.Fatalf("asg at %s, waits %v, events %v", f.wantAt("asg"), f.waits(), f.events)
|
||||
}
|
||||
}
|
||||
@@ -267,10 +267,10 @@ func TestTheToolsReadAndReleaseAnOutsideWait(t *testing.T) {
|
||||
s.st.put(waitsState, k, v)
|
||||
}
|
||||
got, err := s.doWaits(map[string]any{"module": "nats"})
|
||||
if err != nil || len(got.(map[string]any)["waits"].([]waitSaid)) != 1 {
|
||||
if err != nil || len(got.(map[string]any)["waits"].([]any)) != 1 {
|
||||
t.Fatalf("assigner_waits = %+v %v", got, err)
|
||||
}
|
||||
if w := got.(map[string]any)["waits"].([]waitSaid)[0]; w.On != "unreachable:hub" || w.BoundEnds == "" {
|
||||
if w := got.(map[string]any)["waits"].([]any)[0].(waitSaid); w.On != "unreachable:hub" || w.BoundEnds == "" {
|
||||
t.Errorf("wait = %+v", w)
|
||||
}
|
||||
if _, err := s.doRelease(map[string]any{"node": "ace", "module": "nats", "how": "sideways", "why": "x"}, "shanks.node-tools"); err == nil {
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
stdio "git.novox.be/novox/mesh-sdk/go"
|
||||
"git.novox.be/novox/mesh-sdk/go/keys"
|
||||
|
||||
"mesh-delivery-assigner/internal/acting"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
"mesh-delivery-assigner/internal/outside"
|
||||
)
|
||||
@@ -36,6 +37,13 @@ type waitSaid struct {
|
||||
BoundEnds string `json:"bound-ends,omitempty"`
|
||||
}
|
||||
|
||||
// actingSaid is one wait for an acting part's new version as assigner_waits answers it (actingwaits.go).
|
||||
type actingSaid struct {
|
||||
Key string `json:"key"`
|
||||
acting.Wait
|
||||
BoundEnds string `json:"bound-ends,omitempty"`
|
||||
}
|
||||
|
||||
// doWaits is the tool assigner_waits.
|
||||
func (s *putTools) doWaits(args map[string]any) (any, error) {
|
||||
keys, err := s.st.keys(waitsState)
|
||||
@@ -43,12 +51,22 @@ func (s *putTools) doWaits(args map[string]any) (any, error) {
|
||||
return nil, fmt.Errorf("the waits cannot be listed: %w", err)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
out := []waitSaid{}
|
||||
out := []any{}
|
||||
for _, key := range keys {
|
||||
raw, _, found, err := s.st.get(waitsState, key)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
if acting.Is(key) || acting.IsOrder(key) {
|
||||
if a, err := acting.Decode(raw); err == nil && (arg(args, "node") == "" || a.Node == arg(args, "node")) && (arg(args, "module") == "" || a.Module == arg(args, "module")) {
|
||||
said := actingSaid{Key: key, Wait: a}
|
||||
if d, err := time.ParseDuration(a.Bound); err == nil && !a.Released {
|
||||
said.BoundEnds = a.ClockFrom.Add(d).UTC().Format("2006-01-02T15:04:05Z")
|
||||
}
|
||||
out = append(out, said)
|
||||
}
|
||||
continue
|
||||
}
|
||||
w, err := outside.Decode(raw)
|
||||
if err != nil || (arg(args, "node") != "" && w.Node != arg(args, "node")) || (arg(args, "module") != "" && w.Module != arg(args, "module")) {
|
||||
continue
|
||||
@@ -172,6 +190,9 @@ func (s *putTools) doReleaseDelivery(args map[string]any, by string) (any, error
|
||||
// module, which the assigner's process takes at its next pass.
|
||||
func (s *putTools) doReleaseWait(args map[string]any, by string) (any, error) {
|
||||
node, module, how, why := arg(args, "node"), arg(args, "module"), arg(args, "how"), arg(args, "why")
|
||||
if how == acting.HowNow {
|
||||
return s.doReleaseActing(node, module, why, by)
|
||||
}
|
||||
if !slices.Contains([]string{outside.HowGoOn, outside.HowFailed, outside.HowRejudge}, how) {
|
||||
return nil, fmt.Errorf("how is %s, %s or %s, not %q", outside.HowGoOn, outside.HowFailed, outside.HowRejudge, how)
|
||||
}
|
||||
@@ -217,9 +238,61 @@ func (s *putTools) doReleaseWait(args map[string]any, by string) (any, error) {
|
||||
return map[string]any{"released": key, "how": how, "by": by, "hand-act": taken.Log}, nil
|
||||
}
|
||||
|
||||
// doReleaseActing is assigner_release with how `now`: the entry of module on node moves with its acting parts' running versions. The
|
||||
// hand act is recorded first; the release is written on each standing wait of the entry, and the process says wait-ended at its next pass.
|
||||
func (s *putTools) doReleaseActing(node, module, why, by string) (any, error) {
|
||||
if why == "" {
|
||||
return nil, errors.New("release says why: it is kept in the hand-act log")
|
||||
}
|
||||
keys, err := s.st.keys(waitsState)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the waits cannot be listed: %w", err)
|
||||
}
|
||||
type one struct {
|
||||
key string
|
||||
w acting.Wait
|
||||
}
|
||||
var held []one
|
||||
prefix, orderPrefix := acting.Prefix+node+"."+module+".", acting.OrderPrefix+node+"."+module+"."
|
||||
for _, k := range keys {
|
||||
if !strings.HasPrefix(k, prefix) && !strings.HasPrefix(k, orderPrefix) {
|
||||
continue
|
||||
}
|
||||
raw, _, found, err := s.st.get(waitsState, k)
|
||||
if err != nil || !found {
|
||||
continue
|
||||
}
|
||||
if w, err := acting.Decode(raw); err == nil && !w.Released {
|
||||
held = append(held, one{k, w})
|
||||
}
|
||||
}
|
||||
if len(held) == 0 {
|
||||
return nil, fmt.Errorf("%s holds no wait of %s for an acting part: assigner_waits lists them", node, module)
|
||||
}
|
||||
rec := meshhandactsclient.Recorder{Ask: s.ask}
|
||||
taken, err := rec.Record(meshhandactsclient.Request{What: fmt.Sprintf("assigner_release %s %s %s", node, module, acting.HowNow), Why: why,
|
||||
Cause: "acting-wait", Verb: "mesh-wants.assigner_release"})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the hand act could not be recorded, so nothing was released: %w", err)
|
||||
}
|
||||
released := []string{}
|
||||
for _, h := range held {
|
||||
h.w.Released, h.w.By, h.w.Why = true, by+": "+why, why
|
||||
data, err := acting.Encode(h.w)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.st.put(waitsState, h.key, data); err != nil {
|
||||
return nil, fmt.Errorf("the release of %s on %s is not written: %w", module, node, err)
|
||||
}
|
||||
released = append(released, h.key)
|
||||
}
|
||||
return map[string]any{"released": released, "how": acting.HowNow, "by": by, "hand-act": taken.Log}, nil
|
||||
}
|
||||
|
||||
// waitsTool is assigner_waits as the runtime serves it.
|
||||
func (s *putTools) waitsTool() stdio.Tool {
|
||||
return stdio.Tool{Name: toolWaits, Description: "Every wait of the delivery on an outside cause and where it stands: a moved module's failing health check on a " +
|
||||
return stdio.Tool{Name: toolWaits, Description: "Every wait of the delivery and where it stands: an entry waiting for an acting part's new version in composing or placing (§266, §268), with its part, version, step, clock-from and bound; and a moved module's failing health check on a " +
|
||||
"machine whose open conditions name a cause outside the delivery (design 47 G §706), with its module, machine, fingerprint, the " +
|
||||
"condition's key, what it stands `on`, its clock-from, bound, its ask and its state; a settle shows as settling with its end. Narrowed by " +
|
||||
"machine or module. Read-only.",
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-grants/grantsstate"
|
||||
@@ -103,6 +104,14 @@ type passer struct {
|
||||
publishWants publisher
|
||||
// graph keeps the assignment graph revision the wants follow (graphkept.go, design 47 G §292, A-67); nil: none is kept.
|
||||
graph *graphKeeper
|
||||
// orders are §298's order edges computed in this pass, by their wait's key, and orderVisited the modules they were computed for (orderwaits.go).
|
||||
orders map[string]orderEdge
|
||||
orderVisited map[string]bool
|
||||
// orderSaid are the order waits whose wait-held this process said.
|
||||
orderSaid map[string]bool
|
||||
// entries are the catalog entries by fingerprint asked of the seat mesh-catalog's verb `entry` (design 47 G §614, A-217): a
|
||||
// held revert's running entry and its want's (revert.go); nil: no revert is asked about.
|
||||
entries *meshcatalogclient.Entries
|
||||
}
|
||||
|
||||
// wantsSay says one event of the seat mesh-wants; with no bus to say it on, nothing is said.
|
||||
@@ -207,10 +216,12 @@ func (p *passer) pass(ctx context.Context) {
|
||||
p.keepGraph(ctx)
|
||||
p.judgeDelivered()
|
||||
p.deliveredAgain(ctx)
|
||||
p.wants(ctx)
|
||||
if p.graph == nil || !p.graph.full { // a full assignment graph store keeps the last revision and writes no new want (design 47 G §181)
|
||||
p.wants(ctx)
|
||||
}
|
||||
p.puts(ctx)
|
||||
p.nodeWants(ctx)
|
||||
p.backups(ctx)
|
||||
p.backups(ctx, p.reverts(ctx))
|
||||
}
|
||||
|
||||
// nodeWants writes each placed node's own want, `want.<node>.node`: the node composes its own (ADR 0547 §2, ADR 0571 §6).
|
||||
@@ -286,6 +297,10 @@ func (p *passer) wants(ctx context.Context) {
|
||||
defer func() { p.sayNeither(neither, stale, left) }()
|
||||
// Where grants are read from, once for the pass: the grants part's (issue 774, ADR 0571 clause 9).
|
||||
grants := p.w.grantsBucket()
|
||||
released := p.actingReleased()
|
||||
p.orders, p.orderVisited = map[string]orderEdge{}, map[string]bool{}
|
||||
defer func() { p.actingWaits(ctx, viewOf(p.w.trunkName()).nodes, wanted) }()
|
||||
defer func() { p.orderWaits(ctx, viewOf(p.w.trunkName()).nodes, wanted) }()
|
||||
for name := range names {
|
||||
groups := p.groups(name, nodesOf[name], assignedBranches[name])
|
||||
for gi, g := range groups {
|
||||
@@ -356,6 +371,9 @@ func (p *passer) wants(ctx context.Context) {
|
||||
m.MachineHeld = p.outside.Held
|
||||
nodes = p.outsideReview(ctx, name, fp, m.Nodes, nodes, manifests[name].ClaimsSeat(mcseat.Seat), p.movedTo(wanted, nodes))
|
||||
}
|
||||
unread := p.orderEdgesOf(ctx, m, wantedOf[name], trunk)
|
||||
actingOf, orderOf := p.actingOf(name, released), p.orderActing(name, released, unread, func() assign.Nodes { return nodes })
|
||||
m.Acting = func(node string) []assign.ActingEdge { return append(actingOf(node), orderOf(node)...) }
|
||||
out := assign.Wants(m, nodes, wantedOf[name], last)
|
||||
for _, node := range m.Nodes {
|
||||
for _, prov := range m.MissingProvisions(node) {
|
||||
@@ -744,7 +762,7 @@ func (p *passer) lift(ctx context.Context, module, fp string) {
|
||||
|
||||
// staleAsk says why the hold, restore or record ask at key (read as the SDK reads it) is about something the trunk has left
|
||||
// behind, or "" when it is not: a module no longer on the trunk, a fingerprint that is no longer the module's trunk fingerprint,
|
||||
// or, for a record ask, a composition the module's node no longer waits on. A backup ask is stale when the module's want on its
|
||||
// or, for a record ask, a composition the module's node no longer waits on. A backup ask, and a revert ask (design 47 G §710), is stale when the module's want on its
|
||||
// node names another fingerprint or is removed (the move it is about was superseded; issue 1966 note 165); a want not written
|
||||
// yet, or at the same fingerprint in another composition, leaves it (backups asks that again under the next number). An ask of
|
||||
// another kind is never stale here. A module of the trunk whose fingerprint is not read yet, and a have not read yet, are not stale: nothing known is
|
||||
@@ -755,7 +773,7 @@ func (p *passer) staleAsk(key string, trunk map[string]meshcatalog.ModulesByModu
|
||||
return ""
|
||||
}
|
||||
switch k.Kind {
|
||||
case asks.AskKindBackup:
|
||||
case asks.AskKindBackup, asks.AskKindRevert:
|
||||
wkey, err := mwc.WantKey(k.Node, k.Module)
|
||||
if err != nil {
|
||||
return ""
|
||||
@@ -876,6 +894,10 @@ func (p *passer) staleAsks(ctx context.Context, trunk map[string]meshcatalog.Mod
|
||||
}
|
||||
delete(p.said, assign.BackupCondition(k.Node, k.Module, k.FP, "", false).Key)
|
||||
}
|
||||
// A revert ask's finding goes with it: a newer want replaced the held revert (design 47 G §710, §363).
|
||||
if kerr == nil && k.Kind == asks.AskKindRevert {
|
||||
p.revertFinding(assign.RevertHeldFinding(k.Node, k.Module, "", "", ""), false)
|
||||
}
|
||||
p.logf("the ask %s is about a fingerprint or composition the trunk has left behind and is taken away: %s", key, why)
|
||||
p.remove(ctx, buckets.Asks, key)
|
||||
// The held record stays: it is the memory that this fingerprint failed (ADR 0408 §4), and a revert returns the trunk to
|
||||
@@ -995,7 +1017,9 @@ func (p *passer) recordAsk(ctx context.Context, module, fp string, need assign.R
|
||||
// before the `@`). Each answer is another want value (the asking's number is in the want's words), so a backup that fails again
|
||||
// after "back up again" and is answered "again" again changes the key and the node retries. That the backup failed again is seen
|
||||
// in a new have value after the answer was written.
|
||||
func (p *passer) backups(ctx context.Context) {
|
||||
//
|
||||
// A have that holds a revert (reverts, which answers its `<node>.<module>` in reverting) is left to the revert ask.
|
||||
func (p *passer) backups(ctx context.Context, reverting map[string]bool) {
|
||||
wanted := p.w.wantsOf()
|
||||
for key, value := range p.w.read(buckets.Have) {
|
||||
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
|
||||
@@ -1003,7 +1027,7 @@ func (p *passer) backups(ctx context.Context) {
|
||||
continue
|
||||
}
|
||||
have, err := nodeengineclient.DecodeHave(value)
|
||||
if err != nil || have.Phase != nodeengineclient.PhaseHeld {
|
||||
if err != nil || have.Phase != nodeengineclient.PhaseHeld || reverting[node+"."+module] {
|
||||
continue
|
||||
}
|
||||
want, ok := wanted[module][node]
|
||||
|
||||
@@ -334,8 +334,9 @@ func TestTheCatalogHalfIsTakenInPerBranch(t *testing.T) {
|
||||
woken := 0
|
||||
h := &halves{w: w, wake: func() { woken++ }, logf: t.Logf}
|
||||
entry := func(fp string) json.RawMessage {
|
||||
raw, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "web", Fp: fp, Repository: "novox/web", Commit: "c1", Manifest: map[string]any{"module": "web", "version": "1"},
|
||||
Invokes: []string{"seat:x.v"}})
|
||||
raw, _ := json.Marshal(meshcatalog.CatalogEntry{Module: "web", Fp: fp, Repository: "novox/web", Commit: "c1",
|
||||
Manifest: map[string]any{"module": "web", "version": "1", "requires": []any{"database", map[string]any{"name": "cache", "optional": true}}},
|
||||
Invokes: []string{"seat:x.v"}, Requires: []string{"cache", "database"}})
|
||||
return raw
|
||||
}
|
||||
h.changed("main", "web", entry("src2:1"), false)
|
||||
@@ -347,7 +348,8 @@ func TestTheCatalogHalfIsTakenInPerBranch(t *testing.T) {
|
||||
if got := w.manifest(v.Manifest); got.err != "" || got.manifest.Module != "web" {
|
||||
t.Errorf("the entry's manifest is not read by its digest: %+v", got)
|
||||
}
|
||||
if n := w.graphOf("main")["web"]; n.FP != "src2:1" || !slices.Equal(n.Invokes, []string{"seat:x.v"}) {
|
||||
// The entry's requires name the optional cache too; the graph node means those required, not optionally (A-229).
|
||||
if n := w.graphOf("main")["web"]; n.FP != "src2:1" || !slices.Equal(n.Invokes, []string{"seat:x.v"}) || !slices.Equal(n.Requires, []string{"database"}) {
|
||||
t.Errorf("graph node %+v", n)
|
||||
}
|
||||
h.changed("beta", "web", nil, true)
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
|
||||
mcc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||
"git.novox.be/novox/mesh-sdk/go/views"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// revertReasons are the reasons a node's have gives when it holds a revert (design 47 G §300, §567).
|
||||
var revertReasons = []string{"restore point missing", "restore point damaged"}
|
||||
|
||||
// revertReason is the reason of a have's words when they say a held revert, or "".
|
||||
func revertReason(words string) string {
|
||||
for _, r := range revertReasons {
|
||||
if strings.HasPrefix(words, r) {
|
||||
return r
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// reverts sees to each revert a node holds with no restore point (design 47 G §300, §710): the have's phase held with the
|
||||
// reason `restore point missing` or `restore point damaged`, for a want the assignment graph tells a revert (assigngraph.Reverted,
|
||||
// A-217) across a kept data's form (assigngraph.FormChanges, the running entry and the want's read through the catalog seat's verb
|
||||
// `entry`). It asks the operator hold or live, raises the finding revert-held, and on live writes the want again with `revert`
|
||||
// live; hold writes nothing. The finding is cleared once the reverted build runs. It answers the `<node>.<module>` of each have
|
||||
// that holds a revert, which backups leaves to it.
|
||||
func (p *passer) reverts(ctx context.Context) map[string]bool {
|
||||
taken := map[string]bool{}
|
||||
wanted := p.w.wantsOf()
|
||||
for key, value := range p.w.read(buckets.Have) {
|
||||
node, module, isNode, err := nodeengineclient.ParseHaveKey(key)
|
||||
if err != nil || isNode {
|
||||
continue
|
||||
}
|
||||
have, err := nodeengineclient.DecodeHave(value)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
want, ok := wanted[module][node]
|
||||
if !ok || want.Removed {
|
||||
continue
|
||||
}
|
||||
akey, err := asks.RevertAskKey(node, module, want.Fp)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
reason := revertReason(have.Detail)
|
||||
if have.Phase != nodeengineclient.PhaseHeld || reason == "" {
|
||||
if have.Fp == want.Fp && p.w.has(buckets.Asks, akey) {
|
||||
p.revertFinding(assign.RevertHeldFinding(node, module, "", "", ""), false) // the reverted build runs
|
||||
}
|
||||
continue
|
||||
}
|
||||
taken[node+"."+module] = true
|
||||
kept, form, ok := p.revertCrosses(ctx, node, module, want.Fp)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
current, currentAsk, there := assign.DecodeAsk(akey, p.w.read(buckets.Asks)[akey])
|
||||
if there != nil {
|
||||
now := p.clock()
|
||||
a := assign.RevertAsk(akey, node, module, want.Fp, kept, form, have.Detail, 0, now)
|
||||
p.put(ctx, buckets.Asks, akey, func() ([]byte, error) { return assign.EncodeAsk(akey, a) })
|
||||
p.revertFinding(assign.RevertHeldFinding(node, module, kept, form, reason), true)
|
||||
continue
|
||||
}
|
||||
p.revertFinding(assign.RevertHeldFinding(node, module, kept, form, reason), true)
|
||||
answer, answered := p.w.answer(akey)
|
||||
if router, _, _ := strings.Cut(answer.Warrant, "@"); !answered || router != currentAsk.ID || answer.Option != assign.OptionRevertLive {
|
||||
continue // not answered, or answered hold: the hold stands and nothing is written
|
||||
}
|
||||
words := assign.RevertAnswerWords(answer.Option, current.Asking)
|
||||
if want.Revert != mwc.RevertLive || want.Words != words {
|
||||
want.Revert, want.Words = mwc.RevertLive, words
|
||||
p.putWant(ctx, node, module, want)
|
||||
}
|
||||
}
|
||||
return taken
|
||||
}
|
||||
|
||||
// revertCrosses reads whether the want of module on node at wantFp is a revert across a kept data's form: the kept data's names
|
||||
// and the want's forms of them, said as the ask and the finding name them. ok is false while it is not known (the entries not
|
||||
// answered yet, a pass with the assignment graph not kept) or when the want is no revert across a form.
|
||||
func (p *passer) revertCrosses(ctx context.Context, node, module, wantFp string) (kept, form string, ok bool) {
|
||||
if p.graph == nil || p.entries == nil {
|
||||
return "", "", false
|
||||
}
|
||||
run, heard := p.w.runningOf(node, module)
|
||||
if !heard || run.Stopped || run.Fingerprint == "" {
|
||||
return "", "", false
|
||||
}
|
||||
reverted, err := assigngraph.Reverted(ctx, p.graph.objects, node, module, run.Fingerprint, wantFp)
|
||||
if err != nil {
|
||||
p.logf("whether the want of %s on %s at %s is a revert is not read: %v", module, node, wantFp, err)
|
||||
return "", "", false
|
||||
}
|
||||
if !reverted {
|
||||
return "", "", false
|
||||
}
|
||||
changes, known := p.formChange(ctx, module, run.Fingerprint, wantFp)
|
||||
if !known {
|
||||
return "", "", false // asked: its answer wakes a pass
|
||||
}
|
||||
var names, forms []string
|
||||
for _, c := range changes {
|
||||
f := 0
|
||||
if c.Applied != nil {
|
||||
f = c.Applied.Form
|
||||
}
|
||||
names, forms = append(names, c.Name), append(forms, strconv.Itoa(f))
|
||||
}
|
||||
if len(names) == 0 {
|
||||
return "", "", false
|
||||
}
|
||||
return strings.Join(names, ", "), strings.Join(forms, ", "), true
|
||||
}
|
||||
|
||||
// revertFinding writes the finding revert-held, or clears it, once per process each way.
|
||||
func (p *passer) revertFinding(o assign.Observed, raised bool) {
|
||||
key := views.FindingKey(o.Scope, assign.KeyID(o.ID), o.Kind)
|
||||
said := key + ".raised"
|
||||
if !raised {
|
||||
said = key + ".cleared"
|
||||
}
|
||||
if p.said[said] {
|
||||
return
|
||||
}
|
||||
var err error
|
||||
switch {
|
||||
case raised && p.findings.Put != nil:
|
||||
err = p.findings.Put(o.Finding(p.clock()))
|
||||
case !raised && p.findings.Delete != nil:
|
||||
err = p.findings.Delete(key)
|
||||
default:
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
p.logf("the finding %s is not written: %v", key, err)
|
||||
return
|
||||
}
|
||||
p.said[said] = true
|
||||
delete(p.said, map[bool]string{true: key + ".cleared", false: key + ".raised"}[raised])
|
||||
}
|
||||
|
||||
// formChange reads the kept data whose form differs between module's entry at runFp and at wantFp (design 47 G §295 to §298, A-217), as
|
||||
// the catalog seat's verb `entry` answers the two entries; known is false while either is asked, and woken by its answer.
|
||||
func (p *passer) formChange(ctx context.Context, module, runFp, wantFp string) (changes []catalogue.FormChange, known bool) {
|
||||
if p.entries == nil {
|
||||
return nil, false
|
||||
}
|
||||
running, rs := p.entries.Need(ctx, runFp)
|
||||
applied, as := p.entries.Need(ctx, wantFp)
|
||||
if rs == mcc.EntryAsked || as == mcc.EntryAsked {
|
||||
return nil, false
|
||||
}
|
||||
if rs != mcc.EntryAnswered || as != mcc.EntryAnswered {
|
||||
return nil, true // the catalog holds no such entry: no kept data's form is read from it, so none changes
|
||||
}
|
||||
changes, err := assigngraph.FormChanges(running, applied)
|
||||
if err != nil {
|
||||
p.logf("the forms of %s's kept data between %s and %s are not read: %v", module, runFp, wantFp, err)
|
||||
return nil, false
|
||||
}
|
||||
return changes, true
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner"
|
||||
"git.novox.be/novox/mesh-contracts/go/seats/nodeengine"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-assigner/asks"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshcatalogclient"
|
||||
mdc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshdeliveryclient"
|
||||
mwc "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/clients/meshwantsclient"
|
||||
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-contracts/go/seats/meshcatalog"
|
||||
|
||||
"mesh-delivery-assigner/internal/assign"
|
||||
"mesh-delivery-assigner/internal/assigngraph"
|
||||
"mesh-delivery-assigner/internal/buckets"
|
||||
)
|
||||
|
||||
// Design 47 G §300, §710, A-217: a node holding a revert with no restore point gets the revert ask, keyed by the reverted build,
|
||||
// and the finding revert-held, never a backup ask; hold writes no want; live writes the want with `revert` live and its words;
|
||||
// the finding clears once the reverted build runs; a newer want takes the ask away and clears the finding.
|
||||
func TestAHeldRevertIsAskedAndALiveAnswerIsWrittenIntoTheWant(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
at := time.Date(2026, 11, 12, 10, 0, 0, 0, time.UTC)
|
||||
fpOld := meshcatalogclient.SourcePrefix2 + strings.Repeat("a", 64)
|
||||
asksKV, wantsKV := &memKV{values: map[string][]byte{}}, &memKV{values: map[string][]byte{}}
|
||||
p := newPasser(kvSet{buckets.Asks: asksKV, buckets.Wants: wantsKV}, at)
|
||||
p.logf = t.Logf
|
||||
findings := map[string][]byte{}
|
||||
p.findings = assign.Findings{Put: putInto(findings), Delete: func(k string) error { delete(findings, k); return nil }}
|
||||
|
||||
// Two assignment graph revisions: web on ace at fpOld, then at fpTest, which it runs.
|
||||
objects := memObjects{}
|
||||
for i, fp := range []string{fpOld, fpTest} {
|
||||
half := meshcatalogclient.Half{Revision: meshcatalog.CatalogRevision{Branch: trunkBranch, Kind: "trunk", Revision: int64(i + 1), Source: int64(i + 1),
|
||||
Entries: []meshcatalog.EntryRef{{Module: "web", Digest: "sha256:" + strings.Repeat(string(rune('c'+i)), 64)}}},
|
||||
Entries: map[string]meshcatalog.CatalogEntry{"web": {Module: "web", Fp: fp}}}
|
||||
in := assigngraph.Inputs{Halves: map[string]meshcatalogclient.Half{trunkBranch: half}, FirstBranch: func(string) string { return trunkBranch },
|
||||
Placements: map[string]meshmoduleassigner.Assignments{"ace": {Node: "ace", Modules: []meshmoduleassigner.PlacedModule{{Module: "web"}}}}}
|
||||
if _, _, err := assigngraph.Keep(ctx, objects, assigngraph.Compute(in), "test", at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
p.graph = &graphKeeper{objects: objects}
|
||||
// The entries by fingerprint, as the catalog seat's verb entry answers them: the kept data sessions at form 1, then form 2.
|
||||
entry := func(fp string, form int) meshcatalog.CatalogEntry {
|
||||
return meshcatalog.CatalogEntry{Module: "web", Fp: fp, Manifest: map[string]any{"module": "web",
|
||||
"state": []any{map[string]any{"name": "sessions", "form": form}}}}
|
||||
}
|
||||
answered := make(chan struct{}, 4)
|
||||
p.entries = meshcatalogclient.NewEntries(func(_ string, body any) (json.RawMessage, error) {
|
||||
in := body.(meshcatalog.EntryInput)
|
||||
return json.Marshal(meshcatalog.EntryAnswer{Entry: map[string]meshcatalog.CatalogEntry{fpOld: entry(fpOld, 1), fpTest: entry(fpTest, 2)}[in.Fingerprint]})
|
||||
})
|
||||
p.entries.Changed = func(meshcatalogclient.EntryRow) { answered <- struct{}{} }
|
||||
|
||||
p.w.running["ace.web"] = nodeengine.Running{Node: "ace", Module: "web", Fingerprint: fpTest}
|
||||
wkey, _ := mwc.WantKey("ace", "web")
|
||||
want := mwc.Want{Fp: fpOld, Self: true}
|
||||
raw, _ := mwc.EncodeWant(want)
|
||||
p.w.put(buckets.Wants, wkey, raw)
|
||||
hkey, _ := nodeengineclient.HaveKey("ace", "web")
|
||||
have := func(fp, phase, detail string) {
|
||||
raw, err := nodeengineclient.EncodeHave(nodeengine.HaveByNodeModuleValue{Fp: fp, Phase: phase, Detail: detail, Health: nodeengineclient.HealthOK})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.w.put(buckets.Have, hkey, raw)
|
||||
}
|
||||
have(fpTest, nodeengineclient.PhaseHeld, "restore point missing: web's sessions has none of form 1 taken before "+fpOld)
|
||||
pass := func() { p.backups(ctx, p.reverts(ctx)) }
|
||||
|
||||
pass() // the entries are asked; nothing is asked of the operator until they answer
|
||||
for range 2 {
|
||||
select {
|
||||
case <-answered:
|
||||
case <-time.After(10 * time.Second): // a bound on a failure only: the answers come at once
|
||||
t.Fatalf("the entries were not asked: asks %v", asksKV.values)
|
||||
}
|
||||
}
|
||||
pass()
|
||||
rkey, _ := asks.RevertAskKey("ace", "web", fpOld)
|
||||
bkey, _ := asks.BackupAskKey("ace", "web", fpOld)
|
||||
if !p.w.has(buckets.Asks, rkey) || p.w.has(buckets.Asks, bkey) {
|
||||
t.Fatalf("asks %v: want the revert ask %s and no backup ask", asksKV.values, rkey)
|
||||
}
|
||||
_, ask, err := assign.DecodeAsk(rkey, p.w.read(buckets.Asks)[rkey])
|
||||
if err != nil || !strings.Contains(ask.Explanation, "sessions") || !strings.Contains(ask.Explanation, "form 1") {
|
||||
t.Fatalf("the revert ask = %+v, %v", ask, err)
|
||||
}
|
||||
fkey := "delivery.ace.web.revert-held"
|
||||
if _, raised := findings[fkey]; !raised {
|
||||
t.Fatalf("findings %v: want %s", findings, fkey)
|
||||
}
|
||||
|
||||
answer := func(option string) {
|
||||
v, _ := mdc.EncodeAnswer(mdc.Answer{Option: option, Warrant: ask.ID + "@2026-11-12T10:05:00Z", Words: "the operator"})
|
||||
p.w.put(buckets.Answers, rkey, v)
|
||||
}
|
||||
puts := wantsKV.puts
|
||||
answer(assign.OptionRevertHold)
|
||||
pass()
|
||||
if wantsKV.puts != puts {
|
||||
t.Fatal("an answer hold wrote a want")
|
||||
}
|
||||
answer(assign.OptionRevertLive)
|
||||
pass()
|
||||
got, err := mwc.DecodeWant(p.w.read(buckets.Wants)[wkey])
|
||||
if err != nil || got.Fp != fpOld || got.Revert != mwc.RevertLive || got.Words != `the operator answered "live" to the revert ask (asking 1)` {
|
||||
t.Fatalf("the want after live = %+v, %v", got, err)
|
||||
}
|
||||
|
||||
have(fpOld, nodeengineclient.PhaseRunning, "") // the reverted build runs: the finding clears
|
||||
pass()
|
||||
if _, raised := findings[fkey]; raised {
|
||||
t.Fatal("the finding stands while the reverted build runs")
|
||||
}
|
||||
|
||||
// A newer want replaces the held revert: the ask goes and the finding with it.
|
||||
have(fpTest, nodeengineclient.PhaseHeld, "restore point damaged: the point does not read")
|
||||
pass()
|
||||
if _, raised := findings[fkey]; !raised {
|
||||
t.Fatal("the finding is not raised again for a hold again")
|
||||
}
|
||||
raw, _ = mwc.EncodeWant(mwc.Want{Fp: fpTest, Self: true})
|
||||
p.w.put(buckets.Wants, wkey, raw)
|
||||
p.staleAsks(ctx, map[string]meshcatalog.ModulesByModuleValue{"web": {Fp2: fpTest}})
|
||||
if _, raised := findings[fkey]; raised || p.w.has(buckets.Asks, rkey) {
|
||||
t.Fatalf("a newer want left the ask (%v) or the finding (%v)", p.w.has(buckets.Asks, rkey), raised)
|
||||
}
|
||||
}
|
||||
@@ -348,7 +348,7 @@ func (s *putTools) tools() []stdio.Tool {
|
||||
"rules write the machine's want again from their next pass. Nothing changes on the machine until they move it.",
|
||||
Input: schema(map[string]string{"node": "the machine", "module": "the module", "why": "why, kept with the put",
|
||||
"how": "with how, the wait of the delivery on an outside cause (assigner_waits) is released, not a put: go-on (the module counts healthy there), " +
|
||||
"failed (the failing check is the delivery's) or rejudge (judge it again now); written in the hand-act log first",
|
||||
"failed (the failing check is the delivery's), rejudge (judge it again now), or now (the entry waiting for an acting part's new version moves with the running one); written in the hand-act log first",
|
||||
"step": "delivery: the module's wait for its dependency's delivery (assigner_waits) is released, its want written now; no machine is named; written in the hand-act log first"},
|
||||
[]string{"module", "why"}),
|
||||
Run: func(a map[string]any) (any, error) { return s.doRelease(a, stdio.CallerOf(a)) }},
|
||||
|
||||
@@ -47,6 +47,8 @@ type world struct {
|
||||
delivered deliveredVersions
|
||||
// edges are, by watched branch and module, the edges into the module's artifacts its catalog entry names (§374's kinds).
|
||||
edges map[string]map[string][]meshcatalog.Edge
|
||||
// taken is the newest source revision of each watched branch the indexer's take-in.taken named, as heard (orderwaits.go, §298, A-260).
|
||||
taken takenSources
|
||||
// assignments is the seat's state assignments assembled node by node from its keys in either form (mesh-sdk
|
||||
// mmac.Keys; research 072 row 37): the one reader of its keys in the assigner.
|
||||
assignmentKeys *mmac.Keys
|
||||
|
||||
Reference in New Issue
Block a user