Say what a walk touches, its bound and its outcome when it starts and ends (ADR 0308, issue 532)
The router holds what an open walk touches and tells its result once; it can do so only if plan-moved names the walk's start and end, the modules and machines it moves, what a moved tool runner or node-engine carries there, and how long to hold.
This commit is contained in:
jochen
@@ -0,0 +1,306 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"github.com/novox/mesh-controller/internal/broker"
|
||||
"github.com/novox/mesh-controller/internal/catalogue"
|
||||
"github.com/novox/mesh-controller/internal/inventory"
|
||||
"github.com/novox/mesh-controller/internal/link"
|
||||
"github.com/novox/mesh-controller/internal/testbus"
|
||||
)
|
||||
|
||||
// fakeWalkFacts is the inventory as far as a walk's touches read it.
|
||||
type fakeWalkFacts struct {
|
||||
running map[string][]string
|
||||
records map[string]bool // modules whose policy records rather than rolls out
|
||||
sent map[string]map[string]string
|
||||
assigned map[string][]string
|
||||
shelf map[string]catalogue.Manifest
|
||||
}
|
||||
|
||||
func (f fakeWalkFacts) Running(_ context.Context, m string) ([]string, error) {
|
||||
return f.running[m], nil
|
||||
}
|
||||
func (f fakeWalkFacts) UpgradeOf(_ context.Context, m string) (inventory.Upgrade, error) {
|
||||
return inventory.Upgrade{RollOut: !f.records[m]}, nil
|
||||
}
|
||||
func (f fakeWalkFacts) Assigned(_ context.Context, n string) ([]string, error) {
|
||||
return f.assigned[n], nil
|
||||
}
|
||||
func (f fakeWalkFacts) SentBuilds(_ context.Context, n string) (map[string]string, bool, error) {
|
||||
s, ok := f.sent[n]
|
||||
return s, ok, nil
|
||||
}
|
||||
func (f fakeWalkFacts) Catalogue(context.Context) (map[string]catalogue.Manifest, error) {
|
||||
return f.shelf, nil
|
||||
}
|
||||
|
||||
// theMesh is four machines: the tool runner on each, messenger and mesh-issues on novox, gitea on ace.
|
||||
func theMesh() fakeWalkFacts {
|
||||
return fakeWalkFacts{
|
||||
running: map[string][]string{
|
||||
broker.RuntimeModule: {"ace", "g14", "novox", "shanks"},
|
||||
"messenger": {"novox"},
|
||||
"gitea": {"ace"},
|
||||
"plex": {"ace"},
|
||||
},
|
||||
sent: map[string]map[string]string{
|
||||
"novox": {broker.RuntimeModule: "c0", "messenger": "c1", "mesh-issues": "c2", "postgres": "c3"},
|
||||
"ace": {broker.RuntimeModule: "c0", "gitea": "c4"},
|
||||
},
|
||||
assigned: map[string][]string{"g14": {broker.RuntimeModule, "desktop"}, "shanks": {broker.RuntimeModule}},
|
||||
shelf: map[string]catalogue.Manifest{
|
||||
"messenger": {Module: "messenger", Tools: []string{"notify"}},
|
||||
"mesh-issues": {Module: "mesh-issues", Bundles: []catalogue.Bundle{{Name: "tools", Loads: []string{"main"}}}},
|
||||
"postgres": {Module: "postgres"},
|
||||
"gitea": {Module: "gitea", Tools: []string{"gitea_api"}},
|
||||
"desktop": {Module: "desktop", Tools: []string{"notify_send"}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func walkOfTiers(class string, tiers ...[]string) inventory.Plan {
|
||||
cut := time.Date(2026, 10, 12, 11, 0, 0, 0, time.UTC)
|
||||
started := cut.Add(time.Second)
|
||||
p := inventory.Plan{ID: "plan-308", State: inventory.PlanBuilding, Tiers: tiers,
|
||||
Modules: map[string]*inventory.PlanModule{}, Times: &inventory.PlanTimes{Cut: &cut, Class: class, Started: &started}}
|
||||
for _, tier := range tiers {
|
||||
for _, m := range tier {
|
||||
p.Modules[m] = &inventory.PlanModule{}
|
||||
}
|
||||
}
|
||||
p.WalkSaid = inventory.WalkStarted
|
||||
return p
|
||||
}
|
||||
|
||||
// novox/hq ADR 0308 How it is checked §1: the start names the modules the walk moves and the machines it sends
|
||||
// them to, and every module a moved tool runner carries there.
|
||||
func TestAWalksStartNamesWhatItTouchesWithWhatTheToolRunnerCarries(t *testing.T) {
|
||||
p := walkOfTiers(inventory.ClassCore, []string{broker.RuntimeModule}, []string{"messenger"})
|
||||
w := sayWalk(t.Context(), theMesh(), p)
|
||||
if w == nil || w.Walk != "plan-308" || w.Phase != inventory.WalkStarted || w.Outcome != "" {
|
||||
t.Fatalf("the start is said: %+v", w)
|
||||
}
|
||||
want := []walkTouch{
|
||||
{Machine: "ace", Modules: []string{"gitea", broker.RuntimeModule}, Carried: []string{"gitea"}},
|
||||
{Machine: "g14", Modules: []string{"desktop", broker.RuntimeModule}, Carried: []string{"desktop"}},
|
||||
// postgres serves no tools: the tool runner does not carry it.
|
||||
{Machine: "novox", Modules: []string{"mesh-issues", "messenger", broker.RuntimeModule}, Carried: []string{"mesh-issues"}},
|
||||
{Machine: "shanks", Modules: []string{broker.RuntimeModule}},
|
||||
}
|
||||
if !reflect.DeepEqual(w.Touches, want) {
|
||||
t.Fatalf("touches:\n got %+v\nwant %+v", w.Touches, want)
|
||||
}
|
||||
if w.BudgetClass != inventory.ClassCore || w.BoundSeconds != 1200 || w.StartedAt == nil {
|
||||
t.Fatalf("a core walk is bound at 20 minutes: %+v", w)
|
||||
}
|
||||
}
|
||||
|
||||
// A moved node-engine carries every module on its machine; a module whose policy records is not sent, and not
|
||||
// touched.
|
||||
func TestAMovedNodeEngineCarriesEverythingOnItsMachine(t *testing.T) {
|
||||
f := theMesh()
|
||||
f.running[hostModule] = []string{"novox"}
|
||||
f.records = map[string]bool{"plex": true}
|
||||
p := walkOfTiers(inventory.ClassCore, []string{hostModule, "plex"})
|
||||
w := sayWalk(t.Context(), f, p)
|
||||
want := []walkTouch{{Machine: "novox",
|
||||
Modules: []string{"mesh-host", "mesh-issues", "messenger", "node-tools", "postgres"},
|
||||
Carried: []string{"mesh-issues", "messenger", "node-tools", "postgres"}}}
|
||||
if !reflect.DeepEqual(w.Touches, want) {
|
||||
t.Fatalf("touches:\n got %+v\nwant %+v", w.Touches, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The end names what the walk reached — its first machines, its rest, what a gated send carried — and its
|
||||
// outcome; the modules a reached tool runner carries are touched there too.
|
||||
func TestAWalksEndNamesWhatItReachedAndItsOutcome(t *testing.T) {
|
||||
p := walkOfTiers(inventory.ClassCore, []string{broker.RuntimeModule}, []string{"messenger"})
|
||||
ended := p.Times.Started.Add(4 * time.Minute)
|
||||
p.Times.Ended = &ended
|
||||
p.State, p.WalkSaid = inventory.PlanDone, inventory.WalkEnded
|
||||
p.Modules[broker.RuntimeModule] = &inventory.PlanModule{First: []string{"shanks"},
|
||||
Rest: map[string]inventory.SentDeclaration{"novox": {}},
|
||||
Gate: &inventory.PlanGate{Carried: []inventory.CarriedMove{{Module: "gitea", Node: "shanks"}}}}
|
||||
// messenger was built and found unchanged: no move, not reached.
|
||||
p.Modules["messenger"] = &inventory.PlanModule{Why: "no move"}
|
||||
w := sayWalk(t.Context(), theMesh(), p)
|
||||
if w.Phase != inventory.WalkEnded || w.Outcome != inventory.WalkDelivered || w.EndedAt == nil {
|
||||
t.Fatalf("the end is said with its outcome: %+v", w)
|
||||
}
|
||||
want := []walkTouch{
|
||||
{Machine: "novox", Modules: []string{"mesh-issues", "messenger", broker.RuntimeModule},
|
||||
Carried: []string{"mesh-issues", "messenger"}},
|
||||
{Machine: "shanks", Modules: []string{"gitea", broker.RuntimeModule}},
|
||||
}
|
||||
if !reflect.DeepEqual(w.Touches, want) {
|
||||
t.Fatalf("touches:\n got %+v\nwant %+v", w.Touches, want)
|
||||
}
|
||||
for outcome, end := range map[string]func(*inventory.Plan){
|
||||
inventory.WalkFailed: func(p *inventory.Plan) { p.State = inventory.PlanFailed },
|
||||
inventory.WalkStopped: func(p *inventory.Plan) {
|
||||
p.State, p.Delivery = inventory.PlanFailed, &inventory.PlanDelivery{Stopped: "a person"}
|
||||
},
|
||||
} {
|
||||
q := p
|
||||
end(&q)
|
||||
if got := sayWalk(t.Context(), theMesh(), q).Outcome; got != outcome {
|
||||
t.Errorf("outcome %q, want %q", got, outcome)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// novox/hq ADR 0308 How it is checked §1 and decision 5: the bound is twice the largest budget in force among
|
||||
// the walk's deliveries — 20 minutes for a core walk, twice the leaf budget for a leaf walk — and 15 minutes, not
|
||||
// doubled, for a walk none of whose deliveries has a budget.
|
||||
func TestAWalksBoundIsTwiceItsBudgetOrFifteenMinutesWithNone(t *testing.T) {
|
||||
f := theMesh()
|
||||
f.records = map[string]bool{"plex": true}
|
||||
release := walkOfTiers("", []string{"gitea"})
|
||||
release.Times, release.Release = &inventory.PlanTimes{}, &inventory.PlanRelease{Order: []string{"ace"}}
|
||||
for name, c := range map[string]struct {
|
||||
p inventory.Plan
|
||||
class string
|
||||
bound int
|
||||
}{
|
||||
"core": {walkOfTiers(inventory.ClassCore, []string{"messenger"}), inventory.ClassCore, 1200},
|
||||
"leaf": {walkOfTiers(inventory.ClassLeaf, []string{"messenger"}), inventory.ClassLeaf, int(2 * deliveryBudgets[inventory.ClassLeaf] / time.Second)},
|
||||
"held for a person": {walkOfTiers(inventory.ClassLeaf, []string{"plex"}), budgetNone, 900},
|
||||
"a release": {release, budgetNone, 900},
|
||||
"kept without a class": {walkOfTiers("", []string{"messenger"}), budgetNone, 900},
|
||||
} {
|
||||
w := sayWalk(t.Context(), f, c.p)
|
||||
if w.BudgetClass != c.class || w.BoundSeconds != c.bound {
|
||||
t.Errorf("%s: %s bound %ds, want %s %ds", name, w.BudgetClass, w.BoundSeconds, c.class, c.bound)
|
||||
}
|
||||
}
|
||||
// The leaf budget in force is three minutes (ADR 0304 decision 4): a leaf walk is bound at six.
|
||||
if deliveryBudgets[inventory.ClassLeaf] == 3*time.Minute {
|
||||
if w := sayWalk(t.Context(), f, walkOfTiers(inventory.ClassLeaf, []string{"messenger"})); w.BoundSeconds != 360 {
|
||||
t.Errorf("a leaf walk at a three-minute budget is bound at 360 s: %d", w.BoundSeconds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// plan-moved's body: the walk whole as before, and on its start the walk's fields beside it; on any other save
|
||||
// none of them.
|
||||
func TestPlanMovedCarriesTheWalkWholeAndOnItsStartWhatItTouches(t *testing.T) {
|
||||
p := walkOfTiers(inventory.ClassLeaf, []string{"messenger"})
|
||||
body, err := json.Marshal(planMovedBody{Plan: p, walkSaid: sayWalk(t.Context(), theMesh(), p)})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var said map[string]any
|
||||
if err := json.Unmarshal(body, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, k := range []string{"id", "state", "tiers", "modules", "times", "walk", "phase", "touches", "bound_seconds", "budget_class", "started_at"} {
|
||||
if _, ok := said[k]; !ok {
|
||||
t.Errorf("plan-moved lacks %q: %s", k, body)
|
||||
}
|
||||
}
|
||||
if _, ok := said["outcome"]; ok {
|
||||
t.Errorf("a start says no outcome: %s", body)
|
||||
}
|
||||
if said["walk"] != "plan-308" || said["phase"] != "started" || said["budget_class"] != "leaf" {
|
||||
t.Errorf("the walk's fields: %s", body)
|
||||
}
|
||||
p.WalkSaid = ""
|
||||
body, _ = json.Marshal(planMovedBody{Plan: p, walkSaid: sayWalk(t.Context(), theMesh(), p)})
|
||||
said = map[string]any{}
|
||||
_ = json.Unmarshal(body, &said)
|
||||
for _, k := range []string{"walk", "phase", "touches", "bound_seconds"} {
|
||||
if _, ok := said[k]; ok {
|
||||
t.Errorf("a save neither starting nor ending the walk says %q: %s", k, body)
|
||||
}
|
||||
}
|
||||
// A walk that touches nothing says an empty list, never null.
|
||||
q := walkOfTiers(inventory.ClassLeaf, []string{"nobody-runs-this"})
|
||||
body, _ = json.Marshal(planMovedBody{Plan: q, walkSaid: sayWalk(t.Context(), theMesh(), q)})
|
||||
said = map[string]any{}
|
||||
_ = json.Unmarshal(body, &said)
|
||||
if touches, ok := said["touches"].([]any); !ok || len(touches) != 0 {
|
||||
t.Errorf("touches nothing: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
// Live, with the store and a bus: a walk let go by the delivery's verb is said started with what it touches and
|
||||
// its bound, and stopped by the verb is said ended, stopped (novox/hq ADR 0308 How it is checked §1).
|
||||
func TestAWalkLetGoAndStoppedByVerbsIsSaidStartedAndEnded(t *testing.T) {
|
||||
open := aMesh(t)
|
||||
ctx := t.Context()
|
||||
url := testbus.URL(t)
|
||||
js, err := broker.Dial(url)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(js.Close)
|
||||
if err := broker.AssertMeshStreams(js); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before := handActConn
|
||||
handActConn = js.Conn()
|
||||
t.Cleanup(func() { handActConn = before })
|
||||
heard := make(chan *nats.Msg, 8)
|
||||
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
register(t, open, catalogue.Manifest{Module: "app", Version: "1"})
|
||||
if _, err := assign(ctx, open, "anchor", "app"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cut := time.Now().UTC()
|
||||
waiting := inventory.Plan{ID: "plan-308-live", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c8c8c8c8",
|
||||
Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
||||
Modules: map[string]*inventory.PlanModule{"app": {}},
|
||||
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat},
|
||||
Times: &inventory.PlanTimes{Cut: &cut, Class: inventory.ClassLeaf}}
|
||||
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
next := func(phase string) map[string]any {
|
||||
t.Helper()
|
||||
deadline := time.After(5 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case m := <-heard:
|
||||
var said map[string]any
|
||||
if err := json.Unmarshal(m.Data, &said); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if said["phase"] == phase {
|
||||
return said
|
||||
}
|
||||
case <-deadline:
|
||||
t.Fatalf("the walk was not said %s", phase)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
started := next(inventory.WalkStarted)
|
||||
touches, _ := json.Marshal(started["touches"])
|
||||
if started["walk"] != waiting.ID || string(touches) != `[{"machine":"anchor","modules":["app"]}]` ||
|
||||
started["budget_class"] != inventory.ClassLeaf ||
|
||||
started["bound_seconds"] != float64(2*deliveryBudgets[inventory.ClassLeaf]/time.Second) {
|
||||
t.Fatalf("the start: %v", started)
|
||||
}
|
||||
if err := deliveryCommand(ctx, []string{"stop", waiting.ID, "--why", "a test", "--by", "a person"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ended := next(inventory.WalkEnded)
|
||||
touches, _ = json.Marshal(ended["touches"])
|
||||
if ended["walk"] != waiting.ID || ended["outcome"] != inventory.WalkStopped || string(touches) != `[]` {
|
||||
t.Fatalf("the end, which sent nothing: %v", ended)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user