jochen
c5a2edf04a
The group feat/a-machine-joins-through-the-tunnel was refused: the controller's member moved the build agent, which builds the node-engine (built by: controller first), and the node-engine goes before the controller (engine before controller: engine first). Both rules applied to one pair in opposite directions, and every two-way pair was a cycle. Rules now have a precedence (hq ADR 0249): a declared after: line, then what the graph and the change say (built by, version skew), then the rollout default engine-before-controller. The higher rule decides the pair, which says what it won over. Rules of one rank both ways are still refused, as a contradiction naming both rules and how to declare the order. TestReplay309 replays the group with only what orderOf had before.
391 lines
17 KiB
Go
391 lines
17 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"reflect"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/nats-io/nats.go"
|
|
|
|
"github.com/novox/mesh-controller/internal/broker"
|
|
"github.com/novox/mesh-controller/internal/catalogue"
|
|
"github.com/novox/mesh-controller/internal/inventory"
|
|
"github.com/novox/mesh-controller/internal/link"
|
|
"github.com/novox/mesh-controller/internal/testbus"
|
|
)
|
|
|
|
// novox/hq ADR 0239 decision 4: a group's order, from the graph and the pull requests, the same on every
|
|
// reading — the build agent before what it builds, the controller before a manifest that needs it, the
|
|
// node-engine before the controller, a declared `after:` — and a contradiction named, never ordered.
|
|
func TestAGroupIsOrderedByTheGraphAndWhatItsPullRequestsSay(t *testing.T) {
|
|
members := []orderMember{
|
|
{ID: "cat", Repository: "novox/mesh-catalog"},
|
|
{ID: "ctl", Repository: "novox/mesh-controller"},
|
|
{ID: "host", Repository: "novox/mesh-host"},
|
|
{ID: "agent", Repository: "novox/build-agent"},
|
|
{ID: "app", Repository: "novox/app", After: []string{"lab"}},
|
|
{ID: "lab", Repository: "novox/lab"},
|
|
}
|
|
reach := map[string]orderReach{
|
|
"cat": {Moved: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}},
|
|
"ctl": {Moved: []string{"mesh-controller"}, Manifests: []string{"module.json"}},
|
|
"host": {Moved: []string{"mesh-host"}, Manifests: []string{"module.json"}},
|
|
"agent": {Moved: []string{"build-agent"}},
|
|
"app": {Moved: []string{"app"}},
|
|
"lab": {Moved: []string{"lab"}},
|
|
}
|
|
edges := []inventory.Edge{{From: "app", To: "build-agent", Kind: inventory.EdgeBuiltBy},
|
|
{From: "gitea", To: "postgres", Kind: inventory.EdgeDeclared}}
|
|
got := orderOf(members, reach, edges)
|
|
if len(got.Cycle) > 0 {
|
|
t.Fatalf("a cycle where there is none: %v", got.Cycle)
|
|
}
|
|
// By repository among those with nothing before them: the build agent, the lab, then what waited on both.
|
|
want := []string{"agent", "lab", "app", "host", "ctl", "cat"}
|
|
if !reflect.DeepEqual(got.Order, want) {
|
|
t.Fatalf("ordered %v, wanted %v; pairs %+v", got.Order, want, got.Pairs)
|
|
}
|
|
why := map[string]string{}
|
|
for _, p := range got.Pairs {
|
|
why[p.Before+">"+p.After] = p.Why
|
|
}
|
|
for pair, reason := range map[string]string{"host>ctl": orderEngineFirst, "ctl>cat": orderVersionSkew,
|
|
"ctl>host": "", "agent>app": orderBuiltBy, "lab>app": orderDeclared} {
|
|
if why[pair] != reason {
|
|
t.Errorf("%s is ordered %q, wanted %q", pair, why[pair], reason)
|
|
}
|
|
}
|
|
// The same members in another order read the same.
|
|
shuffled := []orderMember{members[5], members[3], members[0], members[4], members[2], members[1]}
|
|
if again := orderOf(shuffled, reach, edges); !reflect.DeepEqual(again.Order, want) {
|
|
t.Fatalf("another reading ordered %v", again.Order)
|
|
}
|
|
|
|
// A declared order against an inferred one wins (novox/hq ADR 0249), and the pair says what it won over.
|
|
members[1].After = []string{"mesh-catalog"}
|
|
got = orderOf(members, reach, edges)
|
|
if len(got.Cycle) > 0 || len(got.Contradictions) > 0 {
|
|
t.Fatalf("a declared order did not win: cycle %v, %+v", got.Cycle, got.Contradictions)
|
|
}
|
|
if want := []string{"agent", "lab", "app", "cat", "host", "ctl"}; !reflect.DeepEqual(got.Order, want) {
|
|
t.Fatalf("ordered %v, wanted %v", got.Order, want)
|
|
}
|
|
if !slices.ContainsFunc(got.Pairs, func(p orderPair) bool {
|
|
return p.Before == "cat" && p.After == "ctl" && p.Why == orderDeclared && reflect.DeepEqual(p.Over, []string{orderVersionSkew})
|
|
}) || slices.ContainsFunc(got.Pairs, func(p orderPair) bool { return p.Before == "ctl" && p.After == "cat" }) {
|
|
t.Fatalf("the declared pair does not say it won over version skew: %+v", got.Pairs)
|
|
}
|
|
|
|
// Declared both ways is a contradiction no precedence resolves: a cycle, and how to end it.
|
|
members[0].After = []string{"novox/mesh-controller"}
|
|
got = orderOf(members, reach, edges)
|
|
if !reflect.DeepEqual(got.Cycle, []string{"cat", "ctl"}) || len(got.Contradictions) != 1 {
|
|
t.Fatalf("the cycle is %v, ordered %v, contradictions %+v", got.Cycle, got.Order, got.Contradictions)
|
|
}
|
|
if c := got.Contradictions[0]; !reflect.DeepEqual(c.Members, []string{"cat", "ctl"}) ||
|
|
!strings.Contains(c.Said, "remove one of the `after:` lines") {
|
|
t.Fatalf("the contradiction is said %+v", c)
|
|
}
|
|
}
|
|
|
|
// novox/hq ADR 0249: when two rules order one pair both ways, the rule of higher precedence wins and the
|
|
// pair says what it overruled; rules of one rank both ways are refused, naming both and how to declare.
|
|
func TestAnOrderRuleYieldsToAStrongerOne(t *testing.T) {
|
|
members := []orderMember{
|
|
{ID: "ctl", Repository: "novox/mesh-controller", Number: 7},
|
|
{ID: "host", Repository: "novox/mesh-host", Number: 9},
|
|
}
|
|
reach := map[string]orderReach{
|
|
"ctl": {Moved: []string{"mesh-controller", "build-agent"}},
|
|
"host": {Moved: []string{"mesh-host"}},
|
|
}
|
|
builtBy := []inventory.Edge{{From: "mesh-host", To: "build-agent", Kind: inventory.EdgeBuiltBy}}
|
|
|
|
// Built by outranks engine before controller.
|
|
got := orderOf(members, reach, builtBy)
|
|
if !reflect.DeepEqual(got.Order, []string{"ctl", "host"}) || len(got.Cycle) > 0 {
|
|
t.Fatalf("ordered %v, cycle %v", got.Order, got.Cycle)
|
|
}
|
|
if want := []orderPair{{Before: "ctl", After: "host", Why: orderBuiltBy, Over: []string{orderEngineFirst}}}; !reflect.DeepEqual(got.Pairs, want) {
|
|
t.Fatalf("pairs %+v, wanted %+v", got.Pairs, want)
|
|
}
|
|
|
|
// A declared line outranks both.
|
|
members[0].After = []string{"mesh-host"}
|
|
got = orderOf(members, reach, builtBy)
|
|
if !reflect.DeepEqual(got.Order, []string{"host", "ctl"}) || len(got.Cycle) > 0 ||
|
|
!reflect.DeepEqual(got.Pairs[0].Over, []string{orderBuiltBy}) || got.Pairs[0].Why != orderDeclared {
|
|
t.Fatalf("a declared order did not win: %v %+v", got.Order, got.Pairs)
|
|
}
|
|
|
|
// Without the build agent, engine before controller stands alone.
|
|
members[0].After = nil
|
|
got = orderOf(members, map[string]orderReach{"ctl": {Moved: []string{"mesh-controller"}}, "host": reach["host"]}, builtBy)
|
|
if !reflect.DeepEqual(got.Order, []string{"host", "ctl"}) || got.Pairs[0].Why != orderEngineFirst || len(got.Pairs[0].Over) > 0 {
|
|
t.Fatalf("engine before controller alone: %v %+v", got.Order, got.Pairs)
|
|
}
|
|
|
|
// Built by both ways is one rank against itself: refused, both rules named, and how to declare.
|
|
both := append(builtBy, inventory.Edge{From: "build-agent", To: "mesh-host", Kind: inventory.EdgeStandsOn})
|
|
got = orderOf(members, reach, both)
|
|
if !reflect.DeepEqual(got.Cycle, []string{"ctl", "host"}) || len(got.Order) > 0 || len(got.Contradictions) != 1 {
|
|
t.Fatalf("a contradiction was ordered: %v, cycle %v, %+v", got.Order, got.Cycle, got.Contradictions)
|
|
}
|
|
said := got.Contradictions[0].Said
|
|
for _, w := range []string{"built by puts ctl first", "built by puts host first",
|
|
"`after: novox/mesh-controller` in novox/mesh-host#9's description", "`after: novox/mesh-host` in novox/mesh-controller#7's"} {
|
|
if !strings.Contains(said, w) {
|
|
t.Errorf("the contradiction does not say %q: %s", w, said)
|
|
}
|
|
}
|
|
}
|
|
|
|
// novox/hq ADR 0239 decision 8: a walk waits for the delivery's owner only while the seat has a holder on
|
|
// record, and never one that moves a module on the controller's own path.
|
|
func TestAWalkWaitsOnlyWhileTheDeliverySeatIsHeldAndNeverForTheCore(t *testing.T) {
|
|
holder := inventory.Entry{Manifest: catalogue.Manifest{Module: "mesh-delivery",
|
|
Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}}
|
|
unassigned := holder
|
|
unassigned.On = nil
|
|
for _, c := range []struct {
|
|
entries []inventory.Entry
|
|
moved []string
|
|
waits bool
|
|
}{
|
|
{nil, []string{"gitea"}, false},
|
|
{[]inventory.Entry{unassigned}, []string{"gitea"}, false},
|
|
{[]inventory.Entry{holder}, []string{"gitea", "plex"}, true},
|
|
{[]inventory.Entry{holder}, []string{"gitea", "mesh-controller"}, false},
|
|
{[]inventory.Entry{holder}, []string{"mesh-host"}, false},
|
|
{[]inventory.Entry{holder}, []string{"node-tools"}, false},
|
|
{[]inventory.Entry{holder}, []string{"nats"}, false},
|
|
{[]inventory.Entry{holder}, []string{"mesh-delivery", "gitea"}, false},
|
|
} {
|
|
d := awaitsFor(c.entries, c.moved)
|
|
if (d != nil) != c.waits {
|
|
t.Errorf("held %v, moving %v: waits %v, wanted %v", len(c.entries) > 0 && len(c.entries[0].On) > 0,
|
|
c.moved, d != nil, c.waits)
|
|
}
|
|
if d != nil && d.Awaits != catalogue.DeliverySeat {
|
|
t.Errorf("waits for %q", d.Awaits)
|
|
}
|
|
}
|
|
}
|
|
|
|
// novox/hq ADR 0239: with mesh-delivery on record, a merge opens its walk and asks nothing until the
|
|
// delivery's word; the word starts it; the core's own merge never waits; a person starts a waiting walk by
|
|
// hand when the owner is down, and the owner's stop ends one as stopped.
|
|
func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
inv := open.inventory
|
|
asked := asksRecorded(t)
|
|
withConditionsInMemory(t)
|
|
for _, name := range []string{"app", "mesh-delivery"} {
|
|
m := catalogue.Manifest{Module: name, Version: "1"}
|
|
if name == "mesh-delivery" {
|
|
m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}
|
|
}
|
|
if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
|
|
Path: "modules/" + name, Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
merge := func(commit string, paths ...string) inventory.Plan {
|
|
t.Helper()
|
|
m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, Paths: paths,
|
|
ModuleDirs: []string{"modules/app", "modules/mesh-delivery"}, ModuleDirsSaid: true}
|
|
if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
recent, err := inv.RecentPlans(ctx, 1)
|
|
if err != nil || len(recent) != 1 || recent[0].Commit != commit {
|
|
t.Fatalf("no plan for %s: %v %v", commit, recent, err)
|
|
}
|
|
return recent[0]
|
|
}
|
|
|
|
// finish ends a walk as done, so the next merge has nothing of it to take over.
|
|
finish := func(id string) {
|
|
t.Helper()
|
|
w, err := inv.PlanByID(ctx, id)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
w.State = inventory.PlanDone
|
|
if err := inv.SavePlan(ctx, &w); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
// No holder on record: the merge starts its walk, as before.
|
|
p := merge("c1aaaaaaaa", "modules/app/index.ts")
|
|
if p.Waiting() || len(*asked) != 1 {
|
|
t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked)
|
|
}
|
|
|
|
// The holder on record: the next merge waits, asking nothing, and an advance asks nothing either.
|
|
if _, err := inv.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p = merge("c2bbbbbbbb", "modules/app/index.ts")
|
|
if !p.Waiting() || len(*asked) != 1 || !strings.Contains(p.Note, "plans go "+p.ID) {
|
|
t.Fatalf("the walk did not wait for its word: waiting %v, asked %v, note %q", p.Waiting(), *asked, p.Note)
|
|
}
|
|
advanceHeld(ctx, open)
|
|
if len(*asked) != 1 {
|
|
t.Fatalf("a waiting walk was advanced into asking: %v", *asked)
|
|
}
|
|
if line := planLine(p, p.Created); strings.Contains(line, "LATE") || !strings.Contains(line, "waits for") {
|
|
t.Errorf("a waiting walk reads %q", line)
|
|
}
|
|
|
|
// The delivery's word starts it.
|
|
if err := deliveryCommand(ctx, []string{"go", p.ID, "--by", catalogue.DeliverySeat, "--why", "its turn"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := deliveryCommand(ctx, []string{"go", p.ID}); err == nil {
|
|
t.Fatal("a walk was let go twice")
|
|
}
|
|
advanceHeld(ctx, open)
|
|
if len(*asked) != 2 {
|
|
t.Fatalf("the word did not start the walk: %v", *asked)
|
|
}
|
|
got, err := inv.PlanByID(ctx, p.ID)
|
|
if err != nil || got.Delivery == nil || got.Delivery.By != catalogue.DeliverySeat || got.Delivery.Why != "its turn" {
|
|
t.Fatalf("the word was not kept: %+v %v", got.Delivery, err)
|
|
}
|
|
|
|
// The delivery's owner's own merge never waits for it — and takes over what the older walk had not
|
|
// built (app, folded in: ADR 0218), which goes with it on the controller's own path.
|
|
p = merge("c3cccccccc", "modules/mesh-delivery/main.go")
|
|
if p.Waiting() || len(*asked) != 4 {
|
|
t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked)
|
|
}
|
|
|
|
// The owner down: a person starts a waiting walk, with why.
|
|
finish(p.ID)
|
|
p = merge("c4dddddddd", "modules/app/index.ts")
|
|
if !p.Waiting() {
|
|
t.Fatal("the walk did not wait")
|
|
}
|
|
if err := plansCommand(ctx, []string{"go", p.ID}); err == nil || !strings.Contains(err.Error(), "--why") {
|
|
t.Fatalf("a walk was started by hand without why: %v", err)
|
|
}
|
|
if err := plansCommand(ctx, []string{"go", p.ID, "--why", "mesh-delivery is down"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
advanceHeld(ctx, open)
|
|
got, _ = inv.PlanByID(ctx, p.ID)
|
|
if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 5 {
|
|
t.Fatalf("a person's word did not start the walk: %+v, asked %v", got.Delivery, *asked)
|
|
}
|
|
|
|
// The owner's stop: failed, said as stopped by it.
|
|
finish(p.ID)
|
|
p = merge("c5eeeeeeee", "modules/app/index.ts")
|
|
if err := deliveryCommand(ctx, []string{"stop", p.ID, "--why", "the operator stopped it", "--by",
|
|
"mesh-delivery for jochen"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, _ = inv.PlanByID(ctx, p.ID)
|
|
if got.State != inventory.PlanFailed || got.Delivery.Stopped != "mesh-delivery for jochen" ||
|
|
!strings.Contains(got.Note, "the operator stopped it") {
|
|
t.Fatalf("the stop was kept as %s %+v %q", got.State, got.Delivery, got.Note)
|
|
}
|
|
|
|
// Unassigned: the mesh is back on the controller's own path.
|
|
if err := inv.Unassign(ctx, "anchor", "mesh-delivery"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
asks := len(*asked)
|
|
if p = merge("c6ffffffff", "modules/app/index.ts"); p.Waiting() || len(*asked) != asks+1 {
|
|
t.Fatalf("with the holder gone the walk waited: %v", p.Waiting())
|
|
}
|
|
}
|
|
|
|
// The verbs mesh-delivery asks with become the commands they name, and nothing a caller sends is passed over.
|
|
func TestTheDeliveryVerbsComposeTheirCommands(t *testing.T) {
|
|
for _, c := range []struct {
|
|
verb string
|
|
args map[string]any
|
|
want []string
|
|
}{
|
|
{"deliver", map[string]any{"plan": "plan-1", "why": "its turn"},
|
|
[]string{"delivery", "go", "plan-1", "--by", "mesh-delivery", "--why", "its turn"}},
|
|
{"delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen"},
|
|
[]string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen"}},
|
|
{"delivery-walks", map[string]any{}, []string{"delivery", "walks"}},
|
|
{"delivery-walks", map[string]any{"plan": "plan-1"}, []string{"delivery", "walks", "--plan", "plan-1"}},
|
|
{"delivery-order", map[string]any{"members": "[]"}, []string{"delivery", "order", "--members", "[]"}},
|
|
{"delivery-check", map[string]any{"group": "feat/x", "members": "[]"},
|
|
[]string{"delivery", "check", "--group", "feat/x", "--members", "[]"}},
|
|
{"delivery-plan", map[string]any{"repository": "novox/a", "paths": "x", "head": "c0"},
|
|
[]string{"delivery", "plan", "--repository", "novox/a", "--paths", "x", "--head", "c0"}},
|
|
{"plans", map[string]any{"go": "plan-1", "why": "down"}, []string{"plans", "go", "plan-1", "--why", "down"}},
|
|
} {
|
|
got, err := argvFor(c.verb, c.args)
|
|
if err != nil || !reflect.DeepEqual(got, c.want) {
|
|
t.Errorf("%s %v → %v %v, wanted %v", c.verb, c.args, got, err, c.want)
|
|
}
|
|
}
|
|
if _, err := argvFor("deliver", map[string]any{}); err == nil {
|
|
t.Error("deliver without a walk was composed")
|
|
}
|
|
if _, err := argvFor("delivery-stop", map[string]any{"plan": "p"}); err == nil {
|
|
t.Error("a stop without why was composed")
|
|
}
|
|
}
|
|
|
|
// A verb runs as a command of its own: what it keeps of a walk is still said as plan-moved, on a bus of the
|
|
// command's own, so the delivery's owner hears it at once and not only when it reads the walks back.
|
|
func TestAWalkLetGoByAVerbIsSaidAsPlanMoved(t *testing.T) {
|
|
open := aMesh(t)
|
|
ctx := t.Context()
|
|
url := testbus.URL(t)
|
|
js, err := broker.Dial(url)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(js.Close)
|
|
if err := broker.AssertMeshStreams(js); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before := handActConn
|
|
handActConn = js.Conn()
|
|
t.Cleanup(func() { handActConn = before })
|
|
heard := make(chan *nats.Msg, 4)
|
|
sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
|
|
|
waiting := inventory.Plan{ID: "plan-waits", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c7c7c7c7",
|
|
Created: time.Now().UTC(), State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
|
|
Modules: map[string]*inventory.PlanModule{"app": {}},
|
|
Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
|
|
if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
select {
|
|
case m := <-heard:
|
|
var said inventory.Plan
|
|
if err := json.Unmarshal(m.Data, &said); err != nil || said.ID != waiting.ID || said.Delivery == nil ||
|
|
said.Delivery.Go == nil {
|
|
t.Fatalf("plan-moved said %s (%v)", m.Data, err)
|
|
}
|
|
case <-time.After(5 * time.Second):
|
|
t.Fatal("a walk let go by a verb was not said")
|
|
}
|
|
if checkEvents != nil || inventory.PlanSaved != nil {
|
|
t.Fatal("the command's own bus was left in place")
|
|
}
|
|
}
|