mesh-controller/cmd/mesh-controller/delivery_test.go

package main

import (
	"encoding/json"
	"reflect"
	"slices"
	"strings"
	"testing"
	"time"

	"github.com/nats-io/nats.go"

	"github.com/novox/mesh-controller/internal/broker"
	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
	"github.com/novox/mesh-controller/internal/testbus"
)

// novox/hq ADR 0239 decision 4: a group's order, from the graph and the pull requests, the same on every
// reading — the build agent before what it builds, the controller before a manifest that needs it, the
// node-engine before the controller, a declared `after:` — and a contradiction named, never ordered.
func TestAGroupIsOrderedByTheGraphAndWhatItsPullRequestsSay(t *testing.T) {
	members := []orderMember{
		{ID: "cat", Repository: "novox/mesh-catalog"},
		{ID: "ctl", Repository: "novox/mesh-controller"},
		{ID: "host", Repository: "novox/mesh-host"},
		{ID: "agent", Repository: "novox/build-agent"},
		{ID: "app", Repository: "novox/app", After: []string{"lab"}},
		{ID: "lab", Repository: "novox/lab"},
	}
	reach := map[string]orderReach{
		"cat":   {Moved: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}},
		"ctl":   {Moved: []string{"mesh-controller"}, Manifests: []string{"module.json"}},
		"host":  {Moved: []string{"mesh-host"}, Manifests: []string{"module.json"}},
		"agent": {Moved: []string{"build-agent"}},
		"app":   {Moved: []string{"app"}},
		"lab":   {Moved: []string{"lab"}},
	}
	edges := []inventory.Edge{{From: "app", To: "build-agent", Kind: inventory.EdgeBuiltBy},
		{From: "gitea", To: "postgres", Kind: inventory.EdgeDeclared}}
	got := orderOf(members, reach, edges)
	if len(got.Cycle) > 0 {
		t.Fatalf("a cycle where there is none: %v", got.Cycle)
	}
	// By repository among those with nothing before them: the build agent, the lab, then what waited on both.
	want := []string{"agent", "lab", "app", "host", "ctl", "cat"}
	if !reflect.DeepEqual(got.Order, want) {
		t.Fatalf("ordered %v, wanted %v; pairs %+v", got.Order, want, got.Pairs)
	}
	why := map[string]string{}
	for _, p := range got.Pairs {
		why[p.Before+">"+p.After] = p.Why
	}
	for pair, reason := range map[string]string{"host>ctl": orderEngineFirst, "ctl>cat": orderVersionSkew,
		"ctl>host": "", "agent>app": orderBuiltBy, "lab>app": orderDeclared} {
		if why[pair] != reason {
			t.Errorf("%s is ordered %q, wanted %q", pair, why[pair], reason)
		}
	}
	// The same members in another order read the same.
	shuffled := []orderMember{members[5], members[3], members[0], members[4], members[2], members[1]}
	if again := orderOf(shuffled, reach, edges); !reflect.DeepEqual(again.Order, want) {
		t.Fatalf("another reading ordered %v", again.Order)
	}

	// A declared order against an inferred one wins (novox/hq ADR 0249), and the pair says what it won over.
	members[1].After = []string{"mesh-catalog"}
	got = orderOf(members, reach, edges)
	if len(got.Cycle) > 0 || len(got.Contradictions) > 0 {
		t.Fatalf("a declared order did not win: cycle %v, %+v", got.Cycle, got.Contradictions)
	}
	if want := []string{"agent", "lab", "app", "cat", "host", "ctl"}; !reflect.DeepEqual(got.Order, want) {
		t.Fatalf("ordered %v, wanted %v", got.Order, want)
	}
	if !slices.ContainsFunc(got.Pairs, func(p orderPair) bool {
		return p.Before == "cat" && p.After == "ctl" && p.Why == orderDeclared && reflect.DeepEqual(p.Over, []string{orderVersionSkew})
	}) || slices.ContainsFunc(got.Pairs, func(p orderPair) bool { return p.Before == "ctl" && p.After == "cat" }) {
		t.Fatalf("the declared pair does not say it won over version skew: %+v", got.Pairs)
	}

	// Declared both ways is a contradiction no precedence resolves: a cycle, and how to end it.
	members[0].After = []string{"novox/mesh-controller"}
	got = orderOf(members, reach, edges)
	if !reflect.DeepEqual(got.Cycle, []string{"cat", "ctl"}) || len(got.Contradictions) != 1 {
		t.Fatalf("the cycle is %v, ordered %v, contradictions %+v", got.Cycle, got.Order, got.Contradictions)
	}
	if c := got.Contradictions[0]; !reflect.DeepEqual(c.Members, []string{"cat", "ctl"}) ||
		!strings.Contains(c.Said, "remove one of the `after:` lines") {
		t.Fatalf("the contradiction is said %+v", c)
	}
}

// novox/hq ADR 0249: when two rules order one pair both ways, the rule of higher precedence wins and the
// pair says what it overruled; rules of one rank both ways are refused, naming both and how to declare.
func TestAnOrderRuleYieldsToAStrongerOne(t *testing.T) {
	members := []orderMember{
		{ID: "ctl", Repository: "novox/mesh-controller", Number: 7},
		{ID: "host", Repository: "novox/mesh-host", Number: 9},
	}
	reach := map[string]orderReach{
		"ctl":  {Moved: []string{"mesh-controller", "build-agent"}},
		"host": {Moved: []string{"mesh-host"}},
	}
	builtBy := []inventory.Edge{{From: "mesh-host", To: "build-agent", Kind: inventory.EdgeBuiltBy}}

	// Built by outranks engine before controller.
	got := orderOf(members, reach, builtBy)
	if !reflect.DeepEqual(got.Order, []string{"ctl", "host"}) || len(got.Cycle) > 0 {
		t.Fatalf("ordered %v, cycle %v", got.Order, got.Cycle)
	}
	if want := []orderPair{{Before: "ctl", After: "host", Why: orderBuiltBy, Over: []string{orderEngineFirst}}}; !reflect.DeepEqual(got.Pairs, want) {
		t.Fatalf("pairs %+v, wanted %+v", got.Pairs, want)
	}

	// A declared line outranks both.
	members[0].After = []string{"mesh-host"}
	got = orderOf(members, reach, builtBy)
	if !reflect.DeepEqual(got.Order, []string{"host", "ctl"}) || len(got.Cycle) > 0 ||
		!reflect.DeepEqual(got.Pairs[0].Over, []string{orderBuiltBy}) || got.Pairs[0].Why != orderDeclared {
		t.Fatalf("a declared order did not win: %v %+v", got.Order, got.Pairs)
	}

	// Without the build agent, engine before controller stands alone.
	members[0].After = nil
	got = orderOf(members, map[string]orderReach{"ctl": {Moved: []string{"mesh-controller"}}, "host": reach["host"]}, builtBy)
	if !reflect.DeepEqual(got.Order, []string{"host", "ctl"}) || got.Pairs[0].Why != orderEngineFirst || len(got.Pairs[0].Over) > 0 {
		t.Fatalf("engine before controller alone: %v %+v", got.Order, got.Pairs)
	}

	// Built by both ways is one rank against itself: refused, both rules named, and how to declare.
	both := append(builtBy, inventory.Edge{From: "build-agent", To: "mesh-host", Kind: inventory.EdgeStandsOn})
	got = orderOf(members, reach, both)
	if !reflect.DeepEqual(got.Cycle, []string{"ctl", "host"}) || len(got.Order) > 0 || len(got.Contradictions) != 1 {
		t.Fatalf("a contradiction was ordered: %v, cycle %v, %+v", got.Order, got.Cycle, got.Contradictions)
	}
	said := got.Contradictions[0].Said
	for _, w := range []string{"built by puts ctl first", "built by puts host first",
		"`after: novox/mesh-controller` in novox/mesh-host#9's description", "`after: novox/mesh-host` in novox/mesh-controller#7's"} {
		if !strings.Contains(said, w) {
			t.Errorf("the contradiction does not say %q: %s", w, said)
		}
	}
}

// novox/hq ADR 0239 decision 8: a walk waits for the delivery's owner only while the seat has a holder on
// record, and never one that moves a module on the controller's own path.
func TestAWalkWaitsOnlyWhileTheDeliverySeatIsHeldAndNeverForTheCore(t *testing.T) {
	holder := inventory.Entry{Manifest: catalogue.Manifest{Module: "mesh-delivery",
		Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, On: []string{"anchor"}}
	unassigned := holder
	unassigned.On = nil
	for _, c := range []struct {
		entries []inventory.Entry
		moved   []string
		waits   bool
	}{
		{nil, []string{"gitea"}, false},
		{[]inventory.Entry{unassigned}, []string{"gitea"}, false},
		{[]inventory.Entry{holder}, []string{"gitea", "plex"}, true},
		{[]inventory.Entry{holder}, []string{"gitea", "mesh-controller"}, false},
		{[]inventory.Entry{holder}, []string{"mesh-host"}, false},
		{[]inventory.Entry{holder}, []string{"node-tools"}, false},
		{[]inventory.Entry{holder}, []string{"nats"}, false},
		{[]inventory.Entry{holder}, []string{"mesh-delivery", "gitea"}, false},
	} {
		d := awaitsFor(c.entries, c.moved)
		if (d != nil) != c.waits {
			t.Errorf("held %v, moving %v: waits %v, wanted %v", len(c.entries) > 0 && len(c.entries[0].On) > 0,
				c.moved, d != nil, c.waits)
		}
		if d != nil && d.Awaits != catalogue.DeliverySeat {
			t.Errorf("waits for %q", d.Awaits)
		}
	}
}

// novox/hq ADR 0239: with mesh-delivery on record, a merge opens its walk and asks nothing until the
// delivery's word; the word starts it; the core's own merge never waits; a person starts a waiting walk by
// hand when the owner is down, and the owner's stop ends one as stopped.
func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	inv := open.inventory
	asked := asksRecorded(t)
	withConditionsInMemory(t)
	for _, name := range []string{"app", "mesh-delivery"} {
		m := catalogue.Manifest{Module: name, Version: "1"}
		if name == "mesh-delivery" {
			m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}
		}
		if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git",
			Path: "modules/" + name, Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil {
			t.Fatal(err)
		}
	}
	merge := func(commit string, paths ...string) inventory.Plan {
		t.Helper()
		m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, Paths: paths,
			ModuleDirs: []string{"modules/app", "modules/mesh-delivery"}, ModuleDirsSaid: true}
		if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
			t.Fatal(err)
		}
		recent, err := inv.RecentPlans(ctx, 1)
		if err != nil || len(recent) != 1 || recent[0].Commit != commit {
			t.Fatalf("no plan for %s: %v %v", commit, recent, err)
		}
		return recent[0]
	}

	// finish ends a walk as done, so the next merge has nothing of it to take over.
	finish := func(id string) {
		t.Helper()
		w, err := inv.PlanByID(ctx, id)
		if err != nil {
			t.Fatal(err)
		}
		w.State = inventory.PlanDone
		if err := inv.SavePlan(ctx, &w); err != nil {
			t.Fatal(err)
		}
	}

	// No holder on record: the merge starts its walk, as before.
	p := merge("c1aaaaaaaa", "modules/app/index.ts")
	if p.Waiting() || len(*asked) != 1 {
		t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked)
	}

	// The holder on record: the next merge waits, asking nothing, and an advance asks nothing either. One walk
	// is open at a time (novox/hq ADR 0276): the first ends before the next merge's batch is cut.
	finish(p.ID)
	if _, err := inv.Assign(ctx, "anchor", "mesh-delivery"); err != nil {
		t.Fatal(err)
	}
	p = merge("c2bbbbbbbb", "modules/app/index.ts")
	if !p.Waiting() || len(*asked) != 1 || !strings.Contains(p.Note, "plans go "+p.ID) {
		t.Fatalf("the walk did not wait for its word: waiting %v, asked %v, note %q", p.Waiting(), *asked, p.Note)
	}
	advanceHeld(ctx, open)
	if len(*asked) != 1 {
		t.Fatalf("a waiting walk was advanced into asking: %v", *asked)
	}
	if line := planLine(p, p.Created); strings.Contains(line, "LATE") || !strings.Contains(line, "waits for") {
		t.Errorf("a waiting walk reads %q", line)
	}

	// The delivery's word starts it.
	if err := deliveryCommand(ctx, []string{"go", p.ID, "--by", catalogue.DeliverySeat, "--why", "its turn"}); err != nil {
		t.Fatal(err)
	}
	if err := deliveryCommand(ctx, []string{"go", p.ID}); err == nil {
		t.Fatal("a walk was let go twice")
	}
	advanceHeld(ctx, open)
	if len(*asked) != 2 {
		t.Fatalf("the word did not start the walk: %v", *asked)
	}
	got, err := inv.PlanByID(ctx, p.ID)
	if err != nil || got.Delivery == nil || got.Delivery.By != catalogue.DeliverySeat || got.Delivery.Why != "its turn" {
		t.Fatalf("the word was not kept: %+v %v", got.Delivery, err)
	}

	// The delivery's owner's own merge never waits for it. A walk that started is never taken over (novox/hq
	// ADR 0276): the merge's batch is cut once it ended.
	finish(p.ID)
	p = merge("c3cccccccc", "modules/mesh-delivery/main.go")
	if p.Waiting() || len(*asked) != 3 {
		t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked)
	}

	// The owner down: a person starts a waiting walk, with why.
	finish(p.ID)
	p = merge("c4dddddddd", "modules/app/index.ts")
	if !p.Waiting() {
		t.Fatal("the walk did not wait")
	}
	if err := plansCommand(ctx, []string{"go", p.ID}); err == nil || !strings.Contains(err.Error(), "--why") {
		t.Fatalf("a walk was started by hand without why: %v", err)
	}
	if err := plansCommand(ctx, []string{"go", p.ID, "--why", "mesh-delivery is down"}); err != nil {
		t.Fatal(err)
	}
	advanceHeld(ctx, open)
	got, _ = inv.PlanByID(ctx, p.ID)
	if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 4 {
		t.Fatalf("a person's word did not start the walk: %+v, asked %v", got.Delivery, *asked)
	}

	// The owner's stop: failed, said as stopped by it.
	finish(p.ID)
	p = merge("c5eeeeeeee", "modules/app/index.ts")
	if err := deliveryCommand(ctx, []string{"stop", p.ID, "--why", "the operator stopped it", "--by",
		"mesh-delivery for jochen"}); err != nil {
		t.Fatal(err)
	}
	got, _ = inv.PlanByID(ctx, p.ID)
	if got.State != inventory.PlanFailed || got.Delivery.Stopped != "mesh-delivery for jochen" ||
		!strings.Contains(got.Note, "the operator stopped it") {
		t.Fatalf("the stop was kept as %s %+v %q", got.State, got.Delivery, got.Note)
	}

	// Unassigned: the mesh is back on the controller's own path.
	if err := inv.Unassign(ctx, "anchor", "mesh-delivery"); err != nil {
		t.Fatal(err)
	}
	asks := len(*asked)
	if p = merge("c6ffffffff", "modules/app/index.ts"); p.Waiting() || len(*asked) != asks+1 {
		t.Fatalf("with the holder gone the walk waited: %v", p.Waiting())
	}
}

// The verbs mesh-delivery asks with become the commands they name, and nothing a caller sends is passed over.
func TestTheDeliveryVerbsComposeTheirCommands(t *testing.T) {
	for _, c := range []struct {
		verb string
		args map[string]any
		want []string
	}{
		{"deliver", map[string]any{"plan": "plan-1", "why": "its turn"},
			[]string{"delivery", "go", "plan-1", "--by", "mesh-delivery", "--why", "its turn"}},
		{"delivery-stop", map[string]any{"plan": "plan-1", "why": "w", "by": "jochen"},
			[]string{"delivery", "stop", "plan-1", "--why", "w", "--by", "mesh-delivery for jochen"}},
		{"delivery-walks", map[string]any{}, []string{"delivery", "walks"}},
		{"delivery-walks", map[string]any{"plan": "plan-1"}, []string{"delivery", "walks", "--plan", "plan-1"}},
		{"delivery-order", map[string]any{"members": "[]"}, []string{"delivery", "order", "--members", "[]"}},
		{"delivery-check", map[string]any{"group": "feat/x", "members": "[]"},
			[]string{"delivery", "check", "--group", "feat/x", "--members", "[]"}},
		{"delivery-plan", map[string]any{"repository": "novox/a", "paths": "x", "head": "c0"},
			[]string{"delivery", "plan", "--repository", "novox/a", "--paths", "x", "--head", "c0"}},
		{"plans", map[string]any{"go": "plan-1", "why": "down"}, []string{"plans", "go", "plan-1", "--why", "down"}},
	} {
		got, err := argvFor(c.verb, c.args)
		if err != nil || !reflect.DeepEqual(got, c.want) {
			t.Errorf("%s %v → %v %v, wanted %v", c.verb, c.args, got, err, c.want)
		}
	}
	if _, err := argvFor("deliver", map[string]any{}); err == nil {
		t.Error("deliver without a walk was composed")
	}
	if _, err := argvFor("delivery-stop", map[string]any{"plan": "p"}); err == nil {
		t.Error("a stop without why was composed")
	}
}

// A verb runs as a command of its own: what it keeps of a walk is still said as plan-moved, on a bus of the
// command's own, so the delivery's owner hears it at once and not only when it reads the walks back.
func TestAWalkLetGoByAVerbIsSaidAsPlanMoved(t *testing.T) {
	open := aMesh(t)
	ctx := t.Context()
	url := testbus.URL(t)
	js, err := broker.Dial(url)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(js.Close)
	if err := broker.AssertMeshStreams(js); err != nil {
		t.Fatal(err)
	}
	before := handActConn
	handActConn = js.Conn()
	t.Cleanup(func() { handActConn = before })
	heard := make(chan *nats.Msg, 4)
	sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard)
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = sub.Unsubscribe() })

	waiting := inventory.Plan{ID: "plan-waits", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c7c7c7c7",
		Created: time.Now().UTC(), State: inventory.PlanBuilding, Tiers: [][]string{{"app"}},
		Modules:  map[string]*inventory.PlanModule{"app": {}},
		Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}}
	if err := open.inventory.SavePlan(ctx, &waiting); err != nil {
		t.Fatal(err)
	}
	if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil {
		t.Fatal(err)
	}
	select {
	case m := <-heard:
		var said inventory.Plan
		if err := json.Unmarshal(m.Data, &said); err != nil || said.ID != waiting.ID || said.Delivery == nil ||
			said.Delivery.Go == nil {
			t.Fatalf("plan-moved said %s (%v)", m.Data, err)
		}
	case <-time.After(5 * time.Second):
		t.Fatal("a walk let go by a verb was not said")
	}
	if checkEvents != nil || inventory.PlanSaved != nil {
		t.Fatal("the command's own bus was left in place")
	}
}