mesh-controller/cmd/mesh-controller/pending_test.go

package main

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"slices"
	"strings"
	"testing"
	"time"

	"github.com/novox/mesh-controller/internal/catalogue"
	"github.com/novox/mesh-controller/internal/conditions"
	"github.com/novox/mesh-controller/internal/inventory"
	"github.com/novox/mesh-controller/internal/link"
)

// failedBuild settles the merge's build of modules/sensors as failed.
func failedBuild(t *testing.T, open *stores, id string) {
	t.Helper()
	if err := (builds{open.inventory, open}).Built(t.Context(), outcome(id, "modules/sensors", "3da80a4b00aa",
		"", "boom")); err != nil {
		t.Fatal(err)
	}
}

func conditionOpen(t *testing.T, p inventory.PendingAssignment) bool {
	t.Helper()
	_, found, err := conditionsFrom.Get(t.Context(), pendingObservation(p).Key())
	if err != nil {
		t.Fatal(err)
	}
	return found
}

func rowOf(t *testing.T, open *stores, id int64) inventory.PendingAssignment {
	t.Helper()
	rows, err := open.inventory.Pending(t.Context(), time.Unix(0, 0))
	if err != nil {
		t.Fatal(err)
	}
	for _, r := range rows {
		if r.ID == id {
			return r
		}
	}
	t.Fatalf("pending assignment %d is not on record", id)
	return inventory.PendingAssignment{}
}

// novox/hq issue 325: an assignment of a module the catalogue does not hold says which case it is in — a
// build in flight (kept pending), known and not built (said, with build), or unknown (refused, with the
// closest names) — and a pending assignment is made when its build registers the module, or ends with why.

// aCatalogueMesh is aMesh with one module held from the catalogue, so a merge of it is acted on.
func aCatalogueMesh(t *testing.T) *stores {
	t.Helper()
	open := aMesh(t)
	if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: "networkmanager", Version: "1"},
		inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main",
			BuiltFrom: "c0", Head: "c0"}); err != nil {
		t.Fatal(err)
	}
	return open
}

// mergeAdding is the catalogue's merge adding one module's directory, acted on as the bus hands it over.
func mergeAdding(t *testing.T, open *stores, dir, commit string) {
	t.Helper()
	m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit,
		Paths: []string{dir + "/module.json"}, ModuleDirs: []string{dir}, ModuleDirsSaid: true}
	if err := (following{open: open}).SourceMoved(t.Context(), m); err != nil {
		t.Fatal(err)
	}
}

// outcome is a build's outcome as the build seat announces it: registered as module, or failed.
func outcome(id, dir, commit, module, failed string) link.BuildResult {
	r := link.BuildResult{ID: id, Repository: "novox/mesh-catalog", Path: dir, Ref: "main", Commit: commit,
		On: "anchor", Failed: failed, Source: &link.SourceOnSeat{Repository: "novox/mesh-catalog", Seat: "git"}}
	if failed == "" {
		r.Module = module
		r.Manifest, _ = json.Marshal(catalogue.Manifest{Module: module, Version: "1"})
	}
	return r
}

func assignedTo(t *testing.T, open *stores, node string) []string {
	t.Helper()
	got, err := open.inventory.Assigned(t.Context(), node)
	if err != nil {
		t.Fatal(err)
	}
	return got
}

func pendingOf(t *testing.T, open *stores) []inventory.PendingAssignment {
	t.Helper()
	got, err := open.inventory.Pending(t.Context(), time.Now().Add(-time.Hour))
	if err != nil {
		t.Fatal(err)
	}
	return got
}

// A build in flight: the merge asked for it, the request is kept as the merge's, the assignment is kept
// pending and said so with the build, status reads it without settling anything, and the build's outcome
// makes it — saying that a push sends it, and nothing more.
func TestAnAssignmentWaitsForTheBuildInFlight(t *testing.T) {
	open := aCatalogueMesh(t)
	ctx := t.Context()
	asksWithPaths(t)
	mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")

	requests, err := open.inventory.RequestsNamed(ctx, "sensors")
	if err != nil || len(requests) != 1 || requests[0].For != "merge" || requests[0].Commit != "3da80a4b00aa" {
		t.Fatalf("the merge's build request: %+v %v", requests, err)
	}

	said, err := assign(ctx, open, "laptop", "sensors")
	if err != nil {
		t.Fatalf("an assignment while its build runs was refused: %v", err)
	}
	t.Logf("assign laptop sensors, while its build runs:\n%s", said)
	for _, want := range []string{"being built from novox/mesh-catalog@3da80a4b (modules/sensors)",
		"build b-modules/sensors", "kept as pending", "the controller assigns sensors to laptop", "`unassign laptop sensors`"} {
		if !strings.Contains(said, want) {
			t.Fatalf("the answer does not say %q:\n%s", want, said)
		}
	}
	if slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
		t.Fatal("a module not registered was assigned")
	}
	pending := pendingOf(t, open)
	if len(pending) != 1 || pending[0].State != inventory.PendingWaiting || pending[0].Build != "b-modules/sensors" {
		t.Fatalf("pending: %+v", pending)
	}
	if again, err := assign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(again, "already waits") {
		t.Fatalf("a second assignment: %q %v", again, err)
	}

	asked, err := theThreeQuestions(ctx, open)
	if err != nil {
		t.Fatal(err)
	}
	if asked.well() {
		t.Fatal("status called the mesh well while an assignment waits")
	}
	body, err := statusAsJSON(asked)
	if err != nil {
		t.Fatal(err)
	}
	var doc meshStatus
	if err := json.Unmarshal(body, &doc); err != nil {
		t.Fatal(err)
	}
	if len(doc.Pending) != 1 || doc.Pending[0].State != "waiting" || doc.Pending[0].Module != "sensors" {
		t.Fatalf("status says pending %+v", doc.Pending)
	}

	if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors",
		"3da80a4b00aa", "sensors", "")); err != nil {
		t.Fatal(err)
	}
	if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
		t.Fatal("the build registered sensors and the pending assignment was not made")
	}
	pending = pendingOf(t, open)
	if len(pending) != 1 || pending[0].State != inventory.PendingApplied ||
		!strings.HasSuffix(pending[0].Note, "`push laptop` sends it") {
		t.Fatalf("pending after the build: %+v", pending)
	}
}

// **A read settles nothing** (review of #150, point 6): status — and so the board, the summary and the
// probe, which compose from the same reading — leaves a pending assignment whose module is registered as it
// is; the controller's tick makes it.
func TestAStatusReadSettlesNothing(t *testing.T) {
	open := aCatalogueMesh(t)
	ctx := t.Context()
	asksWithPaths(t)
	mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
	if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
		t.Fatal(err)
	}
	register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"})
	for range 2 {
		if _, err := theThreeQuestions(ctx, open); err != nil {
			t.Fatal(err)
		}
	}
	if slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingWaiting {
		t.Fatalf("a status read settled a pending assignment: %+v", pendingOf(t, open))
	}
	said := settlePending(ctx, open, time.Now())
	if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingApplied {
		t.Fatalf("the tick did not make it: %v %+v", said, pendingOf(t, open))
	}
}

// The build of a pending assignment fails: it expires with the build's words and nothing is assigned; the
// tick raises it as a condition once; it then reads as known and not built; and `unassign` takes it back,
// after which the tick clears the condition. Status is not called well while the condition is open, and is
// again once it clears — no fixed day of "not well" (review point 6).
func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) {
	open := aCatalogueMesh(t)
	ctx := t.Context()
	asksWithPaths(t)
	mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
	if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
		t.Fatal(err)
	}
	if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors",
		"3da80a4b00aa", "", "go build: undefined: sensorsRead")); err != nil {
		t.Fatal(err)
	}
	pending := pendingOf(t, open)
	if len(pending) != 1 || pending[0].State != inventory.PendingExpired {
		t.Fatalf("pending after a failed build: %+v", pending)
	}
	for _, want := range []string{"expired", "build b-modules/sensors of modules/sensors failed",
		"undefined: sensorsRead", "nothing was assigned", "build \"true\""} {
		if !strings.Contains(pending[0].Note, want) {
			t.Fatalf("the expiry does not say %q: %s", want, pending[0].Note)
		}
	}
	if slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
		t.Fatal("a failed build's module was assigned")
	}

	settlePending(ctx, open, time.Now())
	key := pendingObservation(pending[0]).Key()
	c, found, err := conditionsFrom.Get(ctx, key)
	if err != nil || !found || c.Kind != kindPendingEnded {
		t.Fatalf("no condition %s for the assignment not made: %+v %v %v", key, c, found, err)
	}
	if asked, err := theThreeQuestions(ctx, open); err != nil || asked.well() || len(asked.conditions) == 0 {
		t.Fatalf("status does not hold the open condition: %v", err)
	}

	_, err = assign(ctx, open, "laptop", "sensors")
	if !errors.Is(err, inventory.ErrNoSuchModule) {
		t.Fatalf("a module whose build failed: %v", err)
	}
	for _, want := range []string{"sensors is known and not registered", "its last build, b-modules/sensors",
		"failed: go build: undefined: sensorsRead", "build \"true\"", "repository novox/mesh-catalog, path modules/sensors"} {
		if !strings.Contains(err.Error(), want) {
			t.Fatalf("the refusal does not say %q:\n%v", want, err)
		}
	}

	said, err := unassign(ctx, open, "laptop", "sensors")
	if err != nil || !strings.Contains(said, "taken back") {
		t.Fatalf("unassign of an expired pending assignment: %q %v", said, err)
	}
	settlePending(ctx, open, time.Now())
	if _, found, _ := conditionsFrom.Get(ctx, key); found {
		t.Fatal("the condition stayed open after the assignment was taken back")
	}
	// Nothing of it keeps status from being well any more: no open pending assignment, no open condition of it.
	asked, err := theThreeQuestions(ctx, open)
	if err != nil || pendingOpen(asked.pending) {
		t.Fatalf("status still counts the pending assignment: %+v %v", asked.pending, err)
	}
	for _, c := range asked.conditions {
		if c.Kind == kindPendingEnded {
			t.Fatalf("status still holds the condition: %+v", c)
		}
	}
}

// The condition also clears when the module is later assigned to the machine.
func TestTheConditionClearsWhenTheModuleIsAssignedLater(t *testing.T) {
	open := aCatalogueMesh(t)
	ctx := t.Context()
	asksWithPaths(t)
	mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
	if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
		t.Fatal(err)
	}
	if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors",
		"3da80a4b00aa", "", "boom")); err != nil {
		t.Fatal(err)
	}
	settlePending(ctx, open, time.Now())
	key := pendingObservation(pendingOf(t, open)[0]).Key()
	register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"})
	if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
		t.Fatal(err)
	}
	settlePending(ctx, open, time.Now())
	if _, found, _ := conditionsFrom.Get(ctx, key); found {
		t.Fatal("the condition stayed open after the module was assigned")
	}
}

// Known and not built, asked with build: the build is asked from where the last one was, the request kept as
// assign's, and the assignment kept pending on the new build.
func TestAssignWithBuildAsksForAKnownModule(t *testing.T) {
	open := aCatalogueMesh(t)
	ctx := t.Context()
	inv := open.inventory
	if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog",
		Seat: "git", Path: "modules/sensors", Ref: "main", For: "build", At: time.Now().Add(-time.Hour)}); err != nil {
		t.Fatal(err)
	}
	if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main",
		Path: "modules/sensors", On: "anchor", Failed: "no space left on device"}); err != nil {
		t.Fatal(err)
	}
	asked := asksWithPaths(t)
	said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors")
	if err != nil {
		t.Fatal(err)
	}
	if len(*asked) != 1 || (*asked)[0] != [3]string{"novox/mesh-catalog", "modules/sensors", "main"} {
		t.Fatalf("asked %v", *asked)
	}
	for _, want := range []string{"known and not registered", "no space left on device",
		"build b-modules/sensors of novox/mesh-catalog@main (modules/sensors) is asked for now", "kept as pending"} {
		if !strings.Contains(said, want) {
			t.Fatalf("the answer does not say %q:\n%s", want, said)
		}
	}
	pending := pendingOf(t, open)
	if len(pending) != 1 || pending[0].Build != "b-modules/sensors" {
		t.Fatalf("pending: %+v", pending)
	}
	requests, _ := inv.RequestsNamed(ctx, "sensors")
	if len(requests) != 2 || requests[0].ID != "b-modules/sensors" || requests[0].For != "assign" {
		t.Fatalf("the request is not kept as assign's: %+v", requests)
	}
}

// **build "true" with a pending assignment already waiting** (review point 5): the waiting one is made to
// wait for the new build, and no build is asked that nothing waits on.
func TestAssignWithBuildRepointsTheWaitingAssignment(t *testing.T) {
	open := aCatalogueMesh(t)
	ctx := t.Context()
	inv := open.inventory
	if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog",
		Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge", At: time.Now().Add(-time.Hour)}); err != nil {
		t.Fatal(err)
	}
	if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main",
		Path: "modules/sensors", On: "anchor", Failed: "boom"}); err != nil {
		t.Fatal(err)
	}
	// Waiting on the failed build, its expiry not yet settled.
	if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "laptop", Module: "sensors",
		Build: "build-old", Repository: "novox/mesh-catalog", Path: "modules/sensors"}); err != nil {
		t.Fatal(err)
	}
	_, err := assign(ctx, open, "laptop", "sensors")
	if err == nil || !strings.Contains(err.Error(), "already has a pending assignment of sensors, waiting for build build-old") {
		t.Fatalf("known and not built with a pending assignment waiting: %v", err)
	}
	asked := asksWithPaths(t)
	said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors")
	if err != nil || !strings.Contains(said, "which waited for build build-old, now waits for build b-modules/sensors") {
		t.Fatalf("assign with build: %q %v", said, err)
	}
	if len(*asked) != 1 {
		t.Fatalf("asked %v", *asked)
	}
	pending := pendingOf(t, open)
	if len(pending) != 1 || pending[0].Build != "b-modules/sensors" || pending[0].State != inventory.PendingWaiting {
		t.Fatalf("pending: %+v", pending)
	}
	// The new build in flight now: build "true" again asks nothing.
	if said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil ||
		!strings.Contains(said, "no second build was asked") || len(*asked) != 1 {
		t.Fatalf("a second build true while the build runs: %q %v, asked %v", said, err, *asked)
	}
}

// **A failed ask never reads as in flight** (review point 2): a merge whose ask could not be made keeps the
// request as not asked, and assign says the merge could not ask; a request whose asker could not hand it over
// or stopped waiting reads the same, unless its outcome was heard.
func TestAFailedAskIsNotABuildInFlight(t *testing.T) {
	open := aCatalogueMesh(t)
	ctx := t.Context()
	inv := open.inventory
	was := askABuild
	askABuild = func(context.Context, buildSource, string, string) (string, error) {
		return "", errors.New("cannot submit a build: nats: timeout")
	}
	t.Cleanup(func() { askABuild = was })
	mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
	_, err := assign(ctx, open, "laptop", "sensors")
	if err == nil || !strings.Contains(err.Error(), "the merge that added it (3da80a4b) could not ask for its build: cannot submit") {
		t.Fatalf("a merge that could not ask: %v", err)
	}
	if len(pendingOf(t, open)) != 0 {
		t.Fatal("a pending assignment waits on a build never asked")
	}

	if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-waited", Repository: "novox/mesh-catalog",
		Seat: "git", Path: "modules/gauges", Ref: "main", For: "build"}); err != nil {
		t.Fatal(err)
	}
	if err := inv.MarkNotAsked(ctx, "build-waited", "cannot submit a build: no responders"); err != nil {
		t.Fatal(err)
	}
	_, err = assign(ctx, open, "laptop", "gauges")
	if err == nil || !strings.Contains(err.Error(), "build build-waited, asked by build, was not handed over") {
		t.Fatalf("a build not handed over: %v", err)
	}
	// Heard first, the outcome stands: marking it afterwards changes nothing.
	if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-waited", Repository: "novox/mesh-catalog", Ref: "main",
		Path: "modules/gauges", On: "anchor", Failed: "the real failure"}); err != nil {
		t.Fatal(err)
	}
	_, err = assign(ctx, open, "laptop", "gauges")
	if err == nil || !strings.Contains(err.Error(), "failed: the real failure") {
		t.Fatalf("an outcome heard after a failed wait: %v", err)
	}
}

// A plan's tier keeps its requests as the plan's (review point 3).
func TestAPlansBuildRequestIsThePlans(t *testing.T) {
	open := aCatalogueMesh(t)
	ctx := t.Context()
	asksWithPaths(t)
	if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"},
		inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main",
			BuiltFrom: "c0", Head: "c0"}); err != nil {
		t.Fatal(err)
	}
	m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa",
		Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true}
	if err := (following{open: open}).SourceMoved(ctx, m); err != nil {
		t.Fatal(err)
	}
	requests, err := open.inventory.RequestsNamed(ctx, "app")
	if err != nil || len(requests) != 1 || requests[0].For != "plan" {
		t.Fatalf("the plan's request: %+v %v", requests, err)
	}
}

// **A build heard as built and not registered yet is in flight** (review point 4): the outcome is recorded
// before the module is registered; for that moment assign keeps the assignment pending. Past the grace it is
// recorded and not registered, and the tick ends the pending assignment with that.
func TestABuildBeingRegisteredIsInFlight(t *testing.T) {
	open := aCatalogueMesh(t)
	ctx := t.Context()
	inv := open.inventory
	if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-done", Repository: "novox/mesh-catalog",
		Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge"}); err != nil {
		t.Fatal(err)
	}
	if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-done", Repository: "novox/mesh-catalog", Ref: "main",
		Path: "modules/sensors", Module: "sensors", On: "anchor"}); err != nil {
		t.Fatal(err)
	}
	said, err := assign(ctx, open, "laptop", "sensors")
	if err != nil || !strings.Contains(said, "built and being registered") || !strings.Contains(said, "kept as pending") {
		t.Fatalf("a build being registered: %q %v", said, err)
	}
	settlePending(ctx, open, time.Now().Add(registerGrace+time.Minute))
	p := pendingOf(t, open)
	if len(p) != 1 || p[0].State != inventory.PendingExpired || !strings.Contains(p[0].Note, "recorded and not registered") {
		t.Fatalf("past the grace: %+v", p)
	}
}

// Unknown: refused as no module of that name, claiming only what was looked at, with the closest names; build
// asks for nothing.
func TestAnUnknownModuleIsRefusedWithTheClosestNames(t *testing.T) {
	open := aCatalogueMesh(t)
	ctx := t.Context()
	register(t, open, catalogue.Manifest{Module: "sensord", Version: "1"})
	asked := asksWithPaths(t)
	_, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors")
	if !errors.Is(err, inventory.ErrNoSuchModule) {
		t.Fatalf("an unknown module: %v", err)
	}
	for _, want := range []string{"no module of that name: sensors", "the catalogue holds no module of that name",
		"no build request the controller kept (the last 30 days)", "the closest names it holds: sensord",
		"asks for nothing here"} {
		if !strings.Contains(err.Error(), want) {
			t.Fatalf("the refusal does not say %q:\n%v", want, err)
		}
	}
	if strings.Contains(err.Error(), "merge") {
		t.Fatalf("the refusal claims a merge it never looked at: %v", err)
	}
	if len(*asked) != 0 || len(pendingOf(t, open)) != 0 {
		t.Fatalf("an unknown module asked %v, pending %+v", *asked, pendingOf(t, open))
	}
}

// Several modules in one act, one of them not registered: nothing is assigned and nothing kept pending, and
// each missing one is said.
func TestAnActWithAModuleNotRegisteredIsRefusedWhole(t *testing.T) {
	open := aCatalogueMesh(t)
	ctx := t.Context()
	asksWithPaths(t)
	register(t, open, catalogue.Manifest{Module: "known", Version: "1"})
	mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
	_, err := assign(ctx, open, "laptop", "known", "sensors")
	if err == nil || !strings.Contains(err.Error(), "nothing was assigned") || !strings.Contains(err.Error(), "being built") {
		t.Fatalf("an act naming a module in flight: %v", err)
	}
	if slices.Contains(assignedTo(t, open, "laptop"), "known") || len(pendingOf(t, open)) != 0 {
		t.Fatal("an act refused whole was made in part")
	}
}

// unassign withdraws a waiting pending assignment; the build goes on and registers the module assigned
// nowhere: a withdrawn row is never claimed.
func TestUnassignWithdrawsAPendingAssignment(t *testing.T) {
	open := aCatalogueMesh(t)
	ctx := t.Context()
	asksWithPaths(t)
	mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
	if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
		t.Fatal(err)
	}
	said, err := unassign(ctx, open, "laptop", "sensors")
	if err != nil || !strings.Contains(said, "withdrawn") {
		t.Fatalf("unassign of a pending assignment: %q %v", said, err)
	}
	if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors",
		"3da80a4b00aa", "sensors", "")); err != nil {
		t.Fatal(err)
	}
	settlePending(ctx, open, time.Now())
	if slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
		t.Fatal("a withdrawn assignment was made")
	}
}

// **A claim is never overridden** (review point 1): a pending assignment being made is claimed
// (waiting → applying); an unassign meanwhile is refused and says so, and leaves the claim; making a row
// already withdrawn makes nothing. A claim left by a controller that stopped is settled by the tick from what
// the mesh holds: back to waiting when the module is not assigned, applied when it is.
func TestAWithdrawalNeverOverridesAClaim(t *testing.T) {
	open := aCatalogueMesh(t)
	ctx := t.Context()
	inv := open.inventory
	asksWithPaths(t)
	mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
	if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
		t.Fatal(err)
	}
	p := pendingOf(t, open)[0]
	if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok {
		t.Fatalf("claim: %v %v", ok, err)
	}
	_, err := unassign(ctx, open, "laptop", "sensors")
	if err == nil || !strings.Contains(err.Error(), "is being assigned sensors now") {
		t.Fatalf("unassign of a claimed pending assignment: %v", err)
	}
	if got := pendingOf(t, open)[0]; got.State != inventory.PendingApplying {
		t.Fatalf("the claim was overridden: %+v", got)
	}
	// Pending(zero) is the waiting ones alone (review point 7).
	if waiting, err := inv.Pending(ctx, time.Time{}); err != nil || len(waiting) != 0 {
		t.Fatalf("Pending(zero) read a claimed row: %+v %v", waiting, err)
	}

	// Left by a controller that stopped: back to waiting, since sensors is not assigned.
	settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute))
	if got := pendingOf(t, open)[0]; got.State != inventory.PendingWaiting {
		t.Fatalf("a stale claim was not released: %+v", got)
	}

	// Withdrawn, then the build registers it: applyPending finds nothing to claim.
	if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil {
		t.Fatal(err)
	}
	register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"})
	if line := applyPending(ctx, open, p, "b-modules/sensors"); line != "" {
		t.Fatalf("a withdrawn pending assignment was made: %s", line)
	}
	if slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
		t.Fatal("a withdrawn pending assignment was assigned")
	}

	// A stale claim whose module was assigned is applied.
	q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"})
	if err != nil {
		t.Fatal(err)
	}
	if ok, _ := inv.ClaimPending(ctx, q.ID); !ok {
		t.Fatal("claim")
	}
	if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil {
		t.Fatal(err)
	}
	settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute))
	rows, _ := inv.PendingFor(ctx, "anchor", "sensors")
	if len(rows) != 1 || rows[0].State != inventory.PendingApplied {
		t.Fatalf("a stale claim of an assigned module: %+v", rows)
	}
}

// A build that says nothing within the bound: the pending assignment expires at the next tick, saying so,
// rather than waiting for ever; and a module registered by a process that kept no pending assignment is
// assigned at the next tick.
func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) {
	open := aCatalogueMesh(t)
	ctx := t.Context()
	inv := open.inventory
	for _, r := range []inventory.BuildRequest{
		{ID: "build-lost", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/lost", Ref: "main",
			For: "merge", At: time.Now().Add(-buildRequestBound / 2)},
		{ID: "build-heard", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/heard", Ref: "main",
			For: "merge", At: time.Now()},
	} {
		if err := inv.RecordBuildRequest(ctx, r); err != nil {
			t.Fatal(err)
		}
	}
	for _, m := range []string{"lost", "heard"} {
		if _, err := assign(ctx, open, "laptop", m); err != nil {
			t.Fatal(err)
		}
	}
	register(t, open, catalogue.Manifest{Module: "heard", Version: "1"})

	settlePending(ctx, open, time.Now().Add(buildRequestBound))
	if !slices.Contains(assignedTo(t, open, "laptop"), "heard") {
		t.Fatal("a module registered meanwhile was not assigned at the next tick")
	}
	byModule := map[string]inventory.PendingAssignment{}
	for _, p := range pendingOf(t, open) {
		byModule[p.Module] = p
	}
	if p := byModule["lost"]; p.State != inventory.PendingExpired || !strings.Contains(p.Note, "no outcome of build build-lost") {
		t.Fatalf("a build that said nothing: %+v", p)
	}
	if p := byModule["heard"]; p.State != inventory.PendingApplied {
		t.Fatalf("a module registered meanwhile: %+v", p)
	}

	// Ended rows are deleted after KeptFor (review point 7); open ones never, nor one whose condition is
	// still open (second review, bug B).
	if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil {
		t.Fatal(err)
	}
	said := settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
	rows, err := inv.Pending(ctx, time.Unix(0, 0))
	if err != nil {
		t.Fatal(err)
	}
	for _, r := range rows {
		if !r.Open() && (r.Raised == nil || r.Cleared != nil) {
			t.Fatalf("an ended pending assignment with no open condition outlived %s: %+v (%v)", inventory.KeptFor, r, said)
		}
	}
	if !strings.Contains(strings.Join(said, "\n"), "deleted 1 pending assignment(s)") {
		t.Fatalf("the applied row was not deleted: %v", said)
	}
}

// The seat's assign passes build on; unassign takes none.
func TestTheSeatsAssignPassesBuild(t *testing.T) {
	argv, err := argvFor("assign", map[string]any{"node": "g14", "module": "sensors", "build": "true"})
	if err != nil || !slices.Equal(argv, []string{"assign", "g14", "sensors", "--build"}) {
		t.Fatalf("assign with build: %v %v", argv, err)
	}
	if _, err := argvFor("unassign", map[string]any{"node": "g14", "module": "sensors", "build": "true"}); err == nil {
		t.Fatal("unassign took build")
	}
}

// The condition's words are plain.
func TestAnAssignmentNotMadeIsSaidPlainly(t *testing.T) {
	o := pendingObservation(inventory.PendingAssignment{Node: "g14", Module: "sensors",
		Note: "the pending assignment of sensors to g14 expired: build b-1 of modules/sensors failed: boom"})
	w := plainWordings[kindPendingEnded](o)
	if why, ok := conditions.PlainWords(w, "g14"); !ok {
		t.Fatalf("not plain: %s %+v", why, w)
	}
	if w.Headline != "sensors was not put on g14" {
		t.Fatalf("headline %q", w.Headline)
	}
}

// **One row's condition is never another's** (second review, bug A). The sequence: the first pending
// assignment's build fails and the tick raises it; the person assigns again with build "true", a second
// pending assignment; its build fails too before the next tick. That tick raises the second, and must not
// clear it by judging the first answered by the second: each row has its own condition, and a newer row that
// itself ended unmade answers nothing.
func TestASecondAssignmentNotMadeKeepsItsCondition(t *testing.T) {
	open := aCatalogueMesh(t)
	ctx := t.Context()
	asked := 0
	was := askABuild
	askABuild = func(context.Context, buildSource, string, string) (string, error) {
		asked++
		return fmt.Sprintf("b-%d", asked), nil
	}
	t.Cleanup(func() { askABuild = was })
	mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
	if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
		t.Fatal(err)
	}
	failedBuild(t, open, "b-1")
	settlePending(ctx, open, time.Now())
	first := pendingOf(t, open)[0]
	if !conditionOpen(t, first) {
		t.Fatal("the first assignment not made raised nothing")
	}

	if _, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil {
		t.Fatal(err)
	}
	failedBuild(t, open, "b-2")
	settlePending(ctx, open, time.Now())

	var second inventory.PendingAssignment
	for _, p := range pendingOf(t, open) {
		if p.Build == "b-2" {
			second = p
		}
	}
	if second.State != inventory.PendingExpired {
		t.Fatalf("the second: %+v", second)
	}
	if !conditionOpen(t, second) {
		t.Fatal("the second assignment's condition was cleared in the tick that raised it")
	}
	if !conditionOpen(t, rowOf(t, open, first.ID)) {
		t.Fatal("the first was judged answered by a second that itself was not made")
	}
	// unassign takes both back, and the next tick clears both.
	if said, err := unassign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(said, "b-1") ||
		!strings.Contains(said, "b-2") {
		t.Fatalf("unassign: %q %v", said, err)
	}
	settlePending(ctx, open, time.Now())
	if conditionOpen(t, first) || conditionOpen(t, second) {
		t.Fatal("a condition stayed open after both were taken back")
	}
}

// **A raised row outlives the pruning until its condition clears** (second review, bug B); and a machine's
// removal, which takes its rows with it, clears their conditions at the next tick.
func TestARaisedRowIsKeptUntilItsConditionClears(t *testing.T) {
	open := aCatalogueMesh(t)
	ctx := t.Context()
	inv := open.inventory
	asksWithPaths(t)
	mergeAdding(t, open, "modules/sensors", "3da80a4b00aa")
	if _, err := assign(ctx, open, "laptop", "sensors"); err != nil {
		t.Fatal(err)
	}
	failedBuild(t, open, "b-modules/sensors")
	settlePending(ctx, open, time.Now())
	row := pendingOf(t, open)[0]

	settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
	if rowOf(t, open, row.ID).Raised == nil || !conditionOpen(t, row) {
		t.Fatal("a raised row was pruned, or its condition closed, while the condition was open")
	}
	if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil {
		t.Fatal(err)
	}
	settlePending(ctx, open, time.Now())
	settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour))
	if known, err := inv.PendingKnown(ctx, []int64{row.ID}); err != nil || known[row.ID] {
		t.Fatalf("a cleared row outlived %s: %v", inventory.KeptFor, err)
	}

	// On anchor, then anchor removed: the row goes, and its condition with it at the next tick.
	if _, err := assign(ctx, open, "anchor", "sensors"); err == nil {
		t.Fatal("known and not built was not refused")
	}
	q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"})
	if err != nil {
		t.Fatal(err)
	}
	if _, err := inv.SettlePending(ctx, q.ID, inventory.PendingWaiting, inventory.PendingExpired, "boom"); err != nil {
		t.Fatal(err)
	}
	settlePending(ctx, open, time.Now())
	q.Node = "anchor"
	if !conditionOpen(t, q) {
		t.Fatal("not raised")
	}
	if _, err := inv.RemoveNodeForTest(ctx, "anchor"); err != nil {
		t.Fatal(err)
	}
	settlePending(ctx, open, time.Now())
	if conditionOpen(t, q) {
		t.Fatal("a removed machine's condition stayed open")
	}
}

// **A claim left by a controller that stopped** (second review): the tick settles it from what the mesh
// holds — back to waiting when the module is not assigned there, applied when it is — and leaves a fresh
// claim alone.
func TestAStaleClaimIsSettledFromWhatTheMeshHolds(t *testing.T) {
	open := aCatalogueMesh(t)
	ctx := t.Context()
	inv := open.inventory
	claimed := func(node string) inventory.PendingAssignment {
		p, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: "sensors", Build: "b-x"})
		if err != nil {
			t.Fatal(err)
		}
		if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok {
			t.Fatalf("claim: %v %v", ok, err)
		}
		return p
	}
	notAssigned, assigned := claimed("laptop"), claimed("anchor")
	register(t, open, catalogue.Manifest{Module: "unrelated", Version: "1"})
	if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "sensors", Version: "1"}, inventory.Source{}); err != nil {
		t.Fatal(err)
	}
	if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil {
		t.Fatal(err)
	}

	settlePending(ctx, open, time.Now())
	if rowOf(t, open, notAssigned.ID).State != inventory.PendingApplying || rowOf(t, open, assigned.ID).State != inventory.PendingApplying {
		t.Fatal("a fresh claim was settled")
	}
	settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute))
	if got := rowOf(t, open, assigned.ID); got.State != inventory.PendingApplied {
		t.Fatalf("a stale claim of an assigned module: %+v", got)
	}
	// Released to waiting, and in the same tick made, since sensors is registered now.
	if got := rowOf(t, open, notAssigned.ID); got.State != inventory.PendingApplied ||
		!slices.Contains(assignedTo(t, open, "laptop"), "sensors") {
		t.Fatalf("a stale claim of a module not assigned: %+v", got)
	}
}

// **A waited build whose asker stopped waiting is asked, outcome unknown** (second review): it may still run,
// so it reads as in flight until its outcome or its bound, never as not asked; a build never handed over
// is not asked.
func TestABuildNoLongerWaitedForIsStillInFlight(t *testing.T) {
	open := aCatalogueMesh(t)
	ctx := t.Context()
	inv := open.inventory
	for _, id := range []string{"build-timeout", "build-nothandedover"} {
		dir := "modules/" + strings.TrimPrefix(id, "build-")
		if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-catalog",
			Seat: "git", Path: dir, Ref: "main", For: "build"}); err != nil {
			t.Fatal(err)
		}
	}
	markWaitFailed(ctx, "build-timeout", errors.New("no build machine answered within 10m0s"))
	markWaitFailed(ctx, "build-nothandedover", fmt.Errorf("%w: cannot submit a build: nats: timeout", link.ErrNotHandedOver))

	said, err := assign(ctx, open, "laptop", "timeout")
	if err != nil || !strings.Contains(said, "being built") || !strings.Contains(said, "kept as pending") {
		t.Fatalf("a build no longer waited for: %q %v", said, err)
	}
	if _, err := assign(ctx, open, "laptop", "nothandedover"); err == nil || !strings.Contains(err.Error(), "was not handed over") {
		t.Fatalf("a build never handed over: %v", err)
	}
	settlePending(ctx, open, time.Now().Add(buildRequestBound+time.Minute))
	rows, _ := inv.PendingFor(ctx, "laptop", "timeout")
	if len(rows) != 1 || rows[0].State != inventory.PendingExpired || !strings.Contains(rows[0].Note, "its asker stopped waiting") {
		t.Fatalf("past the bound: %+v", rows)
	}
}